From 237d8d0c51a731c4490cc3672bf5d0a890dd1668 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 1 Apr 2025 19:38:15 +0530 Subject: [PATCH 01/37] changes around ciHandler --- api/restHandler/CoreAppRestHandler.go | 3 +- client/cron/CiTriggerCron.go | 4 +- client/events/EventBuilder.go | 13 +- client/events/EventClient.go | 60 ++---- cmd/external-app/wire_gen.go | 2 +- .../pipelineConfig/CdWorfkflowRepository.go | 19 -- pkg/bean/app.go | 101 ++++----- .../pipeline/bean/CIPipelineMaterialBean.go | 30 +++ pkg/build/pipeline/bean/CiBuildConfig.go | 26 --- .../pipeline/bean/common/CommonBuildBean.go | 27 +++ pkg/build/trigger/Service.go | 1 + pkg/pipeline/BuildPipelineConfigService.go | 29 +-- pkg/pipeline/BuildPipelineSwitchService.go | 20 +- pkg/pipeline/CIHandler_ent.go | 31 +++ pkg/pipeline/CdHandler.go | 9 +- pkg/pipeline/CiCdPipelineOrchestrator.go | 5 +- pkg/pipeline/CiHandler.go | 198 +++++++++--------- pkg/pipeline/CiService.go | 30 ++- pkg/pipeline/WorkflowService.go | 4 +- pkg/pipeline/adapter/adapter.go | 7 +- pkg/pipeline/bean/CdHandlerBean.go | 57 ++--- pkg/pipeline/constants/constants.go | 1 - pkg/pipeline/types/CiCdConfig.go | 24 ++- pkg/pipeline/types/Workflow.go | 63 +++--- pkg/workflow/dag/WorkflowDagExecutor.go | 5 +- wire_gen.go | 2 +- 26 files changed, 404 insertions(+), 367 deletions(-) create mode 100644 pkg/build/pipeline/bean/CIPipelineMaterialBean.go create mode 100644 pkg/build/pipeline/bean/common/CommonBuildBean.go create mode 100644 pkg/build/trigger/Service.go create mode 100644 pkg/pipeline/CIHandler_ent.go diff --git a/api/restHandler/CoreAppRestHandler.go b/api/restHandler/CoreAppRestHandler.go index bb8581dcfc..14d3556db5 100644 --- a/api/restHandler/CoreAppRestHandler.go +++ b/api/restHandler/CoreAppRestHandler.go @@ -29,6 +29,7 @@ import ( "github.com/devtron-labs/devtron/pkg/build/git/gitProvider" "github.com/devtron-labs/devtron/pkg/build/git/gitProvider/read" pipelineBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + common2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" bean3 "github.com/devtron-labs/devtron/pkg/chart/bean" read5 "github.com/devtron-labs/devtron/pkg/chart/read" "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" @@ -1688,7 +1689,7 @@ func (handler CoreAppRestHandlerImpl) createCiPipeline(appId int, userId int32, ParentCiPipeline: ciPipelineData.ParentCiPipeline, ParentAppId: ciPipelineData.ParentAppId, LinkedCount: ciPipelineData.LinkedCount, - PipelineType: pipelineBean.PipelineType(ciPipelineData.PipelineType), + PipelineType: common2.PipelineType(ciPipelineData.PipelineType), }, } diff --git a/client/cron/CiTriggerCron.go b/client/cron/CiTriggerCron.go index 3ded91c038..35f1e9876a 100644 --- a/client/cron/CiTriggerCron.go +++ b/client/cron/CiTriggerCron.go @@ -22,7 +22,7 @@ import ( repository2 "github.com/devtron-labs/devtron/internal/sql/repository" bean2 "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/bean" - pipelineConfigBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/repository" repository3 "github.com/devtron-labs/devtron/pkg/plugin/repository" @@ -101,7 +101,7 @@ func (impl *CiTriggerCronImpl) TriggerCiCron() { CiPipelineMaterial: ciPipelineMaterials, TriggeredBy: bean2.SYSTEM_USER_ID, InvalidateCache: false, - PipelineType: string(pipelineConfigBean.CI_JOB), + PipelineType: string(common.CI_JOB), } _, err = impl.ciHandler.HandleCIManual(ciTriggerRequest) if err != nil { diff --git a/client/events/EventBuilder.go b/client/events/EventBuilder.go index 83f7186811..4464d7aa98 100644 --- a/client/events/EventBuilder.go +++ b/client/events/EventBuilder.go @@ -19,6 +19,7 @@ package client import ( "context" "fmt" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" repository4 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "strings" "time" @@ -38,7 +39,7 @@ import ( type EventFactory interface { Build(eventType util.EventType, sourceId *int, appId int, envId *int, pipelineType util.PipelineType) (Event, error) BuildExtraCDData(event Event, wfr *pipelineConfig.CdWorkflowRunner, pipelineOverrideId int, stage bean2.WorkflowType) Event - BuildExtraCIData(event Event, material *MaterialTriggerInfo) Event + BuildExtraCIData(event Event, material *buildBean.MaterialTriggerInfo) Event //BuildFinalData(event Event) *Payload } @@ -173,7 +174,7 @@ func (impl *EventSimpleFactoryImpl) BuildExtraCDData(event Event, wfr *pipelineC return event } -func (impl *EventSimpleFactoryImpl) BuildExtraCIData(event Event, material *MaterialTriggerInfo) Event { +func (impl *EventSimpleFactoryImpl) BuildExtraCIData(event Event, material *buildBean.MaterialTriggerInfo) Event { if material == nil { materialInfo, err := impl.getCiMaterialInfo(event.PipelineId, event.CiArtifactId) if err != nil { @@ -207,8 +208,8 @@ func (impl *EventSimpleFactoryImpl) BuildExtraCIData(event Event, material *Mate return event } -func (impl *EventSimpleFactoryImpl) getCiMaterialInfo(ciPipelineId int, ciArtifactId int) (*MaterialTriggerInfo, error) { - materialTriggerInfo := &MaterialTriggerInfo{} +func (impl *EventSimpleFactoryImpl) getCiMaterialInfo(ciPipelineId int, ciArtifactId int) (*buildBean.MaterialTriggerInfo, error) { + materialTriggerInfo := &buildBean.MaterialTriggerInfo{} if ciPipelineId > 0 { ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(ciPipelineId) if err != nil { @@ -216,13 +217,13 @@ func (impl *EventSimpleFactoryImpl) getCiMaterialInfo(ciPipelineId int, ciArtifa return nil, err } - var ciMaterialsArr []CiPipelineMaterialResponse + var ciMaterialsArr []buildBean.CiPipelineMaterialResponse for _, m := range ciMaterials { if m.GitMaterial == nil { impl.logger.Warnw("git material are empty", "material", m) continue } - res := CiPipelineMaterialResponse{ + res := buildBean.CiPipelineMaterialResponse{ Id: m.Id, GitMaterialId: m.GitMaterialId, GitMaterialName: m.GitMaterial.Name[strings.Index(m.GitMaterial.Name, "-")+1:], diff --git a/client/events/EventClient.go b/client/events/EventClient.go index a58cadea82..e2ddcd4886 100644 --- a/client/events/EventClient.go +++ b/client/events/EventClient.go @@ -21,20 +21,18 @@ import ( "encoding/json" "errors" "fmt" - bean2 "github.com/devtron-labs/devtron/pkg/attributes/bean" - "github.com/devtron-labs/devtron/pkg/module" - bean3 "github.com/devtron-labs/devtron/pkg/module/bean" - "net/http" - "time" - "github.com/caarlos0/env" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + bean2 "github.com/devtron-labs/devtron/pkg/attributes/bean" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/module" + bean3 "github.com/devtron-labs/devtron/pkg/module/bean" util "github.com/devtron-labs/devtron/util/event" "go.uber.org/zap" + "net/http" ) type EventClientConfig struct { @@ -81,41 +79,19 @@ type Event struct { } type Payload struct { - AppName string `json:"appName"` - EnvName string `json:"envName"` - PipelineName string `json:"pipelineName"` - Source string `json:"source"` - DockerImageUrl string `json:"dockerImageUrl"` - TriggeredBy string `json:"triggeredBy"` - Stage string `json:"stage"` - DeploymentHistoryLink string `json:"deploymentHistoryLink"` - AppDetailLink string `json:"appDetailLink"` - DownloadLink string `json:"downloadLink"` - BuildHistoryLink string `json:"buildHistoryLink"` - MaterialTriggerInfo *MaterialTriggerInfo `json:"material"` - FailureReason string `json:"failureReason"` -} - -type CiPipelineMaterialResponse struct { - Id int `json:"id"` - GitMaterialId int `json:"gitMaterialId"` - GitMaterialUrl string `json:"gitMaterialUrl"` - GitMaterialName string `json:"gitMaterialName"` - Type string `json:"type"` - Value string `json:"value"` - Active bool `json:"active"` - History []*gitSensor.GitCommit `json:"history,omitempty"` - LastFetchTime time.Time `json:"lastFetchTime"` - IsRepoError bool `json:"isRepoError"` - RepoErrorMsg string `json:"repoErrorMsg"` - IsBranchError bool `json:"isBranchError"` - BranchErrorMsg string `json:"branchErrorMsg"` - Url string `json:"url"` -} - -type MaterialTriggerInfo struct { - GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` - CiMaterials []CiPipelineMaterialResponse `json:"ciMaterials"` + AppName string `json:"appName"` + EnvName string `json:"envName"` + PipelineName string `json:"pipelineName"` + Source string `json:"source"` + DockerImageUrl string `json:"dockerImageUrl"` + TriggeredBy string `json:"triggeredBy"` + Stage string `json:"stage"` + DeploymentHistoryLink string `json:"deploymentHistoryLink"` + AppDetailLink string `json:"appDetailLink"` + DownloadLink string `json:"downloadLink"` + BuildHistoryLink string `json:"buildHistoryLink"` + MaterialTriggerInfo *buildBean.MaterialTriggerInfo `json:"material"` + FailureReason string `json:"failureReason"` } type EventRESTClientImpl struct { diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index cc31476cbe..e827637c0a 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index a737a7076a..b0708ad99e 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -20,7 +20,6 @@ import ( "context" "errors" apiBean "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" @@ -149,24 +148,6 @@ func (c *CdWorkflowRunner) IsExternalRun() bool { return isExtCluster } -type CiPipelineMaterialResponse struct { - Id int `json:"id"` - GitMaterialId int `json:"gitMaterialId"` - GitMaterialUrl string `json:"gitMaterialUrl"` - GitMaterialName string `json:"gitMaterialName"` - Type string `json:"type"` - Value string `json:"value"` - Active bool `json:"active"` - History []*gitSensor.GitCommit `json:"history,omitempty"` - LastFetchTime time.Time `json:"lastFetchTime"` - IsRepoError bool `json:"isRepoError"` - RepoErrorMsg string `json:"repoErrorMsg"` - IsBranchError bool `json:"isBranchError"` - BranchErrorMsg string `json:"branchErrorMsg"` - Url string `json:"url"` - Regex string `json:"regex"` -} - type TriggerWorkflowStatus struct { CdWorkflowStatus []*CdWorkflowStatus `json:"cdWorkflowStatus"` CiWorkflowStatus []*CiWorkflowStatus `json:"ciWorkflowStatus"` diff --git a/pkg/bean/app.go b/pkg/bean/app.go index 96c581b5f6..83c6717093 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -29,6 +29,7 @@ import ( "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/bean/common" CiPipeline2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + common2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" "github.com/devtron-labs/devtron/pkg/chartRepo/repository" bean3 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/pipeline/bean" @@ -117,39 +118,39 @@ type CiMaterial struct { } type CiPipeline struct { - IsManual bool `json:"isManual"` - DockerArgs map[string]string `json:"dockerArgs"` - IsExternal bool `json:"isExternal"` - ParentCiPipeline int `json:"parentCiPipeline"` - ParentAppId int `json:"parentAppId"` - AppId int `json:"appId"` - AppName string `json:"appName,omitempty"` - AppType helper.AppType `json:"appType,omitempty"` - ExternalCiConfig ExternalCiConfig `json:"externalCiConfig"` - CiMaterial []*CiMaterial `json:"ciMaterial,omitempty" validate:"dive,min=1"` - Name string `json:"name,omitempty" validate:"name-component,max=100"` // name suffix of corresponding pipeline. required, unique, validation corresponding to gocd pipelineName will be applicable - Id int `json:"id,omitempty" ` - Version string `json:"version,omitempty"` // matchIf token version in gocd . used for update request - Active bool `json:"active,omitempty"` // pipeline is active or not - Deleted bool `json:"deleted,omitempty"` - BeforeDockerBuild []*Task `json:"beforeDockerBuild,omitempty" validate:"dive"` - AfterDockerBuild []*Task `json:"afterDockerBuild,omitempty" validate:"dive"` - BeforeDockerBuildScripts []*CiScript `json:"beforeDockerBuildScripts,omitempty" validate:"dive"` - AfterDockerBuildScripts []*CiScript `json:"afterDockerBuildScripts,omitempty" validate:"dive"` - LinkedCount int `json:"linkedCount"` - PipelineType CiPipeline2.PipelineType `json:"pipelineType,omitempty"` - ScanEnabled bool `json:"scanEnabled,notnull"` - AppWorkflowId int `json:"appWorkflowId,omitempty"` - PreBuildStage *bean.PipelineStageDto `json:"preBuildStage,omitempty" validate:"omitempty,dive"` - PostBuildStage *bean.PipelineStageDto `json:"postBuildStage,omitempty" validate:"omitempty,dive"` - TargetPlatform string `json:"targetPlatform,omitempty"` - IsDockerConfigOverridden bool `json:"isDockerConfigOverridden"` - DockerConfigOverride DockerConfigOverride `json:"dockerConfigOverride,omitempty"` - EnvironmentId int `json:"environmentId,omitempty"` - LastTriggeredEnvId int `json:"lastTriggeredEnvId"` - CustomTagObject *CustomTagData `json:"customTag,omitempty"` - DefaultTag []string `json:"defaultTag,omitempty"` - EnableCustomTag bool `json:"enableCustomTag"` + IsManual bool `json:"isManual"` + DockerArgs map[string]string `json:"dockerArgs"` + IsExternal bool `json:"isExternal"` + ParentCiPipeline int `json:"parentCiPipeline"` + ParentAppId int `json:"parentAppId"` + AppId int `json:"appId"` + AppName string `json:"appName,omitempty"` + AppType helper.AppType `json:"appType,omitempty"` + ExternalCiConfig ExternalCiConfig `json:"externalCiConfig"` + CiMaterial []*CiMaterial `json:"ciMaterial,omitempty" validate:"dive,min=1"` + Name string `json:"name,omitempty" validate:"name-component,max=100"` // name suffix of corresponding pipeline. required, unique, validation corresponding to gocd pipelineName will be applicable + Id int `json:"id,omitempty" ` + Version string `json:"version,omitempty"` // matchIf token version in gocd . used for update request + Active bool `json:"active,omitempty"` // pipeline is active or not + Deleted bool `json:"deleted,omitempty"` + BeforeDockerBuild []*Task `json:"beforeDockerBuild,omitempty" validate:"dive"` + AfterDockerBuild []*Task `json:"afterDockerBuild,omitempty" validate:"dive"` + BeforeDockerBuildScripts []*CiScript `json:"beforeDockerBuildScripts,omitempty" validate:"dive"` + AfterDockerBuildScripts []*CiScript `json:"afterDockerBuildScripts,omitempty" validate:"dive"` + LinkedCount int `json:"linkedCount"` + PipelineType common2.PipelineType `json:"pipelineType,omitempty"` + ScanEnabled bool `json:"scanEnabled,notnull"` + AppWorkflowId int `json:"appWorkflowId,omitempty"` + PreBuildStage *bean.PipelineStageDto `json:"preBuildStage,omitempty" validate:"omitempty,dive"` + PostBuildStage *bean.PipelineStageDto `json:"postBuildStage,omitempty" validate:"omitempty,dive"` + TargetPlatform string `json:"targetPlatform,omitempty"` + IsDockerConfigOverridden bool `json:"isDockerConfigOverridden"` + DockerConfigOverride DockerConfigOverride `json:"dockerConfigOverride,omitempty"` + EnvironmentId int `json:"environmentId,omitempty"` + LastTriggeredEnvId int `json:"lastTriggeredEnvId"` + CustomTagObject *CustomTagData `json:"customTag,omitempty"` + DefaultTag []string `json:"defaultTag,omitempty"` + EnableCustomTag bool `json:"enableCustomTag"` } func (ciPipeline *CiPipeline) IsLinkedCi() bool { @@ -164,14 +165,14 @@ type DockerConfigOverride struct { } type CiPipelineMin struct { - Name string `json:"name,omitempty" validate:"name-component,max=100"` //name suffix of corresponding pipeline. required, unique, validation corresponding to gocd pipelineName will be applicable - Id int `json:"id,omitempty" ` - Version string `json:"version,omitempty"` //matchIf token version in gocd . used for update request - IsExternal bool `json:"isExternal,omitempty"` - ParentCiPipeline int `json:"parentCiPipeline"` - ParentAppId int `json:"parentAppId"` - PipelineType CiPipeline2.PipelineType `json:"pipelineType,omitempty"` - ScanEnabled bool `json:"scanEnabled,notnull"` + Name string `json:"name,omitempty" validate:"name-component,max=100"` //name suffix of corresponding pipeline. required, unique, validation corresponding to gocd pipelineName will be applicable + Id int `json:"id,omitempty" ` + Version string `json:"version,omitempty"` //matchIf token version in gocd . used for update request + IsExternal bool `json:"isExternal,omitempty"` + ParentCiPipeline int `json:"parentCiPipeline"` + ParentAppId int `json:"parentAppId"` + PipelineType common2.PipelineType `json:"pipelineType,omitempty"` + ScanEnabled bool `json:"scanEnabled,notnull"` } type CiScript struct { @@ -303,20 +304,20 @@ type CiPatchRequest struct { IsJob bool `json:"-"` IsCloneJob bool `json:"isCloneJob,omitempty"` - ParentCDPipeline int `json:"parentCDPipeline"` - DeployEnvId int `json:"deployEnvId"` - SwitchFromCiPipelineId int `json:"switchFromCiPipelineId"` - SwitchFromExternalCiPipelineId int `json:"switchFromExternalCiPipelineId"` - SwitchFromCiPipelineType CiPipeline2.PipelineType `json:"-"` - SwitchToCiPipelineType CiPipeline2.PipelineType `json:"-"` + ParentCDPipeline int `json:"parentCDPipeline"` + DeployEnvId int `json:"deployEnvId"` + SwitchFromCiPipelineId int `json:"switchFromCiPipelineId"` + SwitchFromExternalCiPipelineId int `json:"switchFromExternalCiPipelineId"` + SwitchFromCiPipelineType common2.PipelineType `json:"-"` + SwitchToCiPipelineType common2.PipelineType `json:"-"` } -func (ciPatchRequest CiPatchRequest) SwitchSourceInfo() (int, CiPipeline2.PipelineType) { +func (ciPatchRequest CiPatchRequest) SwitchSourceInfo() (int, common2.PipelineType) { // get the ciPipeline - var switchFromType CiPipeline2.PipelineType + var switchFromType common2.PipelineType var switchFromPipelineId int if ciPatchRequest.SwitchFromExternalCiPipelineId != 0 { - switchFromType = CiPipeline2.EXTERNAL + switchFromType = common2.EXTERNAL switchFromPipelineId = ciPatchRequest.SwitchFromExternalCiPipelineId } else { switchFromPipelineId = ciPatchRequest.SwitchFromCiPipelineId diff --git a/pkg/build/pipeline/bean/CIPipelineMaterialBean.go b/pkg/build/pipeline/bean/CIPipelineMaterialBean.go new file mode 100644 index 0000000000..8f09c47ebe --- /dev/null +++ b/pkg/build/pipeline/bean/CIPipelineMaterialBean.go @@ -0,0 +1,30 @@ +package bean + +import ( + "github.com/devtron-labs/devtron/client/gitSensor" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "time" +) + +type CiPipelineMaterialResponse struct { + Id int `json:"id"` + GitMaterialId int `json:"gitMaterialId"` + GitMaterialUrl string `json:"gitMaterialUrl"` + GitMaterialName string `json:"gitMaterialName"` + Type string `json:"type"` + Value string `json:"value"` + Active bool `json:"active"` + History []*gitSensor.GitCommit `json:"history,omitempty"` + LastFetchTime time.Time `json:"lastFetchTime"` + IsRepoError bool `json:"isRepoError"` + RepoErrorMsg string `json:"repoErrorMsg"` + IsBranchError bool `json:"isBranchError"` + BranchErrorMsg string `json:"branchErrorMsg"` + Url string `json:"url"` + Regex string `json:"regex"` +} + +type MaterialTriggerInfo struct { + GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` + CiMaterials []CiPipelineMaterialResponse `json:"ciMaterials"` +} diff --git a/pkg/build/pipeline/bean/CiBuildConfig.go b/pkg/build/pipeline/bean/CiBuildConfig.go index 34357f5e68..39d8671a35 100644 --- a/pkg/build/pipeline/bean/CiBuildConfig.go +++ b/pkg/build/pipeline/bean/CiBuildConfig.go @@ -30,23 +30,6 @@ const UniquePlaceHolderForAppName = "$etron" const PIPELINE_NAME_ALREADY_EXISTS_ERROR = "pipeline name already exist" const PIPELINE_TYPE_IS_NOT_VALID = "PipelineType is not valid for pipeline %s" -type PipelineType string - -// default PipelineType -const DefaultPipelineType = CI_BUILD - -const ( - CI_BUILD PipelineType = "CI_BUILD" - LINKED PipelineType = "LINKED" - EXTERNAL PipelineType = "EXTERNAL" - CI_JOB PipelineType = "CI_JOB" - LINKED_CD PipelineType = "LINKED_CD" -) - -func (pType PipelineType) ToString() string { - return string(pType) -} - type CiBuildConfigBean struct { Id int `json:"id"` GitMaterialId int `json:"gitMaterialId,omitempty" validate:"required"` @@ -81,15 +64,6 @@ type BuildPackConfig struct { ProjectPath string `json:"projectPath,omitempty"` } -func (pType PipelineType) IsValidPipelineType() bool { - switch pType { - case CI_BUILD, LINKED, EXTERNAL, CI_JOB, LINKED_CD: - return true - default: - return false - } -} - const ( ExtraEnvVarExternalCiArtifactKey = "externalCiArtifact" ExtraEnvVarImageDigestKey = "imageDigest" diff --git a/pkg/build/pipeline/bean/common/CommonBuildBean.go b/pkg/build/pipeline/bean/common/CommonBuildBean.go new file mode 100644 index 0000000000..af419f962f --- /dev/null +++ b/pkg/build/pipeline/bean/common/CommonBuildBean.go @@ -0,0 +1,27 @@ +package common + +type PipelineType string + +// default PipelineType +const DefaultPipelineType = CI_BUILD + +const ( + CI_BUILD PipelineType = "CI_BUILD" + LINKED PipelineType = "LINKED" + EXTERNAL PipelineType = "EXTERNAL" + CI_JOB PipelineType = "CI_JOB" + LINKED_CD PipelineType = "LINKED_CD" +) + +func (pType PipelineType) ToString() string { + return string(pType) +} + +func (pType PipelineType) IsValidPipelineType() bool { + switch pType { + case CI_BUILD, LINKED, EXTERNAL, CI_JOB, LINKED_CD: + return true + default: + return false + } +} diff --git a/pkg/build/trigger/Service.go b/pkg/build/trigger/Service.go new file mode 100644 index 0000000000..dee4fe7ace --- /dev/null +++ b/pkg/build/trigger/Service.go @@ -0,0 +1 @@ +package trigger diff --git a/pkg/pipeline/BuildPipelineConfigService.go b/pkg/pipeline/BuildPipelineConfigService.go index fca115326e..95cd8775ac 100644 --- a/pkg/pipeline/BuildPipelineConfigService.go +++ b/pkg/pipeline/BuildPipelineConfigService.go @@ -35,6 +35,7 @@ import ( repository3 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/repository" "github.com/devtron-labs/devtron/pkg/build/pipeline" bean3 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" "github.com/devtron-labs/devtron/pkg/build/pipeline/read" pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/history" @@ -302,11 +303,11 @@ func (impl *CiPipelineConfigServiceImpl) patchCiPipelineUpdateSource(baseCiConfi } // updating PipelineType from db if not present in request if modifiedCiPipeline.PipelineType == "" { - if bean3.PipelineType(pipeline.PipelineType) != "" { - modifiedCiPipeline.PipelineType = bean3.PipelineType(pipeline.PipelineType) + if common.PipelineType(pipeline.PipelineType) != "" { + modifiedCiPipeline.PipelineType = common.PipelineType(pipeline.PipelineType) } else { // updating default pipelineType if not present in request - modifiedCiPipeline.PipelineType = bean3.DefaultPipelineType + modifiedCiPipeline.PipelineType = common.DefaultPipelineType } } @@ -617,7 +618,7 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipeline(appId int) (ciConfig *bea AfterDockerBuildScripts: afterDockerBuildScripts, ScanEnabled: pipeline.ScanEnabled, IsDockerConfigOverridden: pipeline.IsDockerConfigOverridden, - PipelineType: bean3.PipelineType(pipeline.PipelineType), + PipelineType: common.PipelineType(pipeline.PipelineType), } ciEnvMapping, err := impl.ciPipelineRepository.FindCiEnvMappingByCiPipelineId(pipeline.Id) if err != nil && err != pg.ErrNoRows { @@ -773,7 +774,7 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipelineById(pipelineId int) (ciPi AfterDockerBuildScripts: afterDockerBuildScripts, ScanEnabled: pipeline.ScanEnabled, IsDockerConfigOverridden: pipeline.IsDockerConfigOverridden, - PipelineType: bean3.PipelineType(pipeline.PipelineType), + PipelineType: common.PipelineType(pipeline.PipelineType), } customTag, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id)) if err != nil && err != pg.ErrNoRows { @@ -897,7 +898,7 @@ func (impl *CiPipelineConfigServiceImpl) GetTriggerViewCiPipeline(appId int) (*b ParentCiPipeline: pipeline.ParentCiPipeline, ScanEnabled: pipeline.ScanEnabled, IsDockerConfigOverridden: pipeline.IsDockerConfigOverridden, - PipelineType: bean3.PipelineType(pipeline.PipelineType), + PipelineType: common.PipelineType(pipeline.PipelineType), } if ciTemplateBean, ok := ciOverrideTemplateMap[pipeline.Id]; ok { templateOverride := ciTemplateBean.CiTemplateOverride @@ -1263,7 +1264,7 @@ func (impl *CiPipelineConfigServiceImpl) UpdateCiTemplate(updateRequest *bean.Ci } for _, ciTemplateOverride := range ciTemplateOverrides { if _, ok := ciPipelineIdsMap[ciTemplateOverride.CiPipelineId]; ok { - if ciPipelineIdsMap[ciTemplateOverride.CiPipelineId].PipelineType == string(bean3.CI_JOB) { + if ciPipelineIdsMap[ciTemplateOverride.CiPipelineId].PipelineType == string(common.CI_JOB) { ciTemplateOverride.DockerRepository = updateRequest.DockerRepository ciTemplateOverride.DockerRegistryId = updateRequest.DockerRegistry _, err = impl.ciTemplateOverrideRepository.Update(ciTemplateOverride) @@ -1319,7 +1320,7 @@ func (impl *CiPipelineConfigServiceImpl) PatchCiPipeline(request *bean.CiPatchRe impl.logger.Errorw("err in fetching template for pipeline patch, ", "err", err, "appId", request.AppId) return nil, err } - if request.CiPipeline.PipelineType == bean3.CI_JOB { + if request.CiPipeline.PipelineType == common.CI_JOB { request.CiPipeline.IsDockerConfigOverridden = true request.CiPipeline.DockerConfigOverride = bean.DockerConfigOverride{ DockerRegistry: ciConfig.DockerRegistry, @@ -1512,15 +1513,15 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipelineMin(appId int, envIds []in var ciPipelineResp []*bean.CiPipelineMin for _, pipeline := range pipelines { parentCiPipeline := pipelineConfig.CiPipeline{} - pipelineType := bean3.CI_BUILD + pipelineType := common.CI_BUILD if pipelineParentCiMap[pipeline.Id] != nil { parentCiPipeline = *pipelineParentCiMap[pipeline.Id] - pipelineType = bean3.LINKED + pipelineType = common.LINKED } else if pipeline.IsExternal == true { - pipelineType = bean3.EXTERNAL - } else if pipeline.PipelineType == string(bean3.CI_JOB) { - pipelineType = bean3.CI_JOB + pipelineType = common.EXTERNAL + } else if pipeline.PipelineType == string(common.CI_JOB) { + pipelineType = common.CI_JOB } ciPipeline := &bean.CiPipelineMin{ @@ -1736,7 +1737,7 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipelineByEnvironment(request reso ExternalCiConfig: externalCiConfig, ScanEnabled: pipeline.ScanEnabled, IsDockerConfigOverridden: pipeline.IsDockerConfigOverridden, - PipelineType: bean3.PipelineType(pipeline.PipelineType), + PipelineType: common.PipelineType(pipeline.PipelineType), } parentPipelineAppId, ok := pipelineIdVsAppId[parentCiPipelineId] if ok { diff --git a/pkg/pipeline/BuildPipelineSwitchService.go b/pkg/pipeline/BuildPipelineSwitchService.go index 076d95a12e..858f03473c 100644 --- a/pkg/pipeline/BuildPipelineSwitchService.go +++ b/pkg/pipeline/BuildPipelineSwitchService.go @@ -21,7 +21,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/bean" - pipelineConfigBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" "github.com/devtron-labs/devtron/pkg/build/pipeline/read" "github.com/devtron-labs/devtron/pkg/pipeline/adapter" "github.com/devtron-labs/devtron/pkg/pipeline/history" @@ -124,10 +124,10 @@ func (impl *BuildPipelineSwitchServiceImpl) SwitchToCiPipelineExceptExternal(req } //get the ciPipeline - var switchFromType pipelineConfigBean.PipelineType + var switchFromType common.PipelineType var switchFromPipelineId int if request.SwitchFromExternalCiPipelineId != 0 { - switchFromType = pipelineConfigBean.EXTERNAL + switchFromType = common.EXTERNAL switchFromPipelineId = request.SwitchFromExternalCiPipelineId } else { switchFromPipelineId = request.SwitchFromCiPipelineId @@ -145,9 +145,9 @@ func (impl *BuildPipelineSwitchServiceImpl) SwitchToCiPipelineExceptExternal(req return impl.createNewPipelineAndReplaceOldPipelineLinks(request.CiPipeline, ciConfig, switchFromPipelineId, switchFromType, request.UserId) } -func (impl *BuildPipelineSwitchServiceImpl) createNewPipelineAndReplaceOldPipelineLinks(ciPipelineReq *bean.CiPipeline, ciConfig *bean.CiConfigRequest, switchFromPipelineId int, switchFromType pipelineConfigBean.PipelineType, userId int32) (*bean.CiConfigRequest, error) { +func (impl *BuildPipelineSwitchServiceImpl) createNewPipelineAndReplaceOldPipelineLinks(ciPipelineReq *bean.CiPipeline, ciConfig *bean.CiConfigRequest, switchFromPipelineId int, switchFromType common.PipelineType, userId int32) (*bean.CiConfigRequest, error) { - isSelfLinkedCiPipeline := switchFromType != pipelineConfigBean.EXTERNAL && ciPipelineReq.IsLinkedCi() && ciPipelineReq.ParentCiPipeline == switchFromPipelineId + isSelfLinkedCiPipeline := switchFromType != common.EXTERNAL && ciPipelineReq.IsLinkedCi() && ciPipelineReq.ParentCiPipeline == switchFromPipelineId if isSelfLinkedCiPipeline { errMsg := "cannot create linked ci pipeline from the same source" return nil, util.DefaultApiError().WithInternalMessage(errMsg).WithUserMessage(errMsg).WithHttpStatusCode(http.StatusBadRequest) @@ -208,7 +208,7 @@ func (impl *BuildPipelineSwitchServiceImpl) createNewPipelineAndReplaceOldPipeli // add switchType and remove other id // make constants for error msgs -func (impl *BuildPipelineSwitchServiceImpl) validateCiPipelineSwitch(switchFromCiPipelineId int, switchToType, switchFromType pipelineConfigBean.PipelineType) error { +func (impl *BuildPipelineSwitchServiceImpl) validateCiPipelineSwitch(switchFromCiPipelineId int, switchToType, switchFromType common.PipelineType) error { // this will only allow below conversions // ext -> {ci_job,direct,linked} // direct -> {ci_job,linked} @@ -220,13 +220,13 @@ func (impl *BuildPipelineSwitchServiceImpl) validateCiPipelineSwitch(switchFromC } // refer SwitchToExternalCi - if switchToType == pipelineConfigBean.EXTERNAL { + if switchToType == common.EXTERNAL { return errors.New(string(cannotConvertToExternalCi)) } // we should not check the below logic for external_ci type as builds are not built in devtron and // linked pipelines won't be there as per current external-ci-pipeline architecture - if switchFromCiPipelineId > 0 && switchFromType != pipelineConfigBean.EXTERNAL { + if switchFromCiPipelineId > 0 && switchFromType != common.EXTERNAL { err := impl.validateSwitchPreConditions(switchFromCiPipelineId) if err != nil { return err @@ -252,13 +252,13 @@ func (impl *BuildPipelineSwitchServiceImpl) deleteCiAndItsWorkflowMappings(tx *p return err } -func (impl *BuildPipelineSwitchServiceImpl) deleteOldCiPipelineAndWorkflowMappingBeforeSwitch(tx *pg.Tx, switchFromPipelineId int, switchFromType pipelineConfigBean.PipelineType, userId int32) (*appWorkflow.AppWorkflowMapping, error) { +func (impl *BuildPipelineSwitchServiceImpl) deleteOldCiPipelineAndWorkflowMappingBeforeSwitch(tx *pg.Tx, switchFromPipelineId int, switchFromType common.PipelineType, userId int32) (*appWorkflow.AppWorkflowMapping, error) { // 1) delete build pipelines // 2) delete app workflowMappings var err error pipelineId := switchFromPipelineId pipelineType := "" - if switchFromType == pipelineConfigBean.EXTERNAL { + if switchFromType == common.EXTERNAL { err = impl.deleteExternalCi(tx, switchFromPipelineId, userId) pipelineType = appWorkflow.WEBHOOK } else { diff --git a/pkg/pipeline/CIHandler_ent.go b/pkg/pipeline/CIHandler_ent.go new file mode 100644 index 0000000000..7e1986b518 --- /dev/null +++ b/pkg/pipeline/CIHandler_ent.go @@ -0,0 +1,31 @@ +package pipeline + +import ( + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/pkg/bean" + "github.com/devtron-labs/devtron/pkg/bean/common" +) + +type CiHandlerEnt interface { +} + +func (impl *CiHandlerImpl) updateRuntimeParamsForAutoCI(ciPipelineId int, runtimeParameters *common.RuntimeParameters) (*common.RuntimeParameters, error) { + return runtimeParameters, nil +} + +func (impl *CiHandlerImpl) updateResourceStatusInCache(ciWorkflowId int, podName string, namespace string, status string) { + //do nothing +} + +func (impl *CiHandlerImpl) getRuntimeParamsForBuildingManualTriggerHashes(ciTriggerRequest bean.CiTriggerRequest) *common.RuntimeParameters { + return common.NewRuntimeParameters() +} + +func (impl *CiHandlerImpl) getPipelineIdForTriggerView(pipeline *pipelineConfig.CiPipeline) (pipelineId int) { + if pipeline.ParentCiPipeline == 0 { + pipelineId = pipeline.Id + } else { + pipelineId = pipeline.ParentCiPipeline + } + return pipelineId +} diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index ba2618db49..230d35c522 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -26,6 +26,7 @@ import ( cdWorkflow2 "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" bean2 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/build/artifacts/imageTagging" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" "github.com/devtron-labs/devtron/pkg/cluster/adapter" bean3 "github.com/devtron-labs/devtron/pkg/cluster/bean" repository3 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" @@ -435,9 +436,9 @@ func (impl *CdHandlerImpl) GetCdBuildHistory(appId int, environmentId int, pipel return cdWorkflowArtifact, err } - var ciMaterialsArr []pipelineConfig.CiPipelineMaterialResponse + var ciMaterialsArr []buildBean.CiPipelineMaterialResponse for _, ciMaterial := range ciMaterials { - res := pipelineConfig.CiPipelineMaterialResponse{ + res := buildBean.CiPipelineMaterialResponse{ Id: ciMaterial.Id, GitMaterialId: ciMaterial.GitMaterialId, GitMaterialName: ciMaterial.GitMaterial.Name[strings.Index(ciMaterial.GitMaterial.Name, "-")+1:], @@ -689,9 +690,9 @@ func (impl *CdHandlerImpl) FetchCdWorkflowDetails(appId int, environmentId int, return types.WorkflowResponse{}, err } - var ciMaterialsArr []pipelineConfig.CiPipelineMaterialResponse + var ciMaterialsArr []buildBean.CiPipelineMaterialResponse for _, m := range ciMaterials { - res := pipelineConfig.CiPipelineMaterialResponse{ + res := buildBean.CiPipelineMaterialResponse{ Id: m.Id, GitMaterialId: m.GitMaterialId, GitMaterialName: m.GitMaterial.Name[strings.Index(m.GitMaterial.Name, "-")+1:], diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index 6be1e14ab0..313611d76c 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -32,6 +32,7 @@ import ( repository6 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/repository" "github.com/devtron-labs/devtron/pkg/build/pipeline" bean2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + common3 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" read2 "github.com/devtron-labs/devtron/pkg/chart/read" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/deployment/common" @@ -329,7 +330,7 @@ func (impl CiCdPipelineOrchestratorImpl) validateCiPipelineMaterial(ciPipelineMa func (impl CiCdPipelineOrchestratorImpl) getSkipMessage(ciPipeline *pipelineConfig.CiPipeline) string { switch ciPipeline.PipelineType { - case string(bean2.LINKED_CD): + case string(common3.LINKED_CD): return "“Sync with Environment”" default: return "“Linked Build Pipeline”" @@ -1010,7 +1011,7 @@ func (impl CiCdPipelineOrchestratorImpl) CreateCiConf(createRequest *bean.CiConf var pipelineMaterials []*pipelineConfig.CiPipelineMaterial for _, r := range ciPipeline.CiMaterial { - if ciPipeline.PipelineType == bean2.LINKED_CD { + if ciPipeline.PipelineType == common3.LINKED_CD { continue } material := &pipelineConfig.CiPipelineMaterial{ diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 5f05b28ef6..aa018f6b96 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -23,17 +23,18 @@ import ( "fmt" "github.com/devtron-labs/common-lib/utils" "github.com/devtron-labs/common-lib/utils/workFlow" - "github.com/devtron-labs/devtron/internal/sql/constants" + constants2 "github.com/devtron-labs/devtron/internal/sql/constants" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" bean6 "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/bean/common" "github.com/devtron-labs/devtron/pkg/build/artifacts/imageTagging" - bean4 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" "github.com/devtron-labs/devtron/pkg/cluster/adapter" - bean5 "github.com/devtron-labs/devtron/pkg/cluster/bean" + clusterBean "github.com/devtron-labs/devtron/pkg/cluster/bean" "github.com/devtron-labs/devtron/pkg/cluster/environment" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" - constants2 "github.com/devtron-labs/devtron/pkg/pipeline/constants" + "github.com/devtron-labs/devtron/pkg/pipeline/constants" util3 "github.com/devtron-labs/devtron/pkg/pipeline/util" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" "io/ioutil" @@ -52,11 +53,11 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" "github.com/devtron-labs/devtron/pkg/auth/user" "github.com/devtron-labs/devtron/pkg/cluster" - k8s2 "github.com/devtron-labs/devtron/pkg/k8s" - bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + k8sPkg "github.com/devtron-labs/devtron/pkg/k8s" + pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/types" - resourceGroup "github.com/devtron-labs/devtron/pkg/resourceGroup" + "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/util/rbac" errors2 "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/rest" @@ -67,7 +68,6 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/bean" - util2 "github.com/devtron-labs/devtron/util/event" "github.com/go-pg/pg" "go.uber.org/zap" ) @@ -76,16 +76,14 @@ type CiHandler interface { HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error - FetchMaterialsByPipelineId(pipelineId int, showAll bool) ([]pipelineConfig.CiPipelineMaterialResponse, error) - FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId int, gitMaterialId int, showAll bool) ([]pipelineConfig.CiPipelineMaterialResponse, error) + FetchMaterialsByPipelineId(pipelineId int, showAll bool) ([]buildBean.CiPipelineMaterialResponse, error) + FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId int, gitMaterialId int, showAll bool) ([]buildBean.CiPipelineMaterialResponse, error) FetchWorkflowDetails(appId int, pipelineId int, buildId int) (types.WorkflowResponse, error) FetchArtifactsForCiJob(buildId int) (*types.ArtifactsForCiJob, error) - //FetchBuildById(appId int, pipelineId int) (WorkflowResponse, error) CancelBuild(workflowId int, forceAbort bool) (int, error) GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, func() error, error) GetHistoricBuildLogs(workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) - //SyncWorkflows() error GetBuildHistory(pipelineId int, appId int, offset int, size int) ([]types.WorkflowResponse, error) DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) @@ -97,6 +95,7 @@ type CiHandler interface { FetchMaterialInfoByArtifactId(ciArtifactId int, envId int) (*types.GitTriggerInfoResponse, error) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error FetchCiStatusForTriggerViewForEnvironment(request resourceGroup.ResourceGroupingRequest, token string) ([]*pipelineConfig.CiWorkflowStatus, error) + CiHandlerEnt } type CiHandlerImpl struct { @@ -122,7 +121,7 @@ type CiHandlerImpl struct { customTagService CustomTagService appWorkflowRepository appWorkflow.AppWorkflowRepository config *types.CiConfig - k8sCommonService k8s2.K8sCommonService + k8sCommonService k8sPkg.K8sCommonService clusterService cluster.ClusterService blobConfigStorageService BlobStorageConfigService envService environment.EnvironmentService @@ -131,10 +130,11 @@ type CiHandlerImpl struct { func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, gitSensorClient gitSensor.Client, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, workflowService WorkflowService, ciLogService CiLogService, ciArtifactRepository repository.CiArtifactRepository, userService user.UserService, eventClient client.EventClient, eventFactory client.EventFactory, ciPipelineRepository pipelineConfig.CiPipelineRepository, - appListingRepository repository.AppListingRepository, K8sUtil *k8s.K8sServiceImpl, cdPipelineRepository pipelineConfig.PipelineRepository, enforcerUtil rbac.EnforcerUtil, resourceGroupService resourceGroup.ResourceGroupService, envRepository repository2.EnvironmentRepository, - imageTaggingService imageTagging.ImageTaggingService, k8sCommonService k8s2.K8sCommonService, clusterService cluster.ClusterService, blobConfigStorageService BlobStorageConfigService, appWorkflowRepository appWorkflow.AppWorkflowRepository, customTagService CustomTagService, - envService environment.EnvironmentService, - workFlowStageStatusService workflowStatus.WorkFlowStageStatusService) *CiHandlerImpl { + appListingRepository repository.AppListingRepository, cdPipelineRepository pipelineConfig.PipelineRepository, enforcerUtil rbac.EnforcerUtil, resourceGroupService resourceGroup.ResourceGroupService, envRepository repository2.EnvironmentRepository, + imageTaggingService imageTagging.ImageTaggingService, k8sCommonService k8sPkg.K8sCommonService, clusterService cluster.ClusterService, blobConfigStorageService BlobStorageConfigService, appWorkflowRepository appWorkflow.AppWorkflowRepository, customTagService CustomTagService, + envService environment.EnvironmentService, workFlowStageStatusService workflowStatus.WorkFlowStageStatusService, + K8sUtil *k8s.K8sServiceImpl, +) *CiHandlerImpl { cih := &CiHandlerImpl{ Logger: Logger, ciService: ciService, @@ -174,7 +174,7 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline func (impl *CiHandlerImpl) CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error { - //return if re-trigger feature is disabled + // return if re-trigger feature is disabled if !impl.config.WorkflowRetriesEnabled() { impl.Logger.Debug("CI re-trigger is disabled") return nil @@ -224,6 +224,14 @@ func (impl *CiHandlerImpl) reTriggerCi(retryCount int, refCiWorkflow *pipelineCo trigger := types.Trigger{} trigger.BuildTriggerObject(refCiWorkflow, ciMaterials, bean6.SYSTEM_USER_ID, true, nil, "") + + // updating runtime params + trigger.RuntimeParameters, err = impl.updateRuntimeParamsForAutoCI(trigger.PipelineId, trigger.RuntimeParameters) + if err != nil { + impl.Logger.Errorw("err, updateRuntimeParamsForAutoCI", "ciPipelineId", trigger.PipelineId, + "runtimeParameters", trigger.RuntimeParameters, "err", err) + return err + } _, err = impl.ciService.TriggerCiPipeline(trigger) if err != nil { @@ -277,7 +285,7 @@ func (impl *CiHandlerImpl) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTrigger impl.Logger.Errorw("err in getting ci_pipeline by ciPipelineMaterialId", "ciPipelineMaterialId", gitCiTriggerRequest.CiPipelineMaterial.Id, "err", err) return 0, err } - if ciPipeline.IsManual || ciPipeline.PipelineType == bean4.LINKED_CD.ToString() { + if ciPipeline.IsManual || ciPipeline.PipelineType == buildCommonBean.LINKED_CD.ToString() { impl.Logger.Debugw("not handling for manual pipeline or in case of linked cd", "pipelineId", ciPipeline.Id) return 0, err } @@ -297,6 +305,12 @@ func (impl *CiHandlerImpl) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTrigger for k, v := range gitCiTriggerRequest.ExtraEnvironmentVariables { runtimeParams = runtimeParams.AddSystemVariable(k, v) } + runtimeParams, err = impl.updateRuntimeParamsForAutoCI(ciPipeline.Id, runtimeParams) + if err != nil { + impl.Logger.Errorw("err, updateRuntimeParamsForAutoCI", "ciPipelineId", ciPipeline.Id, + "runtimeParameters", runtimeParams, "err", err) + return 0, err + } commitHashes, err := impl.buildAutomaticTriggerCommitHashes(ciMaterials, gitCiTriggerRequest) if err != nil { return 0, err @@ -329,7 +343,7 @@ func (impl *CiHandlerImpl) validateBuildSequence(gitCiTriggerRequest bean.GitCiT ciPipelineMaterial := gitCiTriggerRequest.CiPipelineMaterial - if ciPipelineMaterial.Type == string(constants.SOURCE_TYPE_BRANCH_FIXED) { + if ciPipelineMaterial.Type == string(constants2.SOURCE_TYPE_BRANCH_FIXED) { if ciPipelineMaterial.GitCommit.Date.Before(lastTriggeredBuild.GitTriggers[ciPipelineMaterial.Id].Date) { impl.Logger.Warnw("older commit cannot be built for pipeline", "pipelineId", pipelineId, "ciMaterial", gitCiTriggerRequest.CiPipelineMaterial.Id) isValid = false @@ -347,12 +361,12 @@ func (impl *CiHandlerImpl) RefreshMaterialByCiPipelineMaterialId(gitMaterialId i return refreshRes, err } -func (impl *CiHandlerImpl) FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId int, gitMaterialId int, showAll bool) ([]pipelineConfig.CiPipelineMaterialResponse, error) { +func (impl *CiHandlerImpl) FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId int, gitMaterialId int, showAll bool) ([]buildBean.CiPipelineMaterialResponse, error) { ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineIdAndGitMaterialId(pipelineId, gitMaterialId) if err != nil { impl.Logger.Errorw("ciMaterials fetch failed", "err", err) } - var ciPipelineMaterialResponses []pipelineConfig.CiPipelineMaterialResponse + var ciPipelineMaterialResponses []buildBean.CiPipelineMaterialResponse var responseMap = make(map[int]bool) ciMaterialHistoryMap := make(map[*pipelineConfig.CiPipelineMaterial]*gitSensor.MaterialChangeResp) @@ -369,13 +383,13 @@ func (impl *CiHandlerImpl) FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId impl.Logger.Debugw("commits for material ", "m", m, "commits: ", changesResp) if apiErr != nil { impl.Logger.Warnw("git sensor FetchChanges failed for material", "id", m.Id) - return []pipelineConfig.CiPipelineMaterialResponse{}, apiErr + return []buildBean.CiPipelineMaterialResponse{}, apiErr } ciMaterialHistoryMap[m] = changesResp } for k, v := range ciMaterialHistoryMap { - r := pipelineConfig.CiPipelineMaterialResponse{ + r := buildBean.CiPipelineMaterialResponse{ Id: k.Id, GitMaterialId: k.GitMaterialId, GitMaterialName: k.GitMaterial.Name[strings.Index(k.GitMaterial.Name, "-")+1:], @@ -398,10 +412,10 @@ func (impl *CiHandlerImpl) FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId regexMaterials, err := impl.ciPipelineMaterialRepository.GetRegexByPipelineId(pipelineId) if err != nil { impl.Logger.Errorw("regex ciMaterials fetch failed", "err", err) - return []pipelineConfig.CiPipelineMaterialResponse{}, err + return []buildBean.CiPipelineMaterialResponse{}, err } for _, k := range regexMaterials { - r := pipelineConfig.CiPipelineMaterialResponse{ + r := buildBean.CiPipelineMaterialResponse{ Id: k.Id, GitMaterialId: k.GitMaterialId, GitMaterialName: k.GitMaterial.Name[strings.Index(k.GitMaterial.Name, "-")+1:], @@ -424,12 +438,12 @@ func (impl *CiHandlerImpl) FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId return ciPipelineMaterialResponses, nil } -func (impl *CiHandlerImpl) FetchMaterialsByPipelineId(pipelineId int, showAll bool) ([]pipelineConfig.CiPipelineMaterialResponse, error) { +func (impl *CiHandlerImpl) FetchMaterialsByPipelineId(pipelineId int, showAll bool) ([]buildBean.CiPipelineMaterialResponse, error) { ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(pipelineId) if err != nil { impl.Logger.Errorw("ciMaterials fetch failed", "err", err) } - var ciPipelineMaterialResponses []pipelineConfig.CiPipelineMaterialResponse + var ciPipelineMaterialResponses []buildBean.CiPipelineMaterialResponse var responseMap = make(map[int]bool) ciMaterialHistoryMap := make(map[*pipelineConfig.CiPipelineMaterial]*gitSensor.MaterialChangeResp) @@ -452,7 +466,7 @@ func (impl *CiHandlerImpl) FetchMaterialsByPipelineId(pipelineId int, showAll bo } for k, v := range ciMaterialHistoryMap { - r := pipelineConfig.CiPipelineMaterialResponse{ + r := buildBean.CiPipelineMaterialResponse{ Id: k.Id, GitMaterialId: k.GitMaterialId, GitMaterialName: k.GitMaterial.Name[strings.Index(k.GitMaterial.Name, "-")+1:], @@ -478,7 +492,7 @@ func (impl *CiHandlerImpl) FetchMaterialsByPipelineId(pipelineId int, showAll bo return nil, err } for _, k := range regexMaterials { - r := pipelineConfig.CiPipelineMaterialResponse{ + r := buildBean.CiPipelineMaterialResponse{ Id: k.Id, GitMaterialId: k.GitMaterialId, GitMaterialName: k.GitMaterial.Name[strings.Index(k.GitMaterial.Name, "-")+1:], @@ -507,9 +521,9 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int if err != nil { impl.Logger.Errorw("ciMaterials fetch failed", "err", err) } - var ciPipelineMaterialResponses []pipelineConfig.CiPipelineMaterialResponse + var ciPipelineMaterialResponses []buildBean.CiPipelineMaterialResponse for _, m := range ciMaterials { - r := pipelineConfig.CiPipelineMaterialResponse{ + r := buildBean.CiPipelineMaterialResponse{ Id: m.Id, GitMaterialId: m.GitMaterialId, Type: string(m.Type), @@ -520,7 +534,7 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int } ciPipelineMaterialResponses = append(ciPipelineMaterialResponses, r) } - //this map contains artifactId -> array of tags of that artifact + // this map contains artifactId -> array of tags of that artifact imageTagsDataMap, err := impl.imageTaggingService.GetTagsDataMapByAppId(appId) if err != nil { impl.Logger.Errorw("error in fetching image tags with appId", "err", err, "appId", appId) @@ -531,6 +545,7 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int impl.Logger.Errorw("err", "err", err) return nil, err } + var workflowIds []int var artifactIds []int for _, w := range workFlows { @@ -544,7 +559,7 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int return nil, err } - //this map contains artifactId -> imageComment of that artifact + // this map contains artifactId -> imageComment of that artifact imageCommetnsDataMap, err := impl.imageTaggingService.GetImageCommentsDataMapByArtifactIds(artifactIds) if err != nil { impl.Logger.Errorw("error in fetching imageCommetnsDataMap", "err", err, "appId", appId, "artifactIds", artifactIds) @@ -586,25 +601,25 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int WorkflowExecutionStage: impl.workFlowStageStatusService.ConvertDBWorkflowStageToMap(allWfStagesDetail, w.Id, w.Status, w.PodStatus, w.Message, bean2.CI_WORKFLOW_TYPE.String(), w.StartedOn, w.FinishedOn), } - if w.Message == bean3.ImageTagUnavailableMessage { - customTag, err := impl.customTagService.GetCustomTagByEntityKeyAndValue(bean3.EntityTypeCiPipelineId, strconv.Itoa(w.CiPipelineId)) + if w.Message == pipelineConfigBean.ImageTagUnavailableMessage { + customTag, err := impl.customTagService.GetCustomTagByEntityKeyAndValue(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(w.CiPipelineId)) if err != nil && err != pg.ErrNoRows { - //err == pg.ErrNoRows should never happen + // err == pg.ErrNoRows should never happen return nil, err } appWorkflows, err := impl.appWorkflowRepository.FindWFCIMappingByCIPipelineId(w.CiPipelineId) if err != nil && err != pg.ErrNoRows { return nil, err } - wfResponse.AppWorkflowId = appWorkflows[0].AppWorkflowId //it is guaranteed there will always be 1 entry (in case of ci_pipeline_id) + wfResponse.AppWorkflowId = appWorkflows[0].AppWorkflowId // it is guaranteed there will always be 1 entry (in case of ci_pipeline_id) wfResponse.CustomTag = &bean2.CustomTagErrorResponse{ TagPattern: customTag.TagPattern, AutoIncreasingNumber: customTag.AutoIncreasingNumber, - Message: bean3.ImageTagUnavailableMessage, + Message: pipelineConfigBean.ImageTagUnavailableMessage, } } if imageTagsDataMap[w.CiArtifactId] != nil { - wfResponse.ImageReleaseTags = imageTagsDataMap[w.CiArtifactId] //if artifact is not yet created,empty list will be sent + wfResponse.ImageReleaseTags = imageTagsDataMap[w.CiArtifactId] // if artifact is not yet created,empty list will be sent } if imageCommetnsDataMap[w.CiArtifactId] != nil { wfResponse.ImageComment = imageCommetnsDataMap[w.CiArtifactId] @@ -620,7 +635,7 @@ func (impl *CiHandlerImpl) CancelBuild(workflowId int, forceAbort bool) (int, er impl.Logger.Errorw("error in finding ci-workflow by workflow id", "ciWorkflowId", workflowId, "err", err) return 0, err } - isExt := workflow.Namespace != constants2.DefaultCiWorkflowNamespace + isExt := workflow.Namespace != constants.DefaultCiWorkflowNamespace var env *repository2.Environment var restConfig *rest.Config if isExt { @@ -667,7 +682,7 @@ func (impl *CiHandlerImpl) CancelBuild(workflowId int, forceAbort bool) (int, er workflow.Status = cdWorkflow.WorkflowCancel if workflow.ExecutorType == cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM { workflow.PodStatus = "Failed" - workflow.Message = constants2.TERMINATE_MESSAGE + workflow.Message = constants.TERMINATE_MESSAGE } err = impl.ciService.UpdateCiWorkflowWithStage(workflow) if err != nil { @@ -710,7 +725,7 @@ func (impl *CiHandlerImpl) handleForceAbortCaseForCi(workflow *pipelineConfig.Ci func (impl *CiHandlerImpl) updateWorkflowForForceAbort(workflow *pipelineConfig.CiWorkflow) error { workflow.Status = cdWorkflow.WorkflowCancel workflow.PodStatus = string(bean.Failed) - workflow.Message = constants2.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE + workflow.Message = constants.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE err := impl.ciService.UpdateCiWorkflowWithStage(workflow) if err != nil { impl.Logger.Errorw("error in updating workflow status", "err", err) @@ -766,9 +781,9 @@ func (impl *CiHandlerImpl) FetchWorkflowDetails(appId int, pipelineId int, build return types.WorkflowResponse{}, err } - var ciMaterialsArr []pipelineConfig.CiPipelineMaterialResponse + var ciMaterialsArr []buildBean.CiPipelineMaterialResponse for _, m := range ciMaterials { - res := pipelineConfig.CiPipelineMaterialResponse{ + res := buildBean.CiPipelineMaterialResponse{ Id: m.Id, GitMaterialId: m.GitMaterialId, GitMaterialName: m.GitMaterial.Name[strings.Index(m.GitMaterial.Name, "-")+1:], @@ -794,6 +809,7 @@ func (impl *CiHandlerImpl) FetchWorkflowDetails(appId int, pipelineId int, build impl.ciWorkflowRepository.MigrateIsArtifactUploaded(workflow.Id, ciArtifact.IsArtifactUploaded) isArtifactUploaded = ciArtifact.IsArtifactUploaded } + wfStagesDetail, err := impl.workFlowStageStatusService.GetWorkflowStagesByWorkflowIdsAndWfType([]int{workflow.Id}, bean2.CI_WORKFLOW_TYPE.String()) if err != nil { impl.Logger.Errorw("error in fetching allWfStagesDetail", "err", err, "workflowId", workflow.Id) @@ -811,19 +827,19 @@ func (impl *CiHandlerImpl) FetchWorkflowDetails(appId int, pipelineId int, build CiPipelineId: workflow.CiPipelineId, Namespace: workflow.Namespace, LogLocation: workflow.LogLocation, - BlobStorageEnabled: workflow.BlobStorageEnabled, //TODO default value if value not found in db + BlobStorageEnabled: workflow.BlobStorageEnabled, // TODO default value if value not found in db GitTriggers: workflow.GitTriggers, CiMaterials: ciMaterialsArr, TriggeredBy: workflow.TriggeredBy, TriggeredByEmail: triggeredByUserEmailId, Artifact: ciArtifact.Image, - TargetPlatforms: utils.ConvertTargetPlatformStringToObject(ciArtifact.TargetPlatforms), ArtifactId: ciArtifact.Id, IsArtifactUploaded: isArtifactUploaded, EnvironmentId: workflow.EnvironmentId, EnvironmentName: environmentName, PipelineType: workflow.CiPipeline.PipelineType, PodName: workflow.PodName, + TargetPlatforms: utils.ConvertTargetPlatformStringToObject(ciArtifact.TargetPlatforms), WorkflowExecutionStage: impl.workFlowStageStatusService.ConvertDBWorkflowStageToMap(wfStagesDetail, workflow.Id, workflow.Status, workflow.PodStatus, workflow.Message, bean2.CI_WORKFLOW_TYPE.String(), workflow.StartedOn, workflow.FinishedOn), } return workflowResponse, nil @@ -864,7 +880,7 @@ func (impl *CiHandlerImpl) getWorkflowLogs(ciWorkflow *pipelineConfig.CiWorkflow if err != nil { return nil, nil, err } - var clusterBean bean5.ClusterBean + var clusterBean clusterBean.ClusterBean if env != nil && env.Cluster != nil { clusterBean = adapter.GetClusterBean(*env.Cluster) } @@ -927,8 +943,8 @@ func (impl *CiHandlerImpl) getLogsFromRepository(ciWorkflow *pipelineConfig.CiWo } useExternalBlobStorage := isExternalBlobStorageEnabled(isExt, impl.config.UseBlobStorageConfigInCiWorkflow) if useExternalBlobStorage { - //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - //from them else return. + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) if err != nil { impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) @@ -1014,8 +1030,8 @@ func (impl *CiHandlerImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId i impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig by clusterId", "err", err, "clusterId", envBean.ClusterId) return nil, err } - //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - //from them else return. + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) if err != nil { impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) @@ -1088,8 +1104,8 @@ func (impl *CiHandlerImpl) GetHistoricBuildLogs(workflowId int, ciWorkflow *pipe impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig by clusterId", "err", err, "clusterId", envBean.ClusterId) return nil, err } - //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - //from them else return. + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) if err != nil { impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) @@ -1119,7 +1135,7 @@ func ExtractWorkflowStatus(workflowStatus v1alpha1.WorkflowStatus) (string, stri podName := "" logLocation := "" for k, v := range workflowStatus.Nodes { - if v.TemplateName == bean3.CI_WORKFLOW_NAME { + if v.TemplateName == pipelineConfigBean.CI_WORKFLOW_NAME { if v.BoundaryID == "" { workflowName = k } else { @@ -1194,7 +1210,7 @@ func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus impl.Logger.Errorw("cannot get saved wf", "err", err) return 0, err } - + impl.updateResourceStatusInCache(workflowId, podName, savedWorkflow.Namespace, status) ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() ciArtifactLocation := fmt.Sprintf(ciArtifactLocationFormat, savedWorkflow.Id, savedWorkflow.Id) @@ -1210,12 +1226,12 @@ func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus } if savedWorkflow.ExecutorType == cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM && savedWorkflow.Status == cdWorkflow.WorkflowCancel { savedWorkflow.PodStatus = "Failed" - savedWorkflow.Message = constants2.TERMINATE_MESSAGE + savedWorkflow.Message = constants.TERMINATE_MESSAGE } savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time savedWorkflow.Name = workflowName - //savedWorkflow.LogLocation = "/ci-pipeline/" + strconv.Itoa(savedWorkflow.CiPipelineId) + "/workflow/" + strconv.Itoa(savedWorkflow.Id) + "/logs" //TODO need to fetch from workflow object - //savedWorkflow.LogLocation = logLocation // removed because we are saving log location at trigger + // savedWorkflow.LogLocation = "/ci-pipeline/" + strconv.Itoa(savedWorkflow.CiPipelineId) + "/workflow/" + strconv.Itoa(savedWorkflow.Id) + "/logs" //TODO need to fetch from workflow object + // savedWorkflow.LogLocation = logLocation // removed because we are saving log location at trigger savedWorkflow.CiArtifactLocation = ciArtifactLocation savedWorkflow.PodName = podName impl.Logger.Debugw("updating workflow ", "workflow", savedWorkflow) @@ -1238,7 +1254,7 @@ func (impl *CiHandlerImpl) sendCIFailEvent(savedWorkflow *pipelineConfig.CiWorkf impl.Logger.Warnw("ci failed for workflow: ", "wfId", savedWorkflow.Id) if extractErrorCode(savedWorkflow.Message) != workFlow.CiStageFailErrorCode { - go impl.WriteCIFailEvent(savedWorkflow) + impl.ciService.WriteCIFailEvent(savedWorkflow) } else { impl.Logger.Infof("Step failed notification received for wfID %d with message %s", savedWorkflow.Id, savedWorkflow.Message) } @@ -1257,27 +1273,10 @@ func extractErrorCode(msg string) int { return -1 } -func (impl *CiHandlerImpl) WriteCIFailEvent(ciWorkflow *pipelineConfig.CiWorkflow) { - event, _ := impl.eventFactory.Build(util2.Fail, &ciWorkflow.CiPipelineId, ciWorkflow.CiPipeline.AppId, nil, util2.CI) - material := &client.MaterialTriggerInfo{} - material.GitTriggers = ciWorkflow.GitTriggers - event.CiWorkflowRunnerId = ciWorkflow.Id - event.UserId = int(ciWorkflow.TriggeredBy) - event = impl.eventFactory.BuildExtraCIData(event, material) - event.CiArtifactId = 0 - _, evtErr := impl.eventClient.WriteNotificationEvent(event) - if evtErr != nil { - impl.Logger.Errorw("error in writing event", "err", evtErr) - } -} - func (impl *CiHandlerImpl) BuildPayload(ciWorkflow *pipelineConfig.CiWorkflow) *client.Payload { payload := &client.Payload{} payload.AppName = ciWorkflow.CiPipeline.App.AppName payload.PipelineName = ciWorkflow.CiPipeline.Name - //payload["buildName"] = ciWorkflow.Name - //payload["podStatus"] = ciWorkflow.PodStatus - //payload["message"] = ciWorkflow.Message return payload } @@ -1325,7 +1324,7 @@ func SetGitCommitValuesForBuildingCommitHash(ciMaterial *pipelineConfig.CiPipeli func (impl *CiHandlerImpl) buildManualTriggerCommitHashes(ciTriggerRequest bean.CiTriggerRequest) (map[int]pipelineConfig.GitCommit, *common.RuntimeParameters, error) { commitHashes := map[int]pipelineConfig.GitCommit{} - runtimeParams := common.NewRuntimeParameters() + runtimeParams := impl.getRuntimeParamsForBuildingManualTriggerHashes(ciTriggerRequest) for _, ciPipelineMaterial := range ciTriggerRequest.CiPipelineMaterial { pipeLineMaterialFromDb, err := impl.ciPipelineMaterialRepository.GetById(ciPipelineMaterial.Id) @@ -1335,7 +1334,7 @@ func (impl *CiHandlerImpl) buildManualTriggerCommitHashes(ciTriggerRequest bean. } pipelineType := pipeLineMaterialFromDb.Type - if pipelineType == constants.SOURCE_TYPE_BRANCH_FIXED { + if pipelineType == constants2.SOURCE_TYPE_BRANCH_FIXED { gitCommit, err := impl.BuildManualTriggerCommitHashesForSourceTypeBranchFix(ciPipelineMaterial, pipeLineMaterialFromDb) if err != nil { impl.Logger.Errorw("err", "err", err) @@ -1343,7 +1342,7 @@ func (impl *CiHandlerImpl) buildManualTriggerCommitHashes(ciTriggerRequest bean. } commitHashes[ciPipelineMaterial.Id] = gitCommit - } else if pipelineType == constants.SOURCE_TYPE_WEBHOOK { + } else if pipelineType == constants2.SOURCE_TYPE_WEBHOOK { gitCommit, extraEnvVariables, err := impl.BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial, pipeLineMaterialFromDb) if err != nil { impl.Logger.Errorw("err", "err", err) @@ -1490,12 +1489,7 @@ func (impl *CiHandlerImpl) FetchCiStatusForTriggerView(appId int) ([]*pipelineCo return ciWorkflowStatuses, err } for _, pipeline := range pipelines { - pipelineId := 0 - if pipeline.ParentCiPipeline == 0 { - pipelineId = pipeline.Id - } else { - pipelineId = pipeline.ParentCiPipeline - } + pipelineId := impl.getPipelineIdForTriggerView(pipeline) workflow, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflow(pipelineId) if err != nil && !util.IsErrNoRows(err) { impl.Logger.Errorw("err", "pipelineId", pipelineId, "err", err) @@ -1540,7 +1534,7 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId return &types.GitTriggerInfoResponse{}, err } - ciMaterialsArr := make([]pipelineConfig.CiPipelineMaterialResponse, 0) + ciMaterialsArr := make([]buildBean.CiPipelineMaterialResponse, 0) var triggeredByUserEmailId string //check workflow data only for non external builds if !ciPipeline.IsExternal { @@ -1596,7 +1590,7 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId history = append(history, _gitCommit) - res := pipelineConfig.CiPipelineMaterialResponse{ + res := buildBean.CiPipelineMaterialResponse{ Id: m.Id, GitMaterialId: m.GitMaterialId, GitMaterialName: m.GitMaterial.Name[strings.Index(m.GitMaterial.Name, "-")+1:], @@ -1618,6 +1612,7 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId //GitTriggers: workflow.GitTriggers, CiMaterials: ciMaterialsArr, TriggeredByEmail: triggeredByUserEmailId, + CiPipelineId: ciPipeline.Id, AppId: ciPipeline.AppId, AppName: deployDetail.AppName, EnvironmentId: deployDetail.EnvironmentId, @@ -1632,7 +1627,7 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId } func (impl *CiHandlerImpl) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error { - ciWorkflows, err := impl.ciWorkflowRepository.FindByStatusesIn([]string{constants2.Starting, constants2.Running}) + ciWorkflows, err := impl.ciWorkflowRepository.FindByStatusesIn([]string{constants.Starting, constants.Running}) if err != nil { impl.Logger.Errorw("error on fetching ci workflows", "err", err) return err @@ -1647,7 +1642,7 @@ func (impl *CiHandlerImpl) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuil var isExt bool var env *repository2.Environment var restConfig *rest.Config - if ciWorkflow.Namespace != constants2.DefaultCiWorkflowNamespace { + if ciWorkflow.Namespace != constants.DefaultCiWorkflowNamespace { isExt = true env, err = impl.envRepository.FindById(ciWorkflow.EnvironmentId) if err != nil { @@ -1680,7 +1675,7 @@ func (impl *CiHandlerImpl) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuil //if ci workflow is exists, check its pod if !isEligibleToMarkFailed { - ns := constants2.DefaultCiWorkflowNamespace + ns := constants.DefaultCiWorkflowNamespace if isExt { _, client, err = impl.k8sCommonService.GetCoreClientByClusterId(env.ClusterId) if err != nil { @@ -1706,8 +1701,8 @@ func (impl *CiHandlerImpl) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuil isPodDeleted = true } } else { - //check workflow status,get the status - if wf.Status == string(v1alpha1.WorkflowFailed) && wf.Message == constants2.POD_DELETED_MESSAGE { + // check workflow status,get the status + if wf.Status == string(v1alpha1.WorkflowFailed) && wf.Message == cdWorkflow.POD_DELETED_MESSAGE { isPodDeleted = true } } @@ -1718,7 +1713,7 @@ func (impl *CiHandlerImpl) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuil ciWorkflow.PodStatus = "Failed" if isPodDeleted { ciWorkflow.Message = cdWorkflow.POD_DELETED_MESSAGE - //error logging handled inside handlePodDeleted + // error logging handled inside handlePodDeleted impl.handlePodDeleted(ciWorkflow) } else { ciWorkflow.Message = "marked failed by job" @@ -1761,7 +1756,7 @@ func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewForEnvironment(request res if err != nil { return nil, err } - //override appIds if already provided app group id in request. + // override appIds if already provided app group id in request. request.ResourceIds = appIds } if len(request.ResourceIds) > 0 { @@ -1794,7 +1789,7 @@ func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewForEnvironment(request res if len(ciPipelineIds) == 0 { return ciWorkflowStatuses, nil } - //authorization block starts here + // authorization block starts here var appObjectArr []string objects := impl.enforcerUtil.GetAppObjectByCiPipelineIds(ciPipelineIds) ciPipelineIds = []int{} @@ -1803,17 +1798,12 @@ func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewForEnvironment(request res } appResults, _ := request.CheckAuthBatch(token, appObjectArr, []string{}) for _, ciPipeline := range ciPipelines { - appObject := objects[ciPipeline.Id] //here only app permission have to check + appObject := objects[ciPipeline.Id] // here only app permission have to check if !appResults[appObject] { - //if user unauthorized, skip items + // if user unauthorized, skip items continue } - ciPipelineId := 0 - if ciPipeline.ParentCiPipeline == 0 { - ciPipelineId = ciPipeline.Id - } else { - ciPipelineId = ciPipeline.ParentCiPipeline - } + ciPipelineId := impl.getPipelineIdForTriggerView(ciPipeline) ciPipelineIds = append(ciPipelineIds, ciPipelineId) } if len(ciPipelineIds) == 0 { diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index d5282add0b..da1883339f 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -32,7 +32,8 @@ import ( bean4 "github.com/devtron-labs/devtron/pkg/attributes/bean" "github.com/devtron-labs/devtron/pkg/bean/common" "github.com/devtron-labs/devtron/pkg/build/pipeline" - bean6 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + common2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" repository6 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/pipeline/adapter" pipelineConst "github.com/devtron-labs/devtron/pkg/pipeline/constants" @@ -79,6 +80,7 @@ import ( type CiService interface { TriggerCiPipeline(trigger types.Trigger) (int, error) GetCiMaterials(pipelineId int, ciMaterials []*pipelineConfig.CiPipelineMaterial) ([]*pipelineConfig.CiPipelineMaterial, error) + WriteCIFailEvent(ciWorkflow *pipelineConfig.CiWorkflow) SaveCiWorkflowWithStage(wf *pipelineConfig.CiWorkflow) error UpdateCiWorkflowWithStage(wf *pipelineConfig.CiWorkflow) error } @@ -188,7 +190,7 @@ func (impl *CiServiceImpl) GetCiMaterials(pipelineId int, ciMaterials []*pipelin func (impl *CiServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, workflowRequest *types.WorkflowRequest) error { // externalCi artifact is meant only for CI_JOB - if trigger.PipelineType != string(bean6.CI_JOB) { + if trigger.PipelineType != string(common2.CI_JOB) { return nil } @@ -247,7 +249,7 @@ func (impl *CiServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, } } - if trigger.PipelineType == string(bean6.CI_JOB) && len(ciMaterials) != 0 && !exists && externalCiArtifact == "" { + if trigger.PipelineType == string(common2.CI_JOB) && len(ciMaterials) != 0 && !exists && externalCiArtifact == "" { ciMaterials[0].GitMaterial = nil ciMaterials[0].GitMaterialId = 0 } @@ -490,7 +492,7 @@ func (impl *CiServiceImpl) getEnvironmentForJob(pipeline *pipelineConfig.CiPipel func (impl *CiServiceImpl) WriteCITriggerEvent(trigger types.Trigger, pipeline *pipelineConfig.CiPipeline, workflowRequest *types.WorkflowRequest) { event, _ := impl.eventFactory.Build(util2.Trigger, &pipeline.Id, pipeline.AppId, nil, util2.CI) - material := &client.MaterialTriggerInfo{} + material := &buildBean.MaterialTriggerInfo{} material.GitTriggers = trigger.CommitHashes @@ -663,7 +665,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. var dockerfilePath string var dockerRepository string var checkoutPath string - var ciBuildConfigBean *bean6.CiBuildConfigBean + var ciBuildConfigBean *buildBean.CiBuildConfigBean dockerRegistry := &repository3.DockerArtifactStore{} ciBaseBuildConfigEntity := ciTemplate.CiBuildConfig ciBaseBuildConfigBean, err := adapter.ConvertDbBuildConfigToBean(ciBaseBuildConfigEntity) @@ -774,13 +776,13 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. ciBuildConfigBean.PipelineType = trigger.PipelineType - if ciBuildConfigBean.CiBuildType == bean6.SELF_DOCKERFILE_BUILD_TYPE || ciBuildConfigBean.CiBuildType == bean6.MANAGED_DOCKERFILE_BUILD_TYPE { + if ciBuildConfigBean.CiBuildType == buildBean.SELF_DOCKERFILE_BUILD_TYPE || ciBuildConfigBean.CiBuildType == buildBean.MANAGED_DOCKERFILE_BUILD_TYPE { ciBuildConfigBean.DockerBuildConfig.BuildContext = filepath.Join(buildContextCheckoutPath, ciBuildConfigBean.DockerBuildConfig.BuildContext) dockerBuildConfig := ciBuildConfigBean.DockerBuildConfig dockerfilePath = filepath.Join(checkoutPath, dockerBuildConfig.DockerfilePath) dockerBuildConfig.DockerfilePath = dockerfilePath checkoutPath = dockerfilePath[:strings.LastIndex(dockerfilePath, "/")+1] - } else if ciBuildConfigBean.CiBuildType == bean6.BUILDPACK_BUILD_TYPE { + } else if ciBuildConfigBean.CiBuildType == buildBean.BUILDPACK_BUILD_TYPE { buildPackConfig := ciBuildConfigBean.BuildPackConfig checkoutPath = filepath.Join(checkoutPath, buildPackConfig.ProjectPath) } @@ -1135,6 +1137,20 @@ func _getTruncatedImageTag(imageTag string) string { } } +func (impl *CiServiceImpl) WriteCIFailEvent(ciWorkflow *pipelineConfig.CiWorkflow) { + event, _ := impl.eventFactory.Build(util2.Fail, &ciWorkflow.CiPipelineId, ciWorkflow.CiPipeline.AppId, nil, util2.CI) + material := &buildBean.MaterialTriggerInfo{} + material.GitTriggers = ciWorkflow.GitTriggers + event.CiWorkflowRunnerId = ciWorkflow.Id + event.UserId = int(ciWorkflow.TriggeredBy) + event = impl.eventFactory.BuildExtraCIData(event, material) + event.CiArtifactId = 0 + _, evtErr := impl.eventClient.WriteNotificationEvent(event) + if evtErr != nil { + impl.Logger.Errorw("error in writing event", "err", evtErr) + } +} + func (impl *CiServiceImpl) SaveCiWorkflowWithStage(wf *pipelineConfig.CiWorkflow) error { // implementation tx, err := impl.transactionManager.StartTx() diff --git a/pkg/pipeline/WorkflowService.go b/pkg/pipeline/WorkflowService.go index 3cc2215867..191b973c75 100644 --- a/pkg/pipeline/WorkflowService.go +++ b/pkg/pipeline/WorkflowService.go @@ -28,7 +28,7 @@ import ( "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" - bean2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/config/read" v1 "github.com/devtron-labs/devtron/pkg/infraConfig/bean/v1" @@ -218,7 +218,7 @@ func (impl *WorkflowServiceImpl) createWorkflowTemplate(workflowRequest *types.W func (impl *WorkflowServiceImpl) shouldAddExistingCmCsInWorkflow(workflowRequest *types.WorkflowRequest) bool { // CmCs are not added for CI_JOB if IgnoreCmCsInCiJob is true - if workflowRequest.CiPipelineType == string(bean2.CI_JOB) && impl.ciCdConfig.IgnoreCmCsInCiJob { + if workflowRequest.CiPipelineType == string(common.CI_JOB) && impl.ciCdConfig.IgnoreCmCsInCiJob { return false } return true diff --git a/pkg/pipeline/adapter/adapter.go b/pkg/pipeline/adapter/adapter.go index badb5c0f3e..d480731a2a 100644 --- a/pkg/pipeline/adapter/adapter.go +++ b/pkg/pipeline/adapter/adapter.go @@ -23,6 +23,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/ciPipeline" "github.com/devtron-labs/devtron/pkg/bean" bean2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" bean3 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" @@ -190,7 +191,7 @@ func mergeMap(oldDockerArgs map[string]string, ciLevelDockerArgs map[string]stri // IsLinkedCD will return if the pipelineConfig.CiPipeline is a Linked CD func IsLinkedCD(ci pipelineConfig.CiPipeline) bool { - return ci.ParentCiPipeline != 0 && ci.PipelineType == string(bean2.LINKED_CD) + return ci.ParentCiPipeline != 0 && ci.PipelineType == string(common.LINKED_CD) } // IsLinkedCI will return if the pipelineConfig.CiPipeline is a Linked CI @@ -199,7 +200,7 @@ func IsLinkedCI(ci *pipelineConfig.CiPipeline) bool { return false } return ci.ParentCiPipeline != 0 && - ci.PipelineType == string(bean2.LINKED) + ci.PipelineType == string(common.LINKED) } // IsCIJob will return if the pipelineConfig.CiPipeline is a CI JOB @@ -207,7 +208,7 @@ func IsCIJob(ci *pipelineConfig.CiPipeline) bool { if ci == nil { return false } - return ci.PipelineType == string(bean2.CI_JOB) + return ci.PipelineType == string(common.CI_JOB) } // GetSourceCiDownStreamResponse will take the models []bean.LinkedCIDetails and []pipelineConfig.CdWorkflowRunner (for last deployment status) and generate the []CiPipeline.SourceCiDownStreamResponse diff --git a/pkg/pipeline/bean/CdHandlerBean.go b/pkg/pipeline/bean/CdHandlerBean.go index 4b6480e365..5dc408c226 100644 --- a/pkg/pipeline/bean/CdHandlerBean.go +++ b/pkg/pipeline/bean/CdHandlerBean.go @@ -4,37 +4,38 @@ import ( "github.com/devtron-labs/common-lib/utils/bean" "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" bean2 "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus/bean" "time" ) type CdWorkflowWithArtifact struct { - Id int `json:"id"` - CdWorkflowId int `json:"cd_workflow_id"` - Name string `json:"name"` - Status string `json:"status"` - PodStatus string `json:"pod_status"` - Message string `json:"message"` - StartedOn time.Time `json:"started_on"` - FinishedOn time.Time `json:"finished_on"` - PipelineId int `json:"pipeline_id"` - Namespace string `json:"namespace"` - LogFilePath string `json:"log_file_path"` - TriggeredBy int32 `json:"triggered_by"` - EmailId string `json:"email_id"` - Image string `json:"image"` - TargetPlatforms []*bean.TargetPlatform `json:"targetPlatforms"` - MaterialInfo string `json:"material_info,omitempty"` - DataSource string `json:"data_source,omitempty"` - CiArtifactId int `json:"ci_artifact_id,omitempty"` - IsArtifactUploaded bool `json:"isArtifactUploaded"` - WorkflowType string `json:"workflow_type,omitempty"` - ExecutorType string `json:"executor_type,omitempty"` - BlobStorageEnabled bool `json:"blobStorageEnabled"` - GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` - CiMaterials []pipelineConfig.CiPipelineMaterialResponse `json:"ciMaterials"` - ImageReleaseTags []*repository.ImageTag `json:"imageReleaseTags"` - ImageComment *repository.ImageComment `json:"imageComment"` - RefCdWorkflowRunnerId int `json:"referenceCdWorkflowRunnerId"` - WorkflowExecutionStage map[string][]*bean2.WorkflowStageDto `json:"workflowExecutionStages"` + Id int `json:"id"` + CdWorkflowId int `json:"cd_workflow_id"` + Name string `json:"name"` + Status string `json:"status"` + PodStatus string `json:"pod_status"` + Message string `json:"message"` + StartedOn time.Time `json:"started_on"` + FinishedOn time.Time `json:"finished_on"` + PipelineId int `json:"pipeline_id"` + Namespace string `json:"namespace"` + LogFilePath string `json:"log_file_path"` + TriggeredBy int32 `json:"triggered_by"` + EmailId string `json:"email_id"` + Image string `json:"image"` + TargetPlatforms []*bean.TargetPlatform `json:"targetPlatforms"` + MaterialInfo string `json:"material_info,omitempty"` + DataSource string `json:"data_source,omitempty"` + CiArtifactId int `json:"ci_artifact_id,omitempty"` + IsArtifactUploaded bool `json:"isArtifactUploaded"` + WorkflowType string `json:"workflow_type,omitempty"` + ExecutorType string `json:"executor_type,omitempty"` + BlobStorageEnabled bool `json:"blobStorageEnabled"` + GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` + CiMaterials []buildBean.CiPipelineMaterialResponse `json:"ciMaterials"` + ImageReleaseTags []*repository.ImageTag `json:"imageReleaseTags"` + ImageComment *repository.ImageComment `json:"imageComment"` + RefCdWorkflowRunnerId int `json:"referenceCdWorkflowRunnerId"` + WorkflowExecutionStage map[string][]*bean2.WorkflowStageDto `json:"workflowExecutionStages"` } diff --git a/pkg/pipeline/constants/constants.go b/pkg/pipeline/constants/constants.go index 0d5cd5bcab..243fd6679e 100644 --- a/pkg/pipeline/constants/constants.go +++ b/pkg/pipeline/constants/constants.go @@ -68,7 +68,6 @@ const ( const DefaultCiWorkflowNamespace = "devtron-ci" const Running = "Running" const Starting = "Starting" -const POD_DELETED_MESSAGE = "pod deleted" const TERMINATE_MESSAGE = "workflow shutdown with strategy: Terminate" const FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE = "workflow shutdown with strategy: Force Abort" const POD_TIMEOUT_MESSAGE = "Pod was active on the node longer than the specified deadline" diff --git a/pkg/pipeline/types/CiCdConfig.go b/pkg/pipeline/types/CiCdConfig.go index 961bd9f26f..d6e98efc5c 100644 --- a/pkg/pipeline/types/CiCdConfig.go +++ b/pkg/pipeline/types/CiCdConfig.go @@ -27,6 +27,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/pkg/bean/common" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/pipeline/bean" v12 "k8s.io/api/core/v1" @@ -550,17 +551,18 @@ type ArtifactsForCiJob struct { } type GitTriggerInfoResponse struct { - CiMaterials []pipelineConfig.CiPipelineMaterialResponse `json:"ciMaterials"` - TriggeredByEmail string `json:"triggeredByEmail"` - LastDeployedTime string `json:"lastDeployedTime,omitempty"` - AppId int `json:"appId"` - AppName string `json:"appName"` - EnvironmentId int `json:"environmentId"` - EnvironmentName string `json:"environmentName"` - Default bool `json:"default,omitempty"` - ImageTaggingData ImageTaggingResponseDTO `json:"imageTaggingData"` - Image string `json:"image"` - TargetPlatforms []*bean2.TargetPlatform `json:"targetPlatforms"` + CiMaterials []buildBean.CiPipelineMaterialResponse `json:"ciMaterials"` + TriggeredByEmail string `json:"triggeredByEmail"` + LastDeployedTime string `json:"lastDeployedTime,omitempty"` + CiPipelineId int `json:"ciPipelineId"` + AppId int `json:"appId"` + AppName string `json:"appName"` + EnvironmentId int `json:"environmentId"` + EnvironmentName string `json:"environmentName"` + Default bool `json:"default,omitempty"` + ImageTaggingData ImageTaggingResponseDTO `json:"imageTaggingData"` + Image string `json:"image"` + TargetPlatforms []*bean2.TargetPlatform `json:"targetPlatforms"` } type Trigger struct { diff --git a/pkg/pipeline/types/Workflow.go b/pkg/pipeline/types/Workflow.go index dca45ab8fc..dceb9a03b0 100644 --- a/pkg/pipeline/types/Workflow.go +++ b/pkg/pipeline/types/Workflow.go @@ -31,6 +31,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" bean2 "github.com/devtron-labs/devtron/pkg/bean" bean5 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" repository4 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" infraBean "github.com/devtron-labs/devtron/pkg/infraConfig/bean/v1" "github.com/devtron-labs/devtron/pkg/pipeline/bean" @@ -649,37 +650,37 @@ type ChildCdMetadata struct { } type WorkflowResponse struct { - Id int `json:"id"` - Name string `json:"name"` - Status string `json:"status"` - PodStatus string `json:"podStatus"` - Message string `json:"message"` - StartedOn time.Time `json:"startedOn"` - FinishedOn time.Time `json:"finishedOn"` - CiPipelineId int `json:"ciPipelineId"` - Namespace string `json:"namespace"` - LogLocation string `json:"logLocation"` - BlobStorageEnabled bool `json:"blobStorageEnabled"` - GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` - CiMaterials []pipelineConfig.CiPipelineMaterialResponse `json:"ciMaterials"` - TriggeredBy int32 `json:"triggeredBy"` - Artifact string `json:"artifact"` - TriggeredByEmail string `json:"triggeredByEmail"` - Stage string `json:"stage"` - ArtifactId int `json:"artifactId"` - IsArtifactUploaded bool `json:"isArtifactUploaded"` - IsVirtualEnvironment bool `json:"isVirtualEnvironment"` - PodName string `json:"podName"` - EnvironmentId int `json:"environmentId"` - EnvironmentName string `json:"environmentName"` - ImageReleaseTags []*repository3.ImageTag `json:"imageReleaseTags"` - ImageComment *repository3.ImageComment `json:"imageComment"` - AppWorkflowId int `json:"appWorkflowId"` - CustomTag *bean3.CustomTagErrorResponse `json:"customTag,omitempty"` - PipelineType string `json:"pipelineType"` - ReferenceWorkflowId int `json:"referenceWorkflowId"` - TargetPlatforms []*bean7.TargetPlatform `json:"targetPlatforms"` - WorkflowExecutionStage map[string][]*bean6.WorkflowStageDto `json:"workflowExecutionStages"` + Id int `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + PodStatus string `json:"podStatus"` + Message string `json:"message"` + StartedOn time.Time `json:"startedOn"` + FinishedOn time.Time `json:"finishedOn"` + CiPipelineId int `json:"ciPipelineId"` + Namespace string `json:"namespace"` + LogLocation string `json:"logLocation"` + BlobStorageEnabled bool `json:"blobStorageEnabled"` + GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` + CiMaterials []buildBean.CiPipelineMaterialResponse `json:"ciMaterials"` + TriggeredBy int32 `json:"triggeredBy"` + Artifact string `json:"artifact"` + TriggeredByEmail string `json:"triggeredByEmail"` + Stage string `json:"stage"` + ArtifactId int `json:"artifactId"` + IsArtifactUploaded bool `json:"isArtifactUploaded"` + IsVirtualEnvironment bool `json:"isVirtualEnvironment"` + PodName string `json:"podName"` + EnvironmentId int `json:"environmentId"` + EnvironmentName string `json:"environmentName"` + ImageReleaseTags []*repository3.ImageTag `json:"imageReleaseTags"` + ImageComment *repository3.ImageComment `json:"imageComment"` + AppWorkflowId int `json:"appWorkflowId"` + CustomTag *bean3.CustomTagErrorResponse `json:"customTag,omitempty"` + PipelineType string `json:"pipelineType"` + ReferenceWorkflowId int `json:"referenceWorkflowId"` + TargetPlatforms []*bean7.TargetPlatform `json:"targetPlatforms"` + WorkflowExecutionStage map[string][]*bean6.WorkflowStageDto `json:"workflowExecutionStages"` } type ConfigMapSecretDto struct { diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index d87f6bfd97..515ab4cafc 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -38,6 +38,7 @@ import ( bean7 "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/build/artifacts" bean5 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/manifest" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" @@ -830,7 +831,7 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext trigger var pluginArtifacts []*repository.CiArtifact for registry, artifacts := range request.PluginRegistryArtifactDetails { for _, image := range artifacts { - if pipelineModal.PipelineType == string(bean5.CI_JOB) && image == "" { + if pipelineModal.PipelineType == string(common.CI_JOB) && image == "" { continue } pluginArtifact := &repository.CiArtifact{ @@ -994,7 +995,7 @@ func (impl *WorkflowDagExecutorImpl) HandleCiStepFailedEvent(ciPipelineId int, r func (impl *WorkflowDagExecutorImpl) WriteCiStepFailedEvent(pipeline *pipelineConfig.CiPipeline, request *bean2.CiArtifactWebhookRequest, ciWorkflow *pipelineConfig.CiWorkflow) { event, _ := impl.eventFactory.Build(util2.Fail, &pipeline.Id, pipeline.AppId, nil, util2.CI) - material := &client.MaterialTriggerInfo{} + material := &bean5.MaterialTriggerInfo{} material.GitTriggers = ciWorkflow.GitTriggers event.CiWorkflowRunnerId = ciWorkflow.Id event.UserId = int(ciWorkflow.TriggeredBy) diff --git a/wire_gen.go b/wire_gen.go index 679957d5ec..ff2d593a71 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -685,7 +685,7 @@ func InitializeApp() (*App, error) { } imageTaggingServiceImpl := imageTagging.NewImageTaggingServiceImpl(imageTaggingRepositoryImpl, imageTaggingReadServiceImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, sugaredLogger) blobStorageConfigServiceImpl := pipeline.NewBlobStorageConfigServiceImpl(sugaredLogger, k8sServiceImpl, ciCdConfig) - ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, workflowServiceImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, k8sServiceImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, environmentServiceImpl, workFlowStageStatusServiceImpl) + ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, workflowServiceImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, environmentServiceImpl, workFlowStageStatusServiceImpl, k8sServiceImpl) gitWebhookRepositoryImpl := repository23.NewGitWebhookRepositoryImpl(db) gitWebhookServiceImpl := gitWebhook.NewGitWebhookServiceImpl(sugaredLogger, ciHandlerImpl, gitWebhookRepositoryImpl) gitWebhookRestHandlerImpl := restHandler.NewGitWebhookRestHandlerImpl(sugaredLogger, gitWebhookServiceImpl) From 7e7928d41bed9784cdbae7ef48d8e9f4dac50ffc Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 1 Apr 2025 19:48:13 +0530 Subject: [PATCH 02/37] common constants create --- .../pipelineConfig/CiPipelineRepository.go | 8 ++--- pkg/app/AppListingService.go | 4 +-- .../devtronApps/PreStageTriggerService.go | 4 +-- pkg/pipeline/CiCdPipelineOrchestrator.go | 9 +++--- pkg/pipeline/CiService.go | 7 ++--- pkg/pipeline/constants/constants.go | 31 ------------------- pkg/workflow/dag/WorkflowDagExecutor.go | 7 ++--- 7 files changed, 18 insertions(+), 52 deletions(-) diff --git a/internal/sql/repository/pipelineConfig/CiPipelineRepository.go b/internal/sql/repository/pipelineConfig/CiPipelineRepository.go index fb90f43255..c6adb8b7a8 100644 --- a/internal/sql/repository/pipelineConfig/CiPipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/CiPipelineRepository.go @@ -21,8 +21,8 @@ import ( "fmt" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/ciPipeline" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" - "github.com/devtron-labs/devtron/pkg/pipeline/constants" "github.com/devtron-labs/devtron/pkg/sql" "github.com/devtron-labs/devtron/util/response/pagination" "github.com/go-pg/pg" @@ -641,7 +641,7 @@ func (impl *CiPipelineRepositoryImpl) FindLinkedCiCount(ciPipelineId int) (int, pipeline := &CiPipeline{} cnt, err := impl.dbConnection.Model(pipeline). Where("parent_ci_pipeline = ?", ciPipelineId). - Where("ci_pipeline_type != ?", constants.LINKED_CD). + Where("ci_pipeline_type != ?", buildCommonBean.LINKED_CD). Where("deleted = ?", false). Count() if err == pg.ErrNoRows { @@ -656,7 +656,7 @@ func (impl *CiPipelineRepositoryImpl) GetLinkedCiPipelines(ctx context.Context, var linkedCIPipelines []*CiPipeline err := impl.dbConnection.Model(&linkedCIPipelines). Where("parent_ci_pipeline = ?", ciPipelineId). - Where("ci_pipeline_type != ?", constants.LINKED_CD). + Where("ci_pipeline_type != ?", buildCommonBean.LINKED_CD). Where("deleted = ?", false). Select() if err != nil { @@ -694,7 +694,7 @@ func (impl *CiPipelineRepositoryImpl) GetDownStreamInfo(ctx context.Context, sou JoinOn("e.active = ?", true). // constrains Where("ci_pipeline.parent_ci_pipeline = ?", sourceCiPipelineId). - Where("ci_pipeline.ci_pipeline_type != ?", constants.LINKED_CD). + Where("ci_pipeline.ci_pipeline_type != ?", buildCommonBean.LINKED_CD). Where("ci_pipeline.deleted = ?", false) // app name filtering with lower case if len(appNameMatch) != 0 { diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index ad2659551b..67bf66bc28 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -25,13 +25,13 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" read4 "github.com/devtron-labs/devtron/pkg/app/appDetails/read" userrepository "github.com/devtron-labs/devtron/pkg/auth/user/repository" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" ciConfig "github.com/devtron-labs/devtron/pkg/build/pipeline/read" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/read" "github.com/devtron-labs/devtron/pkg/dockerRegistry" - "github.com/devtron-labs/devtron/pkg/pipeline/constants" errors2 "github.com/juju/errors" "go.opentelemetry.io/otel" "golang.org/x/exp/slices" @@ -607,7 +607,7 @@ func (impl AppListingServiceImpl) setIpAccessProvidedData(ctx context.Context, a } if ciPipeline != nil && ciPipeline.CiTemplate != nil && len(*ciPipeline.CiTemplate.DockerRegistryId) > 0 { - if !ciPipeline.IsExternal || ciPipeline.ParentCiPipeline != 0 && ciPipeline.PipelineType != string(constants.LINKED_CD) { + if !ciPipeline.IsExternal || ciPipeline.ParentCiPipeline != 0 && ciPipeline.PipelineType != string(buildCommonBean.LINKED_CD) { appDetailContainer.IsExternalCi = false } // get dockerRegistryId starts diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go index 580bbf4da9..ec80a7cf19 100644 --- a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go @@ -35,6 +35,7 @@ import ( bean7 "github.com/devtron-labs/devtron/pkg/auth/user/bean" bean4 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/bean/common" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" repository4 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" bean5 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" adapter2 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/adapter" @@ -43,7 +44,6 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/adapter" pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" - "github.com/devtron-labs/devtron/pkg/pipeline/constants" repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/plugin" @@ -532,7 +532,7 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow return nil, err } ciProjectDetail.CommitTime = commitTime.Format(bean4.LayoutRFC3339) - } else if ciPipeline.PipelineType == string(constants.CI_JOB) { + } else if ciPipeline.PipelineType == string(buildCommonBean.CI_JOB) { // This has been done to resolve unmarshalling issue in ci-runner, in case of no commit time(eg- polling container images) ciProjectDetail.CommitTime = time.Time{}.Format(bean4.LayoutRFC3339) } else { diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index 313611d76c..46a08da9d5 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -32,13 +32,12 @@ import ( repository6 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/repository" "github.com/devtron-labs/devtron/pkg/build/pipeline" bean2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" - common3 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" read2 "github.com/devtron-labs/devtron/pkg/chart/read" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/common/read" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" - constants3 "github.com/devtron-labs/devtron/pkg/pipeline/constants" util4 "github.com/devtron-labs/devtron/pkg/pipeline/util" "github.com/devtron-labs/devtron/pkg/plugin" "github.com/devtron-labs/devtron/util/beHelper" @@ -330,7 +329,7 @@ func (impl CiCdPipelineOrchestratorImpl) validateCiPipelineMaterial(ciPipelineMa func (impl CiCdPipelineOrchestratorImpl) getSkipMessage(ciPipeline *pipelineConfig.CiPipeline) string { switch ciPipeline.PipelineType { - case string(common3.LINKED_CD): + case string(buildCommonBean.LINKED_CD): return "“Sync with Environment”" default: return "“Linked Build Pipeline”" @@ -1011,7 +1010,7 @@ func (impl CiCdPipelineOrchestratorImpl) CreateCiConf(createRequest *bean.CiConf var pipelineMaterials []*pipelineConfig.CiPipelineMaterial for _, r := range ciPipeline.CiMaterial { - if ciPipeline.PipelineType == common3.LINKED_CD { + if ciPipeline.PipelineType == buildCommonBean.LINKED_CD { continue } material := &pipelineConfig.CiPipelineMaterial{ @@ -2504,7 +2503,7 @@ func (impl *CiCdPipelineOrchestratorImpl) GetWorkflowCacheConfig(appType helper. if appType == helper.Job { return util4.GetWorkflowCacheConfig(pipelineWorkflowCacheConfig, impl.workflowCacheConfig.IgnoreJob) } else { - if pipelineType == string(constants3.CI_JOB) { + if pipelineType == string(buildCommonBean.CI_JOB) { return util4.GetWorkflowCacheConfigWithBackwardCompatibility(pipelineWorkflowCacheConfig, impl.ciConfig.WorkflowCacheConfig, impl.workflowCacheConfig.IgnoreCIJob, impl.ciConfig.SkipCiJobBuildCachePushPull) } else { return util4.GetWorkflowCacheConfigWithBackwardCompatibility(pipelineWorkflowCacheConfig, impl.ciConfig.WorkflowCacheConfig, impl.workflowCacheConfig.IgnoreCI, impl.ciConfig.IgnoreDockerCacheForCI) diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index da1883339f..a2bdd0a58e 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -36,7 +36,6 @@ import ( common2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" repository6 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/pipeline/adapter" - pipelineConst "github.com/devtron-labs/devtron/pkg/pipeline/constants" "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" bean2 "github.com/devtron-labs/devtron/pkg/plugin/bean" @@ -195,7 +194,7 @@ func (impl *CiServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, } // checking if user has given run time parameters for externalCiArtifact, if given then sending git material to Ci-Runner - externalCiArtifact, exists := trigger.RuntimeParameters.GetGlobalRuntimeVariables()[pipelineConst.ExtraEnvVarExternalCiArtifactKey] + externalCiArtifact, exists := trigger.RuntimeParameters.GetGlobalRuntimeVariables()[buildBean.ExtraEnvVarExternalCiArtifactKey] // validate externalCiArtifact as docker image if exists { externalCiArtifact = strings.TrimSpace(externalCiArtifact) @@ -220,11 +219,11 @@ func (impl *CiServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, } // This will overwrite the existing runtime parameters value for constants.externalCiArtifact - trigger.RuntimeParameters = trigger.RuntimeParameters.AddRuntimeGlobalVariable(pipelineConst.ExtraEnvVarExternalCiArtifactKey, externalCiArtifact) + trigger.RuntimeParameters = trigger.RuntimeParameters.AddRuntimeGlobalVariable(buildBean.ExtraEnvVarExternalCiArtifactKey, externalCiArtifact) var artifactExists bool var err error - imageDigest, ok := trigger.RuntimeParameters.GetGlobalRuntimeVariables()[pipelineConst.ExtraEnvVarImageDigestKey] + imageDigest, ok := trigger.RuntimeParameters.GetGlobalRuntimeVariables()[buildBean.ExtraEnvVarImageDigestKey] if !ok || len(imageDigest) == 0 { artifactExists, err = impl.ciArtifactRepository.IfArtifactExistByImage(externalCiArtifact, trigger.PipelineId) if err != nil { diff --git a/pkg/pipeline/constants/constants.go b/pkg/pipeline/constants/constants.go index 243fd6679e..0028c37d45 100644 --- a/pkg/pipeline/constants/constants.go +++ b/pkg/pipeline/constants/constants.go @@ -19,19 +19,6 @@ package constants const CDPipelineNotFoundErr = "cd pipeline not found" const CiPipelineNotFoundErr = "ci pipeline not found" -type PipelineType string - -// default PipelineType -const DefaultPipelineType = CI_BUILD - -const ( - CI_BUILD PipelineType = "CI_BUILD" - LINKED PipelineType = "LINKED" - EXTERNAL PipelineType = "EXTERNAL" - CI_JOB PipelineType = "CI_JOB" - LINKED_CD PipelineType = "LINKED_CD" -) - type PatchPipelineActionResponse string const ( @@ -43,28 +30,10 @@ const ( PATCH_PIPELINE_ACTION_NOT_APPLICABLE PatchPipelineActionResponse = "N/A" ) -func (r PipelineType) ToString() string { - return string(r) -} - -func (pType PipelineType) IsValidPipelineType() bool { - switch pType { - case CI_BUILD, LINKED, EXTERNAL, CI_JOB, LINKED_CD: - return true - default: - return false - } -} - const ( UNIQUE_DEPLOYMENT_APP_NAME = "unique_deployment_app_name" ) -const ( - ExtraEnvVarExternalCiArtifactKey = "externalCiArtifact" - ExtraEnvVarImageDigestKey = "imageDigest" -) - const DefaultCiWorkflowNamespace = "devtron-ci" const Running = "Running" const Starting = "Starting" diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index 515ab4cafc..ce94737490 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -38,7 +38,7 @@ import ( bean7 "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/build/artifacts" bean5 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" - "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/manifest" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" @@ -47,7 +47,6 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/userDeploymentRequest/service" eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" "github.com/devtron-labs/devtron/pkg/pipeline" - constants2 "github.com/devtron-labs/devtron/pkg/pipeline/constants" repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning" repository3 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/repository" @@ -807,7 +806,7 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext trigger // image scanning plugin can only be applied in Post-ci, scanning in pre-ci doesn't make sense pipelineStage := repository4.PIPELINE_STAGE_TYPE_POST_CI - if pipelineModal.PipelineType == constants2.CI_JOB.ToString() { + if pipelineModal.PipelineType == buildCommonBean.CI_JOB.ToString() { pipelineStage = repository4.PIPELINE_STAGE_TYPE_PRE_CI } // this flag comes from ci-runner when scanning is enabled from ciPipeline modal @@ -831,7 +830,7 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext trigger var pluginArtifacts []*repository.CiArtifact for registry, artifacts := range request.PluginRegistryArtifactDetails { for _, image := range artifacts { - if pipelineModal.PipelineType == string(common.CI_JOB) && image == "" { + if pipelineModal.PipelineType == string(buildCommonBean.CI_JOB) && image == "" { continue } pluginArtifact := &repository.CiArtifact{ From d9dcd092470fcee623b528e53c58d8c91dfd052b Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Wed, 2 Apr 2025 08:36:13 +0530 Subject: [PATCH 03/37] ci service and trigger service creation --- .../pipeline/bean/common/CommonBuildBean.go | 13 + pkg/build/trigger/Service.go | 1086 ++++++++++++++++ .../trigger/Service_ent.go} | 6 +- pkg/build/wire_build.go | 2 + .../devtronApps/PreStageTriggerService.go | 5 +- pkg/pipeline/BuildPipelineConfigService.go | 8 +- pkg/pipeline/CiCdPipelineOrchestrator.go | 3 - pkg/pipeline/CiLogService.go | 4 +- pkg/pipeline/CiService.go | 1087 +---------------- pkg/pipeline/pipelineStageVariableParser.go | 6 +- 10 files changed, 1130 insertions(+), 1090 deletions(-) rename pkg/{pipeline/CiService_ent.go => build/trigger/Service_ent.go} (51%) diff --git a/pkg/build/pipeline/bean/common/CommonBuildBean.go b/pkg/build/pipeline/bean/common/CommonBuildBean.go index af419f962f..81692c377a 100644 --- a/pkg/build/pipeline/bean/common/CommonBuildBean.go +++ b/pkg/build/pipeline/bean/common/CommonBuildBean.go @@ -25,3 +25,16 @@ func (pType PipelineType) IsValidPipelineType() bool { return false } } + +const ( + BEFORE_DOCKER_BUILD = "BEFORE_DOCKER_BUILD" + AFTER_DOCKER_BUILD = "AFTER_DOCKER_BUILD" +) + +type RefPluginName = string + +const ( + COPY_CONTAINER_IMAGE RefPluginName = "Copy container image" + COPY_CONTAINER_IMAGE_VERSION_V1 = "1.0.0" + COPY_CONTAINER_IMAGE_VERSION_V2 = "2.0.0" +) diff --git a/pkg/build/trigger/Service.go b/pkg/build/trigger/Service.go index dee4fe7ace..67cff996ab 100644 --- a/pkg/build/trigger/Service.go +++ b/pkg/build/trigger/Service.go @@ -1 +1,1087 @@ package trigger + +import ( + "encoding/json" + "errors" + "fmt" + "github.com/caarlos0/env" + blob_storage "github.com/devtron-labs/common-lib/blob-storage" + "github.com/devtron-labs/common-lib/utils" + bean4 "github.com/devtron-labs/common-lib/utils/bean" + commonBean "github.com/devtron-labs/common-lib/workflow" + client "github.com/devtron-labs/devtron/client/events" + "github.com/devtron-labs/devtron/internal/middleware" + "github.com/devtron-labs/devtron/internal/sql/constants" + repository5 "github.com/devtron-labs/devtron/internal/sql/repository" + appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" + repository3 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" + "github.com/devtron-labs/devtron/internal/sql/repository/helper" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" + "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/app" + "github.com/devtron-labs/devtron/pkg/attributes" + bean3 "github.com/devtron-labs/devtron/pkg/attributes/bean" + "github.com/devtron-labs/devtron/pkg/auth/user" + "github.com/devtron-labs/devtron/pkg/bean" + "github.com/devtron-labs/devtron/pkg/bean/common" + "github.com/devtron-labs/devtron/pkg/build/pipeline" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" + repository6 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" + pipeline2 "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/pipeline/adapter" + pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/repository" + "github.com/devtron-labs/devtron/pkg/pipeline/types" + "github.com/devtron-labs/devtron/pkg/plugin" + bean2 "github.com/devtron-labs/devtron/pkg/plugin/bean" + repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository" + "github.com/devtron-labs/devtron/pkg/resourceQualifiers" + "github.com/devtron-labs/devtron/pkg/variables" + repository4 "github.com/devtron-labs/devtron/pkg/variables/repository" + "github.com/devtron-labs/devtron/util/sliceUtil" + "github.com/go-pg/pg" + "go.uber.org/zap" + "net/http" + "path" + "path/filepath" + "slices" + "strconv" + "strings" + "time" +) + +type Service interface { + TriggerCiPipeline(trigger types.Trigger) (int, error) +} + +type BuildxCacheFlags struct { + BuildxCacheModeMin bool `env:"BUILDX_CACHE_MODE_MIN" envDefault:"false"` + AsyncBuildxCacheExport bool `env:"ASYNC_BUILDX_CACHE_EXPORT" envDefault:"false"` +} + +type ServiceImpl struct { + Logger *zap.SugaredLogger + workflowService pipeline2.WorkflowService + ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository + ciPipelineRepository pipelineConfig.CiPipelineRepository + ciArtifactRepository repository5.CiArtifactRepository + pipelineStageService pipeline2.PipelineStageService + userService user.UserService + ciTemplateService pipeline.CiTemplateReadService + appCrudOperationService app.AppCrudOperationService + envRepository repository6.EnvironmentRepository + appRepository appRepository.AppRepository + customTagService pipeline2.CustomTagService + config *types.CiConfig + scopedVariableManager variables.ScopedVariableManager + ciCdPipelineOrchestrator pipeline2.CiCdPipelineOrchestrator + buildxCacheFlags *BuildxCacheFlags + attributeService attributes.AttributesService + pluginInputVariableParser pipeline2.PluginInputVariableParser + globalPluginService plugin.GlobalPluginService + ciService pipeline2.CiService +} + +func NewServiceImpl(Logger *zap.SugaredLogger, workflowService pipeline2.WorkflowService, + ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, + ciPipelineRepository pipelineConfig.CiPipelineRepository, + ciArtifactRepository repository5.CiArtifactRepository, + pipelineStageService pipeline2.PipelineStageService, + userService user.UserService, + ciTemplateService pipeline.CiTemplateReadService, + appCrudOperationService app.AppCrudOperationService, + envRepository repository6.EnvironmentRepository, + appRepository appRepository.AppRepository, + scopedVariableManager variables.ScopedVariableManager, + customTagService pipeline2.CustomTagService, + ciCdPipelineOrchestrator pipeline2.CiCdPipelineOrchestrator, attributeService attributes.AttributesService, + pluginInputVariableParser pipeline2.PluginInputVariableParser, + globalPluginService plugin.GlobalPluginService, + ciService pipeline2.CiService, +) *ServiceImpl { + buildxCacheFlags := &BuildxCacheFlags{} + err := env.Parse(buildxCacheFlags) + if err != nil { + Logger.Infow("error occurred while parsing BuildxCacheFlags env,so setting BuildxCacheModeMin and AsyncBuildxCacheExport to default value", "err", err) + } + cis := &ServiceImpl{ + Logger: Logger, + workflowService: workflowService, + ciPipelineMaterialRepository: ciPipelineMaterialRepository, + ciPipelineRepository: ciPipelineRepository, + ciArtifactRepository: ciArtifactRepository, + pipelineStageService: pipelineStageService, + userService: userService, + ciTemplateService: ciTemplateService, + appCrudOperationService: appCrudOperationService, + envRepository: envRepository, + appRepository: appRepository, + scopedVariableManager: scopedVariableManager, + customTagService: customTagService, + ciCdPipelineOrchestrator: ciCdPipelineOrchestrator, + buildxCacheFlags: buildxCacheFlags, + attributeService: attributeService, + pluginInputVariableParser: pluginInputVariableParser, + globalPluginService: globalPluginService, + ciService: ciService, + } + config, err := types.GetCiConfig() + if err != nil { + return nil + } + cis.config = config + return cis +} + +func (impl *ServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { + impl.Logger.Debug("ci pipeline manual trigger") + ciMaterials, err := impl.GetCiMaterials(trigger.PipelineId, trigger.CiMaterials) + if err != nil { + return 0, err + } + + ciPipelineScripts, err := impl.ciPipelineRepository.FindCiScriptsByCiPipelineId(trigger.PipelineId) + if err != nil && !util.IsErrNoRows(err) { + return 0, err + } + + var pipeline *pipelineConfig.CiPipeline + for _, m := range ciMaterials { + pipeline = m.CiPipeline + break + } + + scope := resourceQualifiers.Scope{ + AppId: pipeline.App.Id, + } + ciWorkflowConfigNamespace := impl.config.GetDefaultNamespace() + envModal, isJob, err := impl.getEnvironmentForJob(pipeline, trigger) + if err != nil { + return 0, err + } + if isJob && envModal != nil { + ciWorkflowConfigNamespace = envModal.Namespace + + // This will be populated for jobs running in selected environment + scope.EnvId = envModal.Id + scope.ClusterId = envModal.ClusterId + + scope.SystemMetadata = &resourceQualifiers.SystemMetadata{ + EnvironmentName: envModal.Name, + ClusterName: envModal.Cluster.ClusterName, + Namespace: envModal.Namespace, + } + } + if scope.SystemMetadata == nil { + scope.SystemMetadata = &resourceQualifiers.SystemMetadata{ + Namespace: ciWorkflowConfigNamespace, + AppName: pipeline.App.AppName, + } + } + + savedCiWf, err := impl.saveNewWorkflow(pipeline, ciWorkflowConfigNamespace, trigger.CommitHashes, trigger.TriggeredBy, trigger.EnvironmentId, isJob, trigger.ReferenceCiWorkflowId) + if err != nil { + impl.Logger.Errorw("could not save new workflow", "err", err) + return 0, err + } + + // preCiSteps, postCiSteps, refPluginsData, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(pipeline.Id, ciEvent) + request := pipelineConfigBean.NewBuildPrePostStepDataReq(pipeline.Id, pipelineConfigBean.CiStage, scope) + prePostAndRefPluginResponse, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(request) + if err != nil { + impl.Logger.Errorw("error in getting pre steps data for wf request", "err", err, "ciPipelineId", pipeline.Id) + dbErr := impl.markCurrentCiWorkflowFailed(savedCiWf, err) + if dbErr != nil { + impl.Logger.Errorw("saving workflow error", "err", dbErr) + } + return 0, err + } + preCiSteps := prePostAndRefPluginResponse.PreStageSteps + postCiSteps := prePostAndRefPluginResponse.PostStageSteps + refPluginsData := prePostAndRefPluginResponse.RefPluginData + variableSnapshot := prePostAndRefPluginResponse.VariableSnapshot + + if len(preCiSteps) == 0 && isJob { + errMsg := fmt.Sprintf("No tasks are configured in this job pipeline") + validationErr := util.NewApiError(http.StatusNotFound, errMsg, errMsg) + + return 0, validationErr + } + + // get env variables of git trigger data and add it in the extraEnvVariables + gitTriggerEnvVariables, _, err := impl.ciCdPipelineOrchestrator.GetGitCommitEnvVarDataForCICDStage(savedCiWf.GitTriggers) + if err != nil { + impl.Logger.Errorw("error in getting gitTrigger env data for stage", "gitTriggers", savedCiWf.GitTriggers, "err", err) + return 0, err + } + + for k, v := range gitTriggerEnvVariables { + trigger.RuntimeParameters = trigger.RuntimeParameters.AddSystemVariable(k, v) + } + + workflowRequest, err := impl.buildWfRequestForCiPipeline(pipeline, trigger, ciMaterials, savedCiWf, ciWorkflowConfigNamespace, ciPipelineScripts, preCiSteps, postCiSteps, refPluginsData, isJob) + if err != nil { + impl.Logger.Errorw("make workflow req", "err", err) + return 0, err + } + err = impl.handleRuntimeParamsValidations(trigger, ciMaterials, workflowRequest) + if err != nil { + savedCiWf.Status = cdWorkflow.WorkflowAborted + savedCiWf.Message = err.Error() + err1 := impl.ciService.UpdateCiWorkflowWithStage(savedCiWf) + if err1 != nil { + impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag") + } + return 0, err + } + + workflowRequest.Scope = scope + workflowRequest.AppId = pipeline.AppId + workflowRequest.BuildxCacheModeMin = impl.buildxCacheFlags.BuildxCacheModeMin + workflowRequest.AsyncBuildxCacheExport = impl.buildxCacheFlags.AsyncBuildxCacheExport + if impl.config != nil && impl.config.BuildxK8sDriverOptions != "" { + err = impl.setBuildxK8sDriverData(workflowRequest) + if err != nil { + impl.Logger.Errorw("error in setBuildxK8sDriverData", "BUILDX_K8S_DRIVER_OPTIONS", impl.config.BuildxK8sDriverOptions, "err", err) + return 0, err + } + } + + // savedCiWf.LogLocation = impl.ciCdConfig.CiDefaultBuildLogsKeyPrefix + "/" + workflowRequest.WorkflowNamePrefix + "/main.log" + savedCiWf.LogLocation = fmt.Sprintf("%s/%s/main.log", impl.config.GetDefaultBuildLogsKeyPrefix(), workflowRequest.WorkflowNamePrefix) + err = impl.updateCiWorkflow(workflowRequest, savedCiWf) + + appLabels, err := impl.appCrudOperationService.GetLabelsByAppId(pipeline.AppId) + if err != nil { + return 0, err + } + workflowRequest.AppLabels = appLabels + workflowRequest.Env = envModal + if isJob { + workflowRequest.Type = pipelineConfigBean.JOB_WORKFLOW_PIPELINE_TYPE + } else { + workflowRequest.Type = pipelineConfigBean.CI_WORKFLOW_PIPELINE_TYPE + } + + workflowRequest.CiPipelineType = trigger.PipelineType + err = impl.executeCiPipeline(workflowRequest) + if err != nil { + impl.Logger.Errorw("error in executing ci pipeline", "err", err) + dbErr := impl.markCurrentCiWorkflowFailed(savedCiWf, err) + if dbErr != nil { + impl.Logger.Errorw("update ci workflow error", "err", dbErr) + } + return 0, err + } + impl.Logger.Debugw("ci triggered", " pipeline ", trigger.PipelineId) + + var variableSnapshotHistories = sliceUtil.GetBeansPtr( + repository4.GetSnapshotBean(savedCiWf.Id, repository4.HistoryReferenceTypeCIWORKFLOW, variableSnapshot)) + if len(variableSnapshotHistories) > 0 { + err = impl.scopedVariableManager.SaveVariableHistoriesForTrigger(variableSnapshotHistories, trigger.TriggeredBy) + if err != nil { + impl.Logger.Errorf("Not able to save variable snapshot for CI trigger %s", err) + } + } + + middleware.CiTriggerCounter.WithLabelValues(pipeline.App.AppName, pipeline.Name).Inc() + go impl.ciService.WriteCITriggerEvent(trigger, pipeline, workflowRequest) + return savedCiWf.Id, err +} + +func (impl *ServiceImpl) GetCiMaterials(pipelineId int, ciMaterials []*pipelineConfig.CiPipelineMaterial) ([]*pipelineConfig.CiPipelineMaterial, error) { + if !(len(ciMaterials) == 0) { + return ciMaterials, nil + } else { + ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(pipelineId) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return nil, err + } + impl.Logger.Debug("ciMaterials for pipeline trigger ", ciMaterials) + return ciMaterials, nil + } +} + +func (impl *ServiceImpl) setBuildxK8sDriverData(workflowRequest *types.WorkflowRequest) error { + ciBuildConfig := workflowRequest.CiBuildConfig + if ciBuildConfig != nil { + if dockerBuildConfig := ciBuildConfig.DockerBuildConfig; dockerBuildConfig != nil { + k8sDriverOptions, err := impl.getK8sDriverOptions() + if err != nil { + errMsg := "error in parsing BUILDX_K8S_DRIVER_OPTIONS from the devtron-cm, " + err = errors.New(errMsg + "error : " + err.Error()) + impl.Logger.Errorw(errMsg, "err", err) + } + dockerBuildConfig.BuildxK8sDriverOptions = k8sDriverOptions + + } + } + return nil +} + +func (impl *ServiceImpl) getK8sDriverOptions() ([]map[string]string, error) { + buildxK8sDriverOptions := make([]map[string]string, 0) + err := json.Unmarshal([]byte(impl.config.BuildxK8sDriverOptions), &buildxK8sDriverOptions) + if err != nil { + return nil, err + } else { + return buildxK8sDriverOptions, nil + } +} + +func (impl *ServiceImpl) getEnvironmentForJob(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger) (*repository6.Environment, bool, error) { + app, err := impl.appRepository.FindById(pipeline.AppId) + if err != nil { + impl.Logger.Errorw("could not find app", "err", err) + return nil, false, err + } + + var env *repository6.Environment + isJob := false + if app.AppType == helper.Job { + isJob = true + if trigger.EnvironmentId != 0 { + env, err = impl.envRepository.FindById(trigger.EnvironmentId) + if err != nil { + impl.Logger.Errorw("could not find environment", "err", err) + return nil, isJob, err + } + return env, isJob, nil + } + } + return nil, isJob, nil +} + +// TODO: Send all trigger data +func (impl *ServiceImpl) BuildPayload(trigger types.Trigger, pipeline *pipelineConfig.CiPipeline) *client.Payload { + payload := &client.Payload{} + payload.AppName = pipeline.App.AppName + payload.PipelineName = pipeline.Name + return payload +} + +func (impl *ServiceImpl) saveNewWorkflow(pipeline *pipelineConfig.CiPipeline, ciWorkflowConfigNamespace string, + commitHashes map[int]pipelineConfig.GitCommit, userId int32, EnvironmentId int, isJob bool, refCiWorkflowId int) (wf *pipelineConfig.CiWorkflow, error error) { + + ciWorkflow := &pipelineConfig.CiWorkflow{ + Name: pipeline.Name + "-" + strconv.Itoa(pipeline.Id), + Status: cdWorkflow.WorkflowStarting, // starting CIStage + Message: "", + StartedOn: time.Now(), + CiPipelineId: pipeline.Id, + Namespace: impl.config.GetDefaultNamespace(), + BlobStorageEnabled: impl.config.BlobStorageEnabled, + GitTriggers: commitHashes, + LogLocation: "", + TriggeredBy: userId, + ReferenceCiWorkflowId: refCiWorkflowId, + ExecutorType: impl.config.GetWorkflowExecutorType(), + } + if isJob { + ciWorkflow.Namespace = ciWorkflowConfigNamespace + ciWorkflow.EnvironmentId = EnvironmentId + } + err := impl.ciService.SaveCiWorkflowWithStage(ciWorkflow) + if err != nil { + impl.Logger.Errorw("saving workflow error", "err", err) + return &pipelineConfig.CiWorkflow{}, err + } + impl.Logger.Debugw("workflow saved ", "id", ciWorkflow.Id) + return ciWorkflow, nil +} + +func (impl *ServiceImpl) executeCiPipeline(workflowRequest *types.WorkflowRequest) error { + _, _, err := impl.workflowService.SubmitWorkflow(workflowRequest) + if err != nil { + impl.Logger.Errorw("workflow error", "err", err) + return err + } + return nil +} + +func (impl *ServiceImpl) buildS3ArtifactLocation(ciWorkflowConfigLogsBucket string, savedWf *pipelineConfig.CiWorkflow) (string, string, string) { + ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() + ArtifactLocation := fmt.Sprintf("s3://"+path.Join(ciWorkflowConfigLogsBucket, ciArtifactLocationFormat), savedWf.Id, savedWf.Id) + artifactFileName := fmt.Sprintf(ciArtifactLocationFormat, savedWf.Id, savedWf.Id) + return ArtifactLocation, ciWorkflowConfigLogsBucket, artifactFileName +} + +func (impl *ServiceImpl) buildDefaultArtifactLocation(savedWf *pipelineConfig.CiWorkflow) string { + ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() + ArtifactLocation := fmt.Sprintf(ciArtifactLocationFormat, savedWf.Id, savedWf.Id) + return ArtifactLocation +} + +func (impl *ServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, savedWf *pipelineConfig.CiWorkflow, ciWorkflowConfigNamespace string, ciPipelineScripts []*pipelineConfig.CiPipelineScript, preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, refPluginsData []*pipelineConfigBean.RefPluginObject, isJob bool) (*types.WorkflowRequest, error) { + var ciProjectDetails []pipelineConfigBean.CiProjectDetails + commitHashes := trigger.CommitHashes + for _, ciMaterial := range ciMaterials { + // ignore those materials which have inactive git material + if ciMaterial == nil || ciMaterial.GitMaterial == nil || !ciMaterial.GitMaterial.Active { + continue + } + commitHashForPipelineId := commitHashes[ciMaterial.Id] + ciProjectDetail := pipelineConfigBean.CiProjectDetails{ + GitRepository: ciMaterial.GitMaterial.Url, + MaterialName: ciMaterial.GitMaterial.Name, + CheckoutPath: ciMaterial.GitMaterial.CheckoutPath, + FetchSubmodules: ciMaterial.GitMaterial.FetchSubmodules, + CommitHash: commitHashForPipelineId.Commit, + Author: commitHashForPipelineId.Author, + SourceType: ciMaterial.Type, + SourceValue: ciMaterial.Value, + GitTag: ciMaterial.GitTag, + Message: commitHashForPipelineId.Message, + Type: string(ciMaterial.Type), + CommitTime: commitHashForPipelineId.Date.Format(bean.LayoutRFC3339), + GitOptions: pipelineConfigBean.GitOptions{ + UserName: ciMaterial.GitMaterial.GitProvider.UserName, + Password: ciMaterial.GitMaterial.GitProvider.Password, + SshPrivateKey: ciMaterial.GitMaterial.GitProvider.SshPrivateKey, + AccessToken: ciMaterial.GitMaterial.GitProvider.AccessToken, + AuthMode: ciMaterial.GitMaterial.GitProvider.AuthMode, + EnableTLSVerification: ciMaterial.GitMaterial.GitProvider.EnableTLSVerification, + TlsKey: ciMaterial.GitMaterial.GitProvider.TlsKey, + TlsCert: ciMaterial.GitMaterial.GitProvider.TlsCert, + CaCert: ciMaterial.GitMaterial.GitProvider.CaCert, + }, + } + + if ciMaterial.Type == constants.SOURCE_TYPE_WEBHOOK { + webhookData := commitHashForPipelineId.WebhookData + ciProjectDetail.WebhookData = pipelineConfig.WebhookData{ + Id: webhookData.Id, + EventActionType: webhookData.EventActionType, + Data: webhookData.Data, + } + } + + ciProjectDetails = append(ciProjectDetails, ciProjectDetail) + } + + var beforeDockerBuildScripts []*bean.CiScript + var afterDockerBuildScripts []*bean.CiScript + for _, ciPipelineScript := range ciPipelineScripts { + ciTask := &bean.CiScript{ + Id: ciPipelineScript.Id, + Index: ciPipelineScript.Index, + Name: ciPipelineScript.Name, + Script: ciPipelineScript.Script, + OutputLocation: ciPipelineScript.OutputLocation, + } + + if ciPipelineScript.Stage == buildCommonBean.BEFORE_DOCKER_BUILD { + beforeDockerBuildScripts = append(beforeDockerBuildScripts, ciTask) + } else if ciPipelineScript.Stage == buildCommonBean.AFTER_DOCKER_BUILD { + afterDockerBuildScripts = append(afterDockerBuildScripts, ciTask) + } + } + + if !(len(beforeDockerBuildScripts) == 0 && len(afterDockerBuildScripts) == 0) { + // found beforeDockerBuildScripts/afterDockerBuildScripts + // building preCiSteps & postCiSteps from them, refPluginsData not needed + preCiSteps = buildCiStepsDataFromDockerBuildScripts(beforeDockerBuildScripts) + postCiSteps = buildCiStepsDataFromDockerBuildScripts(afterDockerBuildScripts) + refPluginsData = []*pipelineConfigBean.RefPluginObject{} + } + + host, err := impl.attributeService.GetByKey(bean3.HostUrlKey) + if err != nil { + impl.Logger.Errorw("error in getting host url", "err", err, "hostUrl", host.Value) + return nil, err + } + ciWorkflowConfigCiCacheBucket := impl.config.DefaultCacheBucket + + ciWorkflowConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion + + ciWorkflowConfigCiImage := impl.config.GetDefaultImage() + + ciTemplate := pipeline.CiTemplate + ciLevelArgs := pipeline.DockerArgs + + if ciLevelArgs == "" { + ciLevelArgs = "{}" + } + + if pipeline.CiTemplate.DockerBuildOptions == "" { + pipeline.CiTemplate.DockerBuildOptions = "{}" + } + userEmailId, err := impl.userService.GetActiveEmailById(trigger.TriggeredBy) + if err != nil { + impl.Logger.Errorw("unable to find user email by id", "err", err, "id", trigger.TriggeredBy) + return nil, err + } + var dockerfilePath string + var dockerRepository string + var checkoutPath string + var ciBuildConfigBean *buildBean.CiBuildConfigBean + dockerRegistry := &repository3.DockerArtifactStore{} + ciBaseBuildConfigEntity := ciTemplate.CiBuildConfig + ciBaseBuildConfigBean, err := adapter.ConvertDbBuildConfigToBean(ciBaseBuildConfigEntity) + if err != nil { + impl.Logger.Errorw("error occurred while converting buildconfig dbEntity to configBean", "ciBuildConfigEntity", ciBaseBuildConfigEntity, "err", err) + return nil, errors.New("error while parsing ci build config") + } + if !pipeline.IsExternal && pipeline.IsDockerConfigOverridden { + templateOverrideBean, err := impl.ciTemplateService.FindTemplateOverrideByCiPipelineId(pipeline.Id) + if err != nil { + return nil, err + } + ciBuildConfigBean = templateOverrideBean.CiBuildConfig + // updating args coming from ciBaseBuildConfigEntity because it is not part of Ci override + if ciBuildConfigBean != nil && ciBuildConfigBean.DockerBuildConfig != nil && ciBaseBuildConfigBean != nil && ciBaseBuildConfigBean.DockerBuildConfig != nil { + ciBuildConfigBean.DockerBuildConfig.Args = ciBaseBuildConfigBean.DockerBuildConfig.Args + } + templateOverride := templateOverrideBean.CiTemplateOverride + checkoutPath = templateOverride.GitMaterial.CheckoutPath + dockerfilePath = templateOverride.DockerfilePath + dockerRepository = templateOverride.DockerRepository + dockerRegistry = templateOverride.DockerRegistry + } else { + checkoutPath = ciTemplate.GitMaterial.CheckoutPath + dockerfilePath = ciTemplate.DockerfilePath + dockerRegistry = ciTemplate.DockerRegistry + dockerRepository = ciTemplate.DockerRepository + ciBuildConfigBean = ciBaseBuildConfigBean + if ciBuildConfigBean != nil { + ciBuildConfigBean.BuildContextGitMaterialId = ciTemplate.BuildContextGitMaterialId + } + + } + if checkoutPath == "" { + checkoutPath = "./" + } + var dockerImageTag string + customTag, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id)) + if err != nil && err != pg.ErrNoRows { + return nil, err + } + if customTag.Id != 0 && customTag.Enabled == true { + imagePathReservation, err := impl.customTagService.GenerateImagePath(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id), dockerRegistry.RegistryURL, dockerRepository) + if err != nil { + if errors.Is(err, pipelineConfigBean.ErrImagePathInUse) { + errMsg := pipelineConfigBean.ImageTagUnavailableMessage + validationErr := util.NewApiError(http.StatusConflict, errMsg, errMsg) + dbErr := impl.markCurrentCiWorkflowFailed(savedWf, validationErr) + if dbErr != nil { + impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag", "err", dbErr, "savedWf", savedWf.Id) + } + return nil, err + } + return nil, err + } + savedWf.ImagePathReservationIds = []int{imagePathReservation.Id} + // imagePath = docker.io/avd0/dashboard:fd23414b + imagePathSplit := strings.Split(imagePathReservation.ImagePath, ":") + if len(imagePathSplit) >= 1 { + dockerImageTag = imagePathSplit[len(imagePathSplit)-1] + } + } else { + dockerImageTag = impl.buildImageTag(commitHashes, pipeline.Id, savedWf.Id) + } + + // copyContainerImage plugin specific logic + var registryCredentialMap map[string]bean2.RegistryCredentials + var pluginArtifactStage string + var imageReservationIds []int + var registryDestinationImageMap map[string][]string + if !isJob { + registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imageReservationIds, err = impl.GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps, postCiSteps, dockerImageTag, customTag.Id, + fmt.Sprintf(pipelineConfigBean.ImagePathPattern, + dockerRegistry.RegistryURL, + dockerRepository, + dockerImageTag), + dockerRegistry.Id) + if err != nil { + impl.Logger.Errorw("error in getting env variables for copyContainerImage plugin") + dbErr := impl.markCurrentCiWorkflowFailed(savedWf, err) + if dbErr != nil { + impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag", "err", dbErr, "savedWf", savedWf.Id) + } + return nil, err + } + + savedWf.ImagePathReservationIds = append(savedWf.ImagePathReservationIds, imageReservationIds...) + } + // mergedArgs := string(merged) + oldArgs := ciTemplate.Args + ciBuildConfigBean, err = adapter.OverrideCiBuildConfig(dockerfilePath, oldArgs, ciLevelArgs, ciTemplate.DockerBuildOptions, ciTemplate.TargetPlatform, ciBuildConfigBean) + if err != nil { + impl.Logger.Errorw("error occurred while overriding ci build config", "oldArgs", oldArgs, "ciLevelArgs", ciLevelArgs, "error", err) + return nil, errors.New("error while parsing ci build config") + } + buildContextCheckoutPath, err := impl.ciPipelineMaterialRepository.GetCheckoutPath(ciBuildConfigBean.BuildContextGitMaterialId) + if err != nil && err != pg.ErrNoRows { + impl.Logger.Errorw("error occurred while getting checkout path from git material", "gitMaterialId", ciBuildConfigBean.BuildContextGitMaterialId, "error", err) + return nil, err + } + if buildContextCheckoutPath == "" { + buildContextCheckoutPath = checkoutPath + } + if ciBuildConfigBean.UseRootBuildContext { + // use root build context i.e '.' + buildContextCheckoutPath = "." + } + + ciBuildConfigBean.PipelineType = trigger.PipelineType + + if ciBuildConfigBean.CiBuildType == buildBean.SELF_DOCKERFILE_BUILD_TYPE || ciBuildConfigBean.CiBuildType == buildBean.MANAGED_DOCKERFILE_BUILD_TYPE { + ciBuildConfigBean.DockerBuildConfig.BuildContext = filepath.Join(buildContextCheckoutPath, ciBuildConfigBean.DockerBuildConfig.BuildContext) + dockerBuildConfig := ciBuildConfigBean.DockerBuildConfig + dockerfilePath = filepath.Join(checkoutPath, dockerBuildConfig.DockerfilePath) + dockerBuildConfig.DockerfilePath = dockerfilePath + checkoutPath = dockerfilePath[:strings.LastIndex(dockerfilePath, "/")+1] + } else if ciBuildConfigBean.CiBuildType == buildBean.BUILDPACK_BUILD_TYPE { + buildPackConfig := ciBuildConfigBean.BuildPackConfig + checkoutPath = filepath.Join(checkoutPath, buildPackConfig.ProjectPath) + } + + defaultTargetPlatform := impl.config.DefaultTargetPlatform + useBuildx := impl.config.UseBuildx + + if ciBuildConfigBean.DockerBuildConfig != nil { + if ciBuildConfigBean.DockerBuildConfig.TargetPlatform == "" && useBuildx { + ciBuildConfigBean.DockerBuildConfig.TargetPlatform = defaultTargetPlatform + ciBuildConfigBean.DockerBuildConfig.UseBuildx = useBuildx + } + ciBuildConfigBean.DockerBuildConfig.BuildxProvenanceMode = impl.config.BuildxProvenanceMode + } + + workflowRequest := &types.WorkflowRequest{ + WorkflowNamePrefix: strconv.Itoa(savedWf.Id) + "-" + savedWf.Name, + PipelineName: pipeline.Name, + PipelineId: pipeline.Id, + CiCacheFileName: pipeline.Name + "-" + strconv.Itoa(pipeline.Id) + ".tar.gz", + CiProjectDetails: ciProjectDetails, + Namespace: ciWorkflowConfigNamespace, + BlobStorageConfigured: savedWf.BlobStorageEnabled, + CiImage: ciWorkflowConfigCiImage, + WorkflowId: savedWf.Id, + TriggeredBy: savedWf.TriggeredBy, + CacheLimit: impl.config.CacheLimit, + ScanEnabled: pipeline.ScanEnabled, + CloudProvider: impl.config.CloudProvider, + DefaultAddressPoolBaseCidr: impl.config.GetDefaultAddressPoolBaseCidr(), + DefaultAddressPoolSize: impl.config.GetDefaultAddressPoolSize(), + PreCiSteps: preCiSteps, + PostCiSteps: postCiSteps, + RefPlugins: refPluginsData, + AppName: pipeline.App.AppName, + TriggerByAuthor: userEmailId, + CiBuildConfig: ciBuildConfigBean, + CiBuildDockerMtuValue: impl.config.CiRunnerDockerMTUValue, + CacheInvalidate: trigger.InvalidateCache, + SystemEnvironmentVariables: trigger.RuntimeParameters.GetSystemVariables(), + EnableBuildContext: impl.config.EnableBuildContext, + OrchestratorHost: impl.config.OrchestratorHost, + HostUrl: host.Value, + OrchestratorToken: impl.config.OrchestratorToken, + ImageRetryCount: impl.config.ImageRetryCount, + ImageRetryInterval: impl.config.ImageRetryInterval, + WorkflowExecutor: impl.config.GetWorkflowExecutorType(), + Type: pipelineConfigBean.CI_WORKFLOW_PIPELINE_TYPE, + CiArtifactLastFetch: trigger.CiArtifactLastFetch, + RegistryDestinationImageMap: registryDestinationImageMap, + RegistryCredentialMap: registryCredentialMap, + PluginArtifactStage: pluginArtifactStage, + ImageScanMaxRetries: impl.config.ImageScanMaxRetries, + ImageScanRetryDelay: impl.config.ImageScanRetryDelay, + UseDockerApiToGetDigest: impl.config.UseDockerApiToGetDigest, + } + workflowRequest.SetAwsInspectorConfig("") + //in oss, there is no pipeline level workflow cache config, so we pass inherit to get the app level config + workflowCacheConfig := impl.ciCdPipelineOrchestrator.GetWorkflowCacheConfig(pipeline.App.AppType, trigger.PipelineType, common.WorkflowCacheConfigInherit) + workflowRequest.IgnoreDockerCachePush = !workflowCacheConfig.Value + workflowRequest.IgnoreDockerCachePull = !workflowCacheConfig.Value + impl.Logger.Debugw("Ignore Cache values", "IgnoreDockerCachePush", workflowRequest.IgnoreDockerCachePush, "IgnoreDockerCachePull", workflowRequest.IgnoreDockerCachePull) + if pipeline.App.AppType == helper.Job { + workflowRequest.AppName = pipeline.App.DisplayName + } + if pipeline.ScanEnabled { + scanToolMetadata, scanVia, err := impl.fetchImageScanExecutionMedium() + if err != nil { + impl.Logger.Errorw("error occurred getting scanned via", "err", err) + return nil, err + } + workflowRequest.SetExecuteImageScanningVia(scanVia) + if scanVia.IsScanMediumExternal() { + imageScanExecutionSteps, refPlugins, err := impl.fetchImageScanExecutionStepsForWfRequest(scanToolMetadata) + if err != nil { + impl.Logger.Errorw("error occurred, fetchImageScanExecutionStepsForWfRequest", "scanToolMetadata", scanToolMetadata, "err", err) + return nil, err + } + workflowRequest.SetImageScanningSteps(imageScanExecutionSteps) + workflowRequest.RefPlugins = append(workflowRequest.RefPlugins, refPlugins...) + } + } + if dockerRegistry != nil { + workflowRequest.DockerRegistryId = dockerRegistry.Id + workflowRequest.DockerRegistryType = string(dockerRegistry.RegistryType) + workflowRequest.DockerImageTag = dockerImageTag + workflowRequest.DockerRegistryURL = dockerRegistry.RegistryURL + workflowRequest.DockerRepository = dockerRepository + workflowRequest.CheckoutPath = checkoutPath + workflowRequest.DockerUsername = dockerRegistry.Username + workflowRequest.DockerPassword = dockerRegistry.Password + workflowRequest.AwsRegion = dockerRegistry.AWSRegion + workflowRequest.AccessKey = dockerRegistry.AWSAccessKeyId + workflowRequest.SecretKey = dockerRegistry.AWSSecretAccessKey + workflowRequest.DockerConnection = dockerRegistry.Connection + workflowRequest.DockerCert = dockerRegistry.Cert + + } + ciWorkflowConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() + + switch workflowRequest.CloudProvider { + case types.BLOB_STORAGE_S3: + // No AccessKey is used for uploading artifacts, instead IAM based auth is used + workflowRequest.CiCacheRegion = ciWorkflowConfigCiCacheRegion + workflowRequest.CiCacheLocation = ciWorkflowConfigCiCacheBucket + workflowRequest.CiArtifactLocation, workflowRequest.CiArtifactBucket, workflowRequest.CiArtifactFileName = impl.buildS3ArtifactLocation(ciWorkflowConfigLogsBucket, savedWf) + workflowRequest.BlobStorageS3Config = &blob_storage.BlobStorageS3Config{ + AccessKey: impl.config.BlobStorageS3AccessKey, + Passkey: impl.config.BlobStorageS3SecretKey, + EndpointUrl: impl.config.BlobStorageS3Endpoint, + IsInSecure: impl.config.BlobStorageS3EndpointInsecure, + CiCacheBucketName: ciWorkflowConfigCiCacheBucket, + CiCacheRegion: ciWorkflowConfigCiCacheRegion, + CiCacheBucketVersioning: impl.config.BlobStorageS3BucketVersioned, + CiArtifactBucketName: workflowRequest.CiArtifactBucket, + CiArtifactRegion: impl.config.GetDefaultCdLogsBucketRegion(), + CiArtifactBucketVersioning: impl.config.BlobStorageS3BucketVersioned, + CiLogBucketName: impl.config.GetDefaultBuildLogsBucket(), + CiLogRegion: impl.config.GetDefaultCdLogsBucketRegion(), + CiLogBucketVersioning: impl.config.BlobStorageS3BucketVersioned, + } + case types.BLOB_STORAGE_GCP: + workflowRequest.GcpBlobConfig = &blob_storage.GcpBlobConfig{ + CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, + CacheBucketName: ciWorkflowConfigCiCacheBucket, + LogBucketName: ciWorkflowConfigLogsBucket, + ArtifactBucketName: ciWorkflowConfigLogsBucket, + } + workflowRequest.CiArtifactLocation = impl.buildDefaultArtifactLocation(savedWf) + workflowRequest.CiArtifactFileName = workflowRequest.CiArtifactLocation + case types.BLOB_STORAGE_AZURE: + workflowRequest.AzureBlobConfig = &blob_storage.AzureBlobConfig{ + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, + AccountName: impl.config.AzureAccountName, + BlobContainerCiCache: impl.config.AzureBlobContainerCiCache, + AccountKey: impl.config.AzureAccountKey, + BlobContainerCiLog: impl.config.AzureBlobContainerCiLog, + BlobContainerArtifact: impl.config.AzureBlobContainerCiLog, + } + workflowRequest.BlobStorageS3Config = &blob_storage.BlobStorageS3Config{ + EndpointUrl: impl.config.AzureGatewayUrl, + IsInSecure: impl.config.AzureGatewayConnectionInsecure, + CiLogBucketName: impl.config.AzureBlobContainerCiLog, + CiLogRegion: impl.config.DefaultCacheBucketRegion, + CiLogBucketVersioning: impl.config.BlobStorageS3BucketVersioned, + AccessKey: impl.config.AzureAccountName, + } + workflowRequest.CiArtifactLocation = impl.buildDefaultArtifactLocation(savedWf) + workflowRequest.CiArtifactFileName = workflowRequest.CiArtifactLocation + default: + if impl.config.BlobStorageEnabled { + return nil, fmt.Errorf("blob storage %s not supported", workflowRequest.CloudProvider) + } + } + return workflowRequest, nil +} + +func (impl *ServiceImpl) GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, customTag string, customTagId int, buildImagePath string, buildImagedockerRegistryId string) (map[string][]string, map[string]bean2.RegistryCredentials, string, []int, error) { + + copyContainerImagePluginDetail, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(buildCommonBean.COPY_CONTAINER_IMAGE) + if err != nil && err != pg.ErrNoRows { + impl.Logger.Errorw("error in getting copyContainerImage plugin id", "err", err) + return nil, nil, "", nil, err + } + + pluginIdToVersionMap := make(map[int]string) + for _, p := range copyContainerImagePluginDetail { + pluginIdToVersionMap[p.Id] = p.Version + } + + for _, step := range preCiSteps { + if _, ok := pluginIdToVersionMap[step.RefPluginId]; ok { + // for copyContainerImage plugin parse destination images and save its data in image path reservation table + return nil, nil, "", nil, errors.New("copyContainerImage plugin not allowed in pre-ci step, please remove it and try again") + } + } + + registryCredentialMap := make(map[string]bean2.RegistryCredentials) + registryDestinationImageMap := make(map[string][]string) + var allDestinationImages []string //saving all images to be reserved in this array + + for _, step := range postCiSteps { + if version, ok := pluginIdToVersionMap[step.RefPluginId]; ok { + destinationImageMap, credentialMap, err := impl.pluginInputVariableParser.HandleCopyContainerImagePluginInputVariables(step.InputVars, customTag, buildImagePath, buildImagedockerRegistryId) + if err != nil { + impl.Logger.Errorw("error in parsing copyContainerImage input variable", "err", err) + return nil, nil, "", nil, err + } + if version == buildCommonBean.COPY_CONTAINER_IMAGE_VERSION_V1 { + // this is needed in ci runner only for v1 + registryDestinationImageMap = destinationImageMap + } + for _, images := range destinationImageMap { + allDestinationImages = append(allDestinationImages, images...) + } + for k, v := range credentialMap { + registryCredentialMap[k] = v + } + } + } + + pluginArtifactStage := repository5.POST_CI + for _, image := range allDestinationImages { + if image == buildImagePath { + return nil, registryCredentialMap, pluginArtifactStage, nil, + pipelineConfigBean.ErrImagePathInUse + } + } + + var imagePathReservationIds []int + if len(allDestinationImages) > 0 { + savedCIArtifacts, err := impl.ciArtifactRepository.FindCiArtifactByImagePaths(allDestinationImages) + if err != nil { + impl.Logger.Errorw("error in fetching artifacts by image path", "err", err) + return nil, nil, pluginArtifactStage, nil, err + } + if len(savedCIArtifacts) > 0 { + // if already present in ci artifact, return "image path already in use error" + return nil, nil, pluginArtifactStage, nil, pipelineConfigBean.ErrImagePathInUse + } + imagePathReservationIds, err = impl.ReserveImagesGeneratedAtPlugin(customTagId, allDestinationImages) + if err != nil { + return nil, nil, pluginArtifactStage, imagePathReservationIds, err + } + } + return registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imagePathReservationIds, nil +} + +func (impl *ServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, destinationImages []string) ([]int, error) { + var imagePathReservationIds []int + for _, image := range destinationImages { + imagePathReservationData, err := impl.customTagService.ReserveImagePath(image, customTagId) + if err != nil { + impl.Logger.Errorw("Error in marking custom tag reserved", "err", err) + return imagePathReservationIds, err + } + imagePathReservationIds = append(imagePathReservationIds, imagePathReservationData.Id) + } + return imagePathReservationIds, nil +} + +func buildCiStepsDataFromDockerBuildScripts(dockerBuildScripts []*bean.CiScript) []*pipelineConfigBean.StepObject { + // before plugin support, few variables were set as env vars in ci-runner + // these variables are now moved to global vars in plugin steps, but to avoid error in old scripts adding those variables in payload + inputVars := []*commonBean.VariableObject{ + { + Name: "DOCKER_IMAGE_TAG", + Format: "STRING", + VariableType: commonBean.VariableTypeRefGlobal, + ReferenceVariableName: "DOCKER_IMAGE_TAG", + }, + { + Name: "DOCKER_REPOSITORY", + Format: "STRING", + VariableType: commonBean.VariableTypeRefGlobal, + ReferenceVariableName: "DOCKER_REPOSITORY", + }, + { + Name: "DOCKER_REGISTRY_URL", + Format: "STRING", + VariableType: commonBean.VariableTypeRefGlobal, + ReferenceVariableName: "DOCKER_REGISTRY_URL", + }, + { + Name: "DOCKER_IMAGE", + Format: "STRING", + VariableType: commonBean.VariableTypeRefGlobal, + ReferenceVariableName: "DOCKER_IMAGE", + }, + } + var ciSteps []*pipelineConfigBean.StepObject + for _, dockerBuildScript := range dockerBuildScripts { + ciStep := &pipelineConfigBean.StepObject{ + Name: dockerBuildScript.Name, + Index: dockerBuildScript.Index, + Script: dockerBuildScript.Script, + ArtifactPaths: []string{dockerBuildScript.OutputLocation}, + StepType: string(repository.PIPELINE_STEP_TYPE_INLINE), + ExecutorType: string(repository2.SCRIPT_TYPE_SHELL), + InputVars: inputVars, + } + ciSteps = append(ciSteps, ciStep) + } + return ciSteps +} + +func (impl *ServiceImpl) buildImageTag(commitHashes map[int]pipelineConfig.GitCommit, id int, wfId int) string { + dockerImageTag := "" + toAppendDevtronParamInTag := true + for _, v := range commitHashes { + if v.WebhookData.Id == 0 { + if v.Commit == "" { + continue + } + dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _getTruncatedImageTag(v.Commit)) + } else { + _targetCheckout := v.WebhookData.Data[bean.WEBHOOK_SELECTOR_TARGET_CHECKOUT_NAME] + if _targetCheckout == "" { + continue + } + // if not PR based then meaning tag based + isPRBasedEvent := v.WebhookData.EventActionType == bean.WEBHOOK_EVENT_MERGED_ACTION_TYPE + if !isPRBasedEvent && impl.config.CiCdConfig.UseImageTagFromGitProviderForTagBasedBuild { + dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _targetCheckout) + } else { + dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _getTruncatedImageTag(_targetCheckout)) + } + if isPRBasedEvent { + _sourceCheckout := v.WebhookData.Data[bean.WEBHOOK_SELECTOR_SOURCE_CHECKOUT_NAME] + dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _getTruncatedImageTag(_sourceCheckout)) + } else { + toAppendDevtronParamInTag = !impl.config.CiCdConfig.UseImageTagFromGitProviderForTagBasedBuild + } + } + } + toAppendDevtronParamInTag = toAppendDevtronParamInTag && dockerImageTag != "" + if toAppendDevtronParamInTag { + dockerImageTag = fmt.Sprintf("%s-%d-%d", dockerImageTag, id, wfId) + } + // replace / with underscore, as docker image tag doesn't support slash. it gives error + dockerImageTag = strings.ReplaceAll(dockerImageTag, "/", "_") + return dockerImageTag +} + +func getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, commitOrCheckOutData string) string { + if dockerImageTag == "" { + dockerImageTag = commitOrCheckOutData + } else { + if commitOrCheckOutData != "" { + dockerImageTag = fmt.Sprintf("%s-%s", dockerImageTag, commitOrCheckOutData) + } + } + return dockerImageTag +} + +func (impl *ServiceImpl) updateCiWorkflow(request *types.WorkflowRequest, savedWf *pipelineConfig.CiWorkflow) error { + ciBuildConfig := request.CiBuildConfig + ciBuildType := string(ciBuildConfig.CiBuildType) + savedWf.CiBuildType = ciBuildType + return impl.ciService.UpdateCiWorkflowWithStage(savedWf) +} + +func (impl *ServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, workflowRequest *types.WorkflowRequest) error { + // externalCi artifact is meant only for CI_JOB + if trigger.PipelineType != string(buildCommonBean.CI_JOB) { + return nil + } + + // checking if user has given run time parameters for externalCiArtifact, if given then sending git material to Ci-Runner + externalCiArtifact, exists := trigger.RuntimeParameters.GetGlobalRuntimeVariables()[buildBean.ExtraEnvVarExternalCiArtifactKey] + // validate externalCiArtifact as docker image + if exists { + externalCiArtifact = strings.TrimSpace(externalCiArtifact) + if !strings.Contains(externalCiArtifact, ":") { + if utils.IsValidDockerTagName(externalCiArtifact) { + fullImageUrl, err := utils.BuildDockerImagePath(bean4.DockerRegistryInfo{ + DockerImageTag: externalCiArtifact, + DockerRegistryId: workflowRequest.DockerRegistryId, + DockerRegistryType: workflowRequest.DockerRegistryType, + DockerRegistryURL: workflowRequest.DockerRegistryURL, + DockerRepository: workflowRequest.DockerRepository, + }) + if err != nil { + impl.Logger.Errorw("Error in building docker image", "err", err) + return err + } + externalCiArtifact = fullImageUrl + } else { + impl.Logger.Errorw("validation error", "externalCiArtifact", externalCiArtifact) + return fmt.Errorf("invalid image name or url given in externalCiArtifact") + } + + } + // This will overwrite the existing runtime parameters value for constants.externalCiArtifact + trigger.RuntimeParameters = trigger.RuntimeParameters.AddRuntimeGlobalVariable(buildBean.ExtraEnvVarExternalCiArtifactKey, externalCiArtifact) + var artifactExists bool + var err error + + imageDigest, ok := trigger.RuntimeParameters.GetGlobalRuntimeVariables()[buildBean.ExtraEnvVarImageDigestKey] + if !ok || len(imageDigest) == 0 { + artifactExists, err = impl.ciArtifactRepository.IfArtifactExistByImage(externalCiArtifact, trigger.PipelineId) + if err != nil { + impl.Logger.Errorw("error in fetching ci artifact", "err", err) + return err + } + if artifactExists { + impl.Logger.Errorw("ci artifact already exists with same image name", "artifact", externalCiArtifact) + return fmt.Errorf("ci artifact already exists with same image name") + } + } else { + artifactExists, err = impl.ciArtifactRepository.IfArtifactExistByImageDigest(imageDigest, externalCiArtifact, trigger.PipelineId) + if err != nil { + impl.Logger.Errorw("error in fetching ci artifact", "err", err, "imageDigest", imageDigest) + return err + } + if artifactExists { + impl.Logger.Errorw("ci artifact already exists with same digest", "artifact", externalCiArtifact) + return fmt.Errorf("ci artifact already exists with same digest") + } + } + } + if trigger.PipelineType == string(buildCommonBean.CI_JOB) && len(ciMaterials) != 0 && !exists && externalCiArtifact == "" { + ciMaterials[0].GitMaterial = nil + ciMaterials[0].GitMaterialId = 0 + } + return nil +} + +func _getTruncatedImageTag(imageTag string) string { + _length := len(imageTag) + if _length == 0 { + return imageTag + } + + _truncatedLength := 8 + + if _length < _truncatedLength { + return imageTag + } else { + return imageTag[:_truncatedLength] + } +} + +func (impl *ServiceImpl) markCurrentCiWorkflowFailed(savedCiWf *pipelineConfig.CiWorkflow, validationErr error) error { + // currently such requirement is not there + if savedCiWf == nil { + return nil + } + if savedCiWf.Id != 0 && slices.Contains(cdWorkflow.WfrTerminalStatusList, savedCiWf.Status) { + impl.Logger.Debug("workflow is already in terminal state", "status", savedCiWf.Status, "workflowId", savedCiWf.Id, "message", savedCiWf.Message) + return nil + } + + savedCiWf.Status = cdWorkflow.WorkflowFailed + savedCiWf.Message = validationErr.Error() + savedCiWf.FinishedOn = time.Now() + + var dbErr error + if savedCiWf.Id == 0 { + dbErr = impl.ciService.SaveCiWorkflowWithStage(savedCiWf) + } else { + dbErr = impl.ciService.UpdateCiWorkflowWithStage(savedCiWf) + } + + if dbErr != nil { + impl.Logger.Errorw("save/update workflow error", "err", dbErr) + return dbErr + } + return nil +} diff --git a/pkg/pipeline/CiService_ent.go b/pkg/build/trigger/Service_ent.go similarity index 51% rename from pkg/pipeline/CiService_ent.go rename to pkg/build/trigger/Service_ent.go index 342429d052..b20094eca7 100644 --- a/pkg/pipeline/CiService_ent.go +++ b/pkg/build/trigger/Service_ent.go @@ -1,4 +1,4 @@ -package pipeline +package trigger import ( "github.com/devtron-labs/common-lib/imageScan/bean" @@ -7,10 +7,10 @@ import ( "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" ) -func (impl *CiServiceImpl) fetchImageScanExecutionMedium() (*repository.ScanToolMetadata, bean.ScanExecutionMedium, error) { +func (impl *ServiceImpl) fetchImageScanExecutionMedium() (*repository.ScanToolMetadata, bean.ScanExecutionMedium, error) { return &repository.ScanToolMetadata{}, "", nil } -func (impl *CiServiceImpl) fetchImageScanExecutionStepsForWfRequest(scanToolMetadata *repository.ScanToolMetadata) ([]*types.ImageScanningSteps, []*bean2.RefPluginObject, error) { +func (impl *ServiceImpl) fetchImageScanExecutionStepsForWfRequest(scanToolMetadata *repository.ScanToolMetadata) ([]*types.ImageScanningSteps, []*bean2.RefPluginObject, error) { return nil, nil, nil } diff --git a/pkg/build/wire_build.go b/pkg/build/wire_build.go index 6610dd24cc..32c2fc0b0b 100644 --- a/pkg/build/wire_build.go +++ b/pkg/build/wire_build.go @@ -20,6 +20,7 @@ import ( "github.com/devtron-labs/devtron/pkg/build/artifacts" "github.com/devtron-labs/devtron/pkg/build/git" "github.com/devtron-labs/devtron/pkg/build/pipeline" + "github.com/devtron-labs/devtron/pkg/build/trigger" "github.com/google/wire" ) @@ -27,4 +28,5 @@ var WireSet = wire.NewSet( artifacts.WireSet, pipeline.WireSet, git.GitWireSet, + trigger.WireSet, ) diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go index ec80a7cf19..ef11bedf2c 100644 --- a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go @@ -41,7 +41,6 @@ import ( adapter2 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/adapter" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" - "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/adapter" pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" @@ -327,7 +326,7 @@ func (impl *TriggerServiceImpl) checkVulnerabilityStatusAndFailWfIfNeeded(ctx co // setCopyContainerImagePluginDataAndReserveImages sets required fields in cdStageWorkflowRequest and reserve images generated by plugin func (impl *TriggerServiceImpl) setCopyContainerImagePluginDataAndReserveImages(cdStageWorkflowRequest *types.WorkflowRequest, pipelineId int, pipelineStage string, artifact *repository.CiArtifact) ([]int, error) { - copyContainerImagePluginDetail, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(pipeline.COPY_CONTAINER_IMAGE) + copyContainerImagePluginDetail, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(buildCommonBean.COPY_CONTAINER_IMAGE) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in getting copyContainerImage plugin id", "err", err) return nil, err @@ -363,7 +362,7 @@ func (impl *TriggerServiceImpl) setCopyContainerImagePluginDataAndReserveImages( impl.logger.Errorw("error in parsing copyContainerImage input variable", "err", err) return nil, err } - if version == pipeline.COPY_CONTAINER_IMAGE_VERSION_V1 { + if version == buildCommonBean.COPY_CONTAINER_IMAGE_VERSION_V1 { // this is needed in ci runner only for v1 cdStageWorkflowRequest.RegistryDestinationImageMap = registryDestinationImageMap } diff --git a/pkg/pipeline/BuildPipelineConfigService.go b/pkg/pipeline/BuildPipelineConfigService.go index 95cd8775ac..f49e04ba53 100644 --- a/pkg/pipeline/BuildPipelineConfigService.go +++ b/pkg/pipeline/BuildPipelineConfigService.go @@ -591,9 +591,9 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipeline(appId int) (ciConfig *bea Script: ciScript.Script, OutputLocation: ciScript.OutputLocation, } - if ciScript.Stage == BEFORE_DOCKER_BUILD { + if ciScript.Stage == common.BEFORE_DOCKER_BUILD { beforeDockerBuildScripts = append(beforeDockerBuildScripts, ciScriptResp) - } else if ciScript.Stage == AFTER_DOCKER_BUILD { + } else if ciScript.Stage == common.AFTER_DOCKER_BUILD { afterDockerBuildScripts = append(afterDockerBuildScripts, ciScriptResp) } } @@ -746,9 +746,9 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipelineById(pipelineId int) (ciPi Script: ciScript.Script, OutputLocation: ciScript.OutputLocation, } - if ciScript.Stage == BEFORE_DOCKER_BUILD { + if ciScript.Stage == common.BEFORE_DOCKER_BUILD { beforeDockerBuildScripts = append(beforeDockerBuildScripts, ciScriptResp) - } else if ciScript.Stage == AFTER_DOCKER_BUILD { + } else if ciScript.Stage == common.AFTER_DOCKER_BUILD { afterDockerBuildScripts = append(afterDockerBuildScripts, ciScriptResp) } } diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index 46a08da9d5..0f2e5c7f2e 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -226,9 +226,6 @@ func NewCiCdPipelineOrchestrator( } } -const BEFORE_DOCKER_BUILD string = "BEFORE_DOCKER_BUILD" -const AFTER_DOCKER_BUILD string = "AFTER_DOCKER_BUILD" - func (impl CiCdPipelineOrchestratorImpl) PatchCiMaterialSource(patchRequest *bean.CiMaterialPatchRequest, userId int32) (*bean.CiMaterialPatchRequest, error) { pipeline, err := impl.findUniquePipelineForAppIdAndEnvironmentId(patchRequest.AppId, patchRequest.EnvironmentId) if err != nil { diff --git a/pkg/pipeline/CiLogService.go b/pkg/pipeline/CiLogService.go index 6ad0752f5c..d554917831 100644 --- a/pkg/pipeline/CiLogService.go +++ b/pkg/pipeline/CiLogService.go @@ -36,12 +36,11 @@ type CiLogService interface { type CiLogServiceImpl struct { logger *zap.SugaredLogger - ciService CiService kubeClient *kubernetes.Clientset k8sUtil *k8s.K8sServiceImpl } -func NewCiLogServiceImpl(logger *zap.SugaredLogger, ciService CiService, k8sUtil *k8s.K8sServiceImpl) (*CiLogServiceImpl, error) { +func NewCiLogServiceImpl(logger *zap.SugaredLogger, k8sUtil *k8s.K8sServiceImpl) (*CiLogServiceImpl, error) { _, _, clientSet, err := k8sUtil.GetK8sInClusterConfigAndClients() if err != nil { logger.Errorw("error in getting k8s in cluster client set", "err", err) @@ -49,7 +48,6 @@ func NewCiLogServiceImpl(logger *zap.SugaredLogger, ciService CiService, k8sUtil } return &CiLogServiceImpl{ logger: logger, - ciService: ciService, kubeClient: clientSet, k8sUtil: k8sUtil, }, nil diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index a2bdd0a58e..c88ae5ca11 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -17,153 +17,50 @@ package pipeline import ( - "encoding/json" - "errors" - "fmt" "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" - "github.com/caarlos0/env" - "github.com/devtron-labs/common-lib/utils" - bean3 "github.com/devtron-labs/common-lib/utils/bean" - commonBean "github.com/devtron-labs/common-lib/workflow" bean5 "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/internal/sql/constants" + client "github.com/devtron-labs/devtron/client/events" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" - "github.com/devtron-labs/devtron/pkg/attributes" - bean4 "github.com/devtron-labs/devtron/pkg/attributes/bean" - "github.com/devtron-labs/devtron/pkg/bean/common" - "github.com/devtron-labs/devtron/pkg/build/pipeline" buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" - common2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" - repository6 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" - "github.com/devtron-labs/devtron/pkg/pipeline/adapter" - "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" - bean2 "github.com/devtron-labs/devtron/pkg/plugin/bean" "github.com/devtron-labs/devtron/pkg/sql" util3 "github.com/devtron-labs/devtron/util" - "github.com/devtron-labs/devtron/util/sliceUtil" - "path" - "path/filepath" - "slices" - "strconv" - "strings" - "time" - - repository5 "github.com/devtron-labs/devtron/internal/sql/repository" - appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" - repository3 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" - "github.com/devtron-labs/devtron/internal/sql/repository/helper" - "github.com/devtron-labs/devtron/pkg/app" - "github.com/devtron-labs/devtron/pkg/auth/user" - pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" - "github.com/devtron-labs/devtron/pkg/pipeline/repository" - "github.com/devtron-labs/devtron/pkg/pipeline/types" - "github.com/devtron-labs/devtron/pkg/plugin" - repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository" - "github.com/devtron-labs/devtron/pkg/resourceQualifiers" - "github.com/devtron-labs/devtron/pkg/variables" - repository4 "github.com/devtron-labs/devtron/pkg/variables/repository" - "github.com/go-pg/pg" - "net/http" - - "github.com/devtron-labs/common-lib/blob-storage" - client "github.com/devtron-labs/devtron/client/events" - "github.com/devtron-labs/devtron/internal/middleware" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/internal/util" - "github.com/devtron-labs/devtron/pkg/bean" util2 "github.com/devtron-labs/devtron/util/event" "go.uber.org/zap" ) type CiService interface { - TriggerCiPipeline(trigger types.Trigger) (int, error) - GetCiMaterials(pipelineId int, ciMaterials []*pipelineConfig.CiPipelineMaterial) ([]*pipelineConfig.CiPipelineMaterial, error) + WriteCITriggerEvent(trigger types.Trigger, pipeline *pipelineConfig.CiPipeline, workflowRequest *types.WorkflowRequest) WriteCIFailEvent(ciWorkflow *pipelineConfig.CiWorkflow) SaveCiWorkflowWithStage(wf *pipelineConfig.CiWorkflow) error UpdateCiWorkflowWithStage(wf *pipelineConfig.CiWorkflow) error } -type BuildxCacheFlags struct { - BuildxCacheModeMin bool `env:"BUILDX_CACHE_MODE_MIN" envDefault:"false"` - AsyncBuildxCacheExport bool `env:"ASYNC_BUILDX_CACHE_EXPORT" envDefault:"false"` -} type CiServiceImpl struct { - Logger *zap.SugaredLogger - workflowService WorkflowService - ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository - workflowStageStatusService workflowStatus.WorkFlowStageStatusService - eventClient client.EventClient - eventFactory client.EventFactory - ciPipelineRepository pipelineConfig.CiPipelineRepository - ciArtifactRepository repository5.CiArtifactRepository - pipelineStageService PipelineStageService - userService user.UserService - ciTemplateService pipeline.CiTemplateReadService - appCrudOperationService app.AppCrudOperationService - envRepository repository6.EnvironmentRepository - appRepository appRepository.AppRepository - customTagService CustomTagService - config *types.CiConfig - scopedVariableManager variables.ScopedVariableManager - pluginInputVariableParser PluginInputVariableParser - globalPluginService plugin.GlobalPluginService - infraProvider infraProviders.InfraProvider - ciCdPipelineOrchestrator CiCdPipelineOrchestrator - buildxCacheFlags *BuildxCacheFlags - attributeService attributes.AttributesService - ciWorkflowRepository pipelineConfig.CiWorkflowRepository - transactionManager sql.TransactionWrapper + Logger *zap.SugaredLogger + workflowStageStatusService workflowStatus.WorkFlowStageStatusService + eventClient client.EventClient + eventFactory client.EventFactory + config *types.CiConfig + ciWorkflowRepository pipelineConfig.CiWorkflowRepository + transactionManager sql.TransactionWrapper } -func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService, - ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, +func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowStageStatusService workflowStatus.WorkFlowStageStatusService, eventClient client.EventClient, eventFactory client.EventFactory, - ciPipelineRepository pipelineConfig.CiPipelineRepository, - ciArtifactRepository repository5.CiArtifactRepository, - pipelineStageService PipelineStageService, - userService user.UserService, - ciTemplateService pipeline.CiTemplateReadService, appCrudOperationService app.AppCrudOperationService, envRepository repository6.EnvironmentRepository, appRepository appRepository.AppRepository, - scopedVariableManager variables.ScopedVariableManager, - customTagService CustomTagService, - pluginInputVariableParser PluginInputVariableParser, - globalPluginService plugin.GlobalPluginService, - infraProvider infraProviders.InfraProvider, - ciCdPipelineOrchestrator CiCdPipelineOrchestrator, attributeService attributes.AttributesService, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, transactionManager sql.TransactionWrapper, ) *CiServiceImpl { - buildxCacheFlags := &BuildxCacheFlags{} - err := env.Parse(buildxCacheFlags) - if err != nil { - Logger.Infow("error occurred while parsing BuildxCacheFlags env,so setting BuildxCacheModeMin and AsyncBuildxCacheExport to default value", "err", err) - } cis := &CiServiceImpl{ - Logger: Logger, - workflowService: workflowService, - ciPipelineMaterialRepository: ciPipelineMaterialRepository, - workflowStageStatusService: workflowStageStatusService, - eventClient: eventClient, - eventFactory: eventFactory, - ciPipelineRepository: ciPipelineRepository, - ciArtifactRepository: ciArtifactRepository, - pipelineStageService: pipelineStageService, - userService: userService, - ciTemplateService: ciTemplateService, - appCrudOperationService: appCrudOperationService, - envRepository: envRepository, - appRepository: appRepository, - scopedVariableManager: scopedVariableManager, - customTagService: customTagService, - pluginInputVariableParser: pluginInputVariableParser, - globalPluginService: globalPluginService, - infraProvider: infraProvider, - ciCdPipelineOrchestrator: ciCdPipelineOrchestrator, - buildxCacheFlags: buildxCacheFlags, - attributeService: attributeService, - ciWorkflowRepository: ciWorkflowRepository, - transactionManager: transactionManager, + Logger: Logger, + workflowStageStatusService: workflowStageStatusService, + eventClient: eventClient, + eventFactory: eventFactory, + ciWorkflowRepository: ciWorkflowRepository, + transactionManager: transactionManager, } config, err := types.GetCiConfig() if err != nil { @@ -173,322 +70,6 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService return cis } -func (impl *CiServiceImpl) GetCiMaterials(pipelineId int, ciMaterials []*pipelineConfig.CiPipelineMaterial) ([]*pipelineConfig.CiPipelineMaterial, error) { - if !(len(ciMaterials) == 0) { - return ciMaterials, nil - } else { - ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(pipelineId) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return nil, err - } - impl.Logger.Debug("ciMaterials for pipeline trigger ", ciMaterials) - return ciMaterials, nil - } -} - -func (impl *CiServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, workflowRequest *types.WorkflowRequest) error { - // externalCi artifact is meant only for CI_JOB - if trigger.PipelineType != string(common2.CI_JOB) { - return nil - } - - // checking if user has given run time parameters for externalCiArtifact, if given then sending git material to Ci-Runner - externalCiArtifact, exists := trigger.RuntimeParameters.GetGlobalRuntimeVariables()[buildBean.ExtraEnvVarExternalCiArtifactKey] - // validate externalCiArtifact as docker image - if exists { - externalCiArtifact = strings.TrimSpace(externalCiArtifact) - if !strings.Contains(externalCiArtifact, ":") { - if utils.IsValidDockerTagName(externalCiArtifact) { - fullImageUrl, err := utils.BuildDockerImagePath(bean3.DockerRegistryInfo{ - DockerImageTag: externalCiArtifact, - DockerRegistryId: workflowRequest.DockerRegistryId, - DockerRegistryType: workflowRequest.DockerRegistryType, - DockerRegistryURL: workflowRequest.DockerRegistryURL, - DockerRepository: workflowRequest.DockerRepository, - }) - if err != nil { - impl.Logger.Errorw("Error in building docker image", "err", err) - return err - } - externalCiArtifact = fullImageUrl - } else { - impl.Logger.Errorw("validation error", "externalCiArtifact", externalCiArtifact) - return fmt.Errorf("invalid image name or url given in externalCiArtifact") - } - - } - // This will overwrite the existing runtime parameters value for constants.externalCiArtifact - trigger.RuntimeParameters = trigger.RuntimeParameters.AddRuntimeGlobalVariable(buildBean.ExtraEnvVarExternalCiArtifactKey, externalCiArtifact) - var artifactExists bool - var err error - - imageDigest, ok := trigger.RuntimeParameters.GetGlobalRuntimeVariables()[buildBean.ExtraEnvVarImageDigestKey] - if !ok || len(imageDigest) == 0 { - artifactExists, err = impl.ciArtifactRepository.IfArtifactExistByImage(externalCiArtifact, trigger.PipelineId) - if err != nil { - impl.Logger.Errorw("error in fetching ci artifact", "err", err) - return err - } - if artifactExists { - impl.Logger.Errorw("ci artifact already exists with same image name", "artifact", externalCiArtifact) - return fmt.Errorf("ci artifact already exists with same image name") - } - } else { - artifactExists, err = impl.ciArtifactRepository.IfArtifactExistByImageDigest(imageDigest, externalCiArtifact, trigger.PipelineId) - if err != nil { - impl.Logger.Errorw("error in fetching ci artifact", "err", err, "imageDigest", imageDigest) - return err - } - if artifactExists { - impl.Logger.Errorw("ci artifact already exists with same digest", "artifact", externalCiArtifact) - return fmt.Errorf("ci artifact already exists with same digest") - } - - } - - } - if trigger.PipelineType == string(common2.CI_JOB) && len(ciMaterials) != 0 && !exists && externalCiArtifact == "" { - ciMaterials[0].GitMaterial = nil - ciMaterials[0].GitMaterialId = 0 - } - return nil -} - -func (impl *CiServiceImpl) markCurrentCiWorkflowFailed(savedCiWf *pipelineConfig.CiWorkflow, validationErr error) error { - // currently such requirement is not there - if savedCiWf == nil { - return nil - } - if savedCiWf.Id != 0 && slices.Contains(cdWorkflow.WfrTerminalStatusList, savedCiWf.Status) { - impl.Logger.Debug("workflow is already in terminal state", "status", savedCiWf.Status, "workflowId", savedCiWf.Id, "message", savedCiWf.Message) - return nil - } - - savedCiWf.Status = cdWorkflow.WorkflowFailed - savedCiWf.Message = validationErr.Error() - savedCiWf.FinishedOn = time.Now() - - var dbErr error - if savedCiWf.Id == 0 { - dbErr = impl.SaveCiWorkflowWithStage(savedCiWf) - } else { - dbErr = impl.UpdateCiWorkflowWithStage(savedCiWf) - } - - if dbErr != nil { - impl.Logger.Errorw("save/update workflow error", "err", dbErr) - return dbErr - } - return nil -} - -func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { - impl.Logger.Debug("ci pipeline manual trigger") - ciMaterials, err := impl.GetCiMaterials(trigger.PipelineId, trigger.CiMaterials) - if err != nil { - return 0, err - } - - ciPipelineScripts, err := impl.ciPipelineRepository.FindCiScriptsByCiPipelineId(trigger.PipelineId) - if err != nil && !util.IsErrNoRows(err) { - return 0, err - } - - var pipeline *pipelineConfig.CiPipeline - for _, m := range ciMaterials { - pipeline = m.CiPipeline - break - } - - scope := resourceQualifiers.Scope{ - AppId: pipeline.App.Id, - } - ciWorkflowConfigNamespace := impl.config.GetDefaultNamespace() - envModal, isJob, err := impl.getEnvironmentForJob(pipeline, trigger) - if err != nil { - return 0, err - } - if isJob && envModal != nil { - ciWorkflowConfigNamespace = envModal.Namespace - - // This will be populated for jobs running in selected environment - scope.EnvId = envModal.Id - scope.ClusterId = envModal.ClusterId - - scope.SystemMetadata = &resourceQualifiers.SystemMetadata{ - EnvironmentName: envModal.Name, - ClusterName: envModal.Cluster.ClusterName, - Namespace: envModal.Namespace, - } - } - if scope.SystemMetadata == nil { - scope.SystemMetadata = &resourceQualifiers.SystemMetadata{ - Namespace: ciWorkflowConfigNamespace, - AppName: pipeline.App.AppName, - } - } - - savedCiWf, err := impl.saveNewWorkflow(pipeline, ciWorkflowConfigNamespace, trigger.CommitHashes, trigger.TriggeredBy, trigger.EnvironmentId, isJob, trigger.ReferenceCiWorkflowId) - if err != nil { - impl.Logger.Errorw("could not save new workflow", "err", err) - return 0, err - } - - // preCiSteps, postCiSteps, refPluginsData, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(pipeline.Id, ciEvent) - request := pipelineConfigBean.NewBuildPrePostStepDataReq(pipeline.Id, pipelineConfigBean.CiStage, scope) - prePostAndRefPluginResponse, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(request) - if err != nil { - impl.Logger.Errorw("error in getting pre steps data for wf request", "err", err, "ciPipelineId", pipeline.Id) - dbErr := impl.markCurrentCiWorkflowFailed(savedCiWf, err) - if dbErr != nil { - impl.Logger.Errorw("saving workflow error", "err", dbErr) - } - return 0, err - } - preCiSteps := prePostAndRefPluginResponse.PreStageSteps - postCiSteps := prePostAndRefPluginResponse.PostStageSteps - refPluginsData := prePostAndRefPluginResponse.RefPluginData - variableSnapshot := prePostAndRefPluginResponse.VariableSnapshot - - if len(preCiSteps) == 0 && isJob { - errMsg := fmt.Sprintf("No tasks are configured in this job pipeline") - validationErr := util.NewApiError(http.StatusNotFound, errMsg, errMsg) - - return 0, validationErr - } - - // get env variables of git trigger data and add it in the extraEnvVariables - gitTriggerEnvVariables, _, err := impl.ciCdPipelineOrchestrator.GetGitCommitEnvVarDataForCICDStage(savedCiWf.GitTriggers) - if err != nil { - impl.Logger.Errorw("error in getting gitTrigger env data for stage", "gitTriggers", savedCiWf.GitTriggers, "err", err) - return 0, err - } - - for k, v := range gitTriggerEnvVariables { - trigger.RuntimeParameters = trigger.RuntimeParameters.AddSystemVariable(k, v) - } - - workflowRequest, err := impl.buildWfRequestForCiPipeline(pipeline, trigger, ciMaterials, savedCiWf, ciWorkflowConfigNamespace, ciPipelineScripts, preCiSteps, postCiSteps, refPluginsData, isJob) - if err != nil { - impl.Logger.Errorw("make workflow req", "err", err) - return 0, err - } - err = impl.handleRuntimeParamsValidations(trigger, ciMaterials, workflowRequest) - if err != nil { - savedCiWf.Status = cdWorkflow.WorkflowAborted - savedCiWf.Message = err.Error() - err1 := impl.UpdateCiWorkflowWithStage(savedCiWf) - if err1 != nil { - impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag") - } - return 0, err - } - - workflowRequest.Scope = scope - workflowRequest.AppId = pipeline.AppId - workflowRequest.BuildxCacheModeMin = impl.buildxCacheFlags.BuildxCacheModeMin - workflowRequest.AsyncBuildxCacheExport = impl.buildxCacheFlags.AsyncBuildxCacheExport - if impl.config != nil && impl.config.BuildxK8sDriverOptions != "" { - err = impl.setBuildxK8sDriverData(workflowRequest) - if err != nil { - impl.Logger.Errorw("error in setBuildxK8sDriverData", "BUILDX_K8S_DRIVER_OPTIONS", impl.config.BuildxK8sDriverOptions, "err", err) - return 0, err - } - } - - // savedCiWf.LogLocation = impl.ciCdConfig.CiDefaultBuildLogsKeyPrefix + "/" + workflowRequest.WorkflowNamePrefix + "/main.log" - savedCiWf.LogLocation = fmt.Sprintf("%s/%s/main.log", impl.config.GetDefaultBuildLogsKeyPrefix(), workflowRequest.WorkflowNamePrefix) - err = impl.updateCiWorkflow(workflowRequest, savedCiWf) - - appLabels, err := impl.appCrudOperationService.GetLabelsByAppId(pipeline.AppId) - if err != nil { - return 0, err - } - workflowRequest.AppLabels = appLabels - workflowRequest.Env = envModal - if isJob { - workflowRequest.Type = pipelineConfigBean.JOB_WORKFLOW_PIPELINE_TYPE - } else { - workflowRequest.Type = pipelineConfigBean.CI_WORKFLOW_PIPELINE_TYPE - } - - workflowRequest.CiPipelineType = trigger.PipelineType - err = impl.executeCiPipeline(workflowRequest) - if err != nil { - impl.Logger.Errorw("error in executing ci pipeline", "err", err) - dbErr := impl.markCurrentCiWorkflowFailed(savedCiWf, err) - if dbErr != nil { - impl.Logger.Errorw("update ci workflow error", "err", dbErr) - } - return 0, err - } - impl.Logger.Debugw("ci triggered", " pipeline ", trigger.PipelineId) - - var variableSnapshotHistories = sliceUtil.GetBeansPtr( - repository4.GetSnapshotBean(savedCiWf.Id, repository4.HistoryReferenceTypeCIWORKFLOW, variableSnapshot)) - if len(variableSnapshotHistories) > 0 { - err = impl.scopedVariableManager.SaveVariableHistoriesForTrigger(variableSnapshotHistories, trigger.TriggeredBy) - if err != nil { - impl.Logger.Errorf("Not able to save variable snapshot for CI trigger %s", err) - } - } - - middleware.CiTriggerCounter.WithLabelValues(pipeline.App.AppName, pipeline.Name).Inc() - go impl.WriteCITriggerEvent(trigger, pipeline, workflowRequest) - return savedCiWf.Id, err -} - -func (impl *CiServiceImpl) setBuildxK8sDriverData(workflowRequest *types.WorkflowRequest) error { - ciBuildConfig := workflowRequest.CiBuildConfig - if ciBuildConfig != nil { - if dockerBuildConfig := ciBuildConfig.DockerBuildConfig; dockerBuildConfig != nil { - k8sDriverOptions, err := impl.getK8sDriverOptions() - if err != nil { - errMsg := "error in parsing BUILDX_K8S_DRIVER_OPTIONS from the devtron-cm, " - err = errors.New(errMsg + "error : " + err.Error()) - impl.Logger.Errorw(errMsg, "err", err) - } - dockerBuildConfig.BuildxK8sDriverOptions = k8sDriverOptions - - } - } - return nil -} - -func (impl *CiServiceImpl) getK8sDriverOptions() ([]map[string]string, error) { - buildxK8sDriverOptions := make([]map[string]string, 0) - err := json.Unmarshal([]byte(impl.config.BuildxK8sDriverOptions), &buildxK8sDriverOptions) - if err != nil { - return nil, err - } else { - return buildxK8sDriverOptions, nil - } -} - -func (impl *CiServiceImpl) getEnvironmentForJob(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger) (*repository6.Environment, bool, error) { - app, err := impl.appRepository.FindById(pipeline.AppId) - if err != nil { - impl.Logger.Errorw("could not find app", "err", err) - return nil, false, err - } - - var env *repository6.Environment - isJob := false - if app.AppType == helper.Job { - isJob = true - if trigger.EnvironmentId != 0 { - env, err = impl.envRepository.FindById(trigger.EnvironmentId) - if err != nil { - impl.Logger.Errorw("could not find environment", "err", err) - return nil, isJob, err - } - return env, isJob, nil - } - } - return nil, isJob, nil -} - func (impl *CiServiceImpl) WriteCITriggerEvent(trigger types.Trigger, pipeline *pipelineConfig.CiPipeline, workflowRequest *types.WorkflowRequest) { event, _ := impl.eventFactory.Build(util2.Trigger, &pipeline.Id, pipeline.AppId, nil, util2.CI) material := &buildBean.MaterialTriggerInfo{} @@ -504,638 +85,6 @@ func (impl *CiServiceImpl) WriteCITriggerEvent(trigger types.Trigger, pipeline * } } -// TODO: Send all trigger data -func (impl *CiServiceImpl) BuildPayload(trigger types.Trigger, pipeline *pipelineConfig.CiPipeline) *client.Payload { - payload := &client.Payload{} - payload.AppName = pipeline.App.AppName - payload.PipelineName = pipeline.Name - return payload -} - -func (impl *CiServiceImpl) saveNewWorkflow(pipeline *pipelineConfig.CiPipeline, ciWorkflowConfigNamespace string, - commitHashes map[int]pipelineConfig.GitCommit, userId int32, EnvironmentId int, isJob bool, refCiWorkflowId int) (wf *pipelineConfig.CiWorkflow, error error) { - - ciWorkflow := &pipelineConfig.CiWorkflow{ - Name: pipeline.Name + "-" + strconv.Itoa(pipeline.Id), - Status: cdWorkflow.WorkflowStarting, // starting CIStage - Message: "", - StartedOn: time.Now(), - CiPipelineId: pipeline.Id, - Namespace: impl.config.GetDefaultNamespace(), - BlobStorageEnabled: impl.config.BlobStorageEnabled, - GitTriggers: commitHashes, - LogLocation: "", - TriggeredBy: userId, - ReferenceCiWorkflowId: refCiWorkflowId, - ExecutorType: impl.config.GetWorkflowExecutorType(), - } - if isJob { - ciWorkflow.Namespace = ciWorkflowConfigNamespace - ciWorkflow.EnvironmentId = EnvironmentId - } - err := impl.SaveCiWorkflowWithStage(ciWorkflow) - if err != nil { - impl.Logger.Errorw("saving workflow error", "err", err) - return &pipelineConfig.CiWorkflow{}, err - } - impl.Logger.Debugw("workflow saved ", "id", ciWorkflow.Id) - return ciWorkflow, nil -} - -func (impl *CiServiceImpl) executeCiPipeline(workflowRequest *types.WorkflowRequest) error { - _, _, err := impl.workflowService.SubmitWorkflow(workflowRequest) - if err != nil { - impl.Logger.Errorw("workflow error", "err", err) - return err - } - return nil -} - -func (impl *CiServiceImpl) buildS3ArtifactLocation(ciWorkflowConfigLogsBucket string, savedWf *pipelineConfig.CiWorkflow) (string, string, string) { - ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() - ArtifactLocation := fmt.Sprintf("s3://"+path.Join(ciWorkflowConfigLogsBucket, ciArtifactLocationFormat), savedWf.Id, savedWf.Id) - artifactFileName := fmt.Sprintf(ciArtifactLocationFormat, savedWf.Id, savedWf.Id) - return ArtifactLocation, ciWorkflowConfigLogsBucket, artifactFileName -} - -func (impl *CiServiceImpl) buildDefaultArtifactLocation(savedWf *pipelineConfig.CiWorkflow) string { - ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() - ArtifactLocation := fmt.Sprintf(ciArtifactLocationFormat, savedWf.Id, savedWf.Id) - return ArtifactLocation -} - -func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, savedWf *pipelineConfig.CiWorkflow, ciWorkflowConfigNamespace string, ciPipelineScripts []*pipelineConfig.CiPipelineScript, preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, refPluginsData []*pipelineConfigBean.RefPluginObject, isJob bool) (*types.WorkflowRequest, error) { - var ciProjectDetails []pipelineConfigBean.CiProjectDetails - commitHashes := trigger.CommitHashes - for _, ciMaterial := range ciMaterials { - // ignore those materials which have inactive git material - if ciMaterial == nil || ciMaterial.GitMaterial == nil || !ciMaterial.GitMaterial.Active { - continue - } - commitHashForPipelineId := commitHashes[ciMaterial.Id] - ciProjectDetail := pipelineConfigBean.CiProjectDetails{ - GitRepository: ciMaterial.GitMaterial.Url, - MaterialName: ciMaterial.GitMaterial.Name, - CheckoutPath: ciMaterial.GitMaterial.CheckoutPath, - FetchSubmodules: ciMaterial.GitMaterial.FetchSubmodules, - CommitHash: commitHashForPipelineId.Commit, - Author: commitHashForPipelineId.Author, - SourceType: ciMaterial.Type, - SourceValue: ciMaterial.Value, - GitTag: ciMaterial.GitTag, - Message: commitHashForPipelineId.Message, - Type: string(ciMaterial.Type), - CommitTime: commitHashForPipelineId.Date.Format(bean.LayoutRFC3339), - GitOptions: pipelineConfigBean.GitOptions{ - UserName: ciMaterial.GitMaterial.GitProvider.UserName, - Password: ciMaterial.GitMaterial.GitProvider.Password, - SshPrivateKey: ciMaterial.GitMaterial.GitProvider.SshPrivateKey, - AccessToken: ciMaterial.GitMaterial.GitProvider.AccessToken, - AuthMode: ciMaterial.GitMaterial.GitProvider.AuthMode, - EnableTLSVerification: ciMaterial.GitMaterial.GitProvider.EnableTLSVerification, - TlsKey: ciMaterial.GitMaterial.GitProvider.TlsKey, - TlsCert: ciMaterial.GitMaterial.GitProvider.TlsCert, - CaCert: ciMaterial.GitMaterial.GitProvider.CaCert, - }, - } - - if ciMaterial.Type == constants.SOURCE_TYPE_WEBHOOK { - webhookData := commitHashForPipelineId.WebhookData - ciProjectDetail.WebhookData = pipelineConfig.WebhookData{ - Id: webhookData.Id, - EventActionType: webhookData.EventActionType, - Data: webhookData.Data, - } - } - - ciProjectDetails = append(ciProjectDetails, ciProjectDetail) - } - - var beforeDockerBuildScripts []*bean.CiScript - var afterDockerBuildScripts []*bean.CiScript - for _, ciPipelineScript := range ciPipelineScripts { - ciTask := &bean.CiScript{ - Id: ciPipelineScript.Id, - Index: ciPipelineScript.Index, - Name: ciPipelineScript.Name, - Script: ciPipelineScript.Script, - OutputLocation: ciPipelineScript.OutputLocation, - } - - if ciPipelineScript.Stage == BEFORE_DOCKER_BUILD { - beforeDockerBuildScripts = append(beforeDockerBuildScripts, ciTask) - } else if ciPipelineScript.Stage == AFTER_DOCKER_BUILD { - afterDockerBuildScripts = append(afterDockerBuildScripts, ciTask) - } - } - - if !(len(beforeDockerBuildScripts) == 0 && len(afterDockerBuildScripts) == 0) { - // found beforeDockerBuildScripts/afterDockerBuildScripts - // building preCiSteps & postCiSteps from them, refPluginsData not needed - preCiSteps = buildCiStepsDataFromDockerBuildScripts(beforeDockerBuildScripts) - postCiSteps = buildCiStepsDataFromDockerBuildScripts(afterDockerBuildScripts) - refPluginsData = []*pipelineConfigBean.RefPluginObject{} - } - host, err := impl.attributeService.GetByKey(bean4.HostUrlKey) - if err != nil { - impl.Logger.Errorw("error in getting host url", "err", err, "hostUrl", host.Value) - return nil, err - } - ciWorkflowConfigCiCacheBucket := impl.config.DefaultCacheBucket - - ciWorkflowConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion - - ciWorkflowConfigCiImage := impl.config.GetDefaultImage() - ciTemplate := pipeline.CiTemplate - ciLevelArgs := pipeline.DockerArgs - - if ciLevelArgs == "" { - ciLevelArgs = "{}" - } - - if pipeline.CiTemplate.DockerBuildOptions == "" { - pipeline.CiTemplate.DockerBuildOptions = "{}" - } - userEmailId, err := impl.userService.GetActiveEmailById(trigger.TriggeredBy) - if err != nil { - impl.Logger.Errorw("unable to find user email by id", "err", err, "id", trigger.TriggeredBy) - return nil, err - } - var dockerfilePath string - var dockerRepository string - var checkoutPath string - var ciBuildConfigBean *buildBean.CiBuildConfigBean - dockerRegistry := &repository3.DockerArtifactStore{} - ciBaseBuildConfigEntity := ciTemplate.CiBuildConfig - ciBaseBuildConfigBean, err := adapter.ConvertDbBuildConfigToBean(ciBaseBuildConfigEntity) - if err != nil { - impl.Logger.Errorw("error occurred while converting buildconfig dbEntity to configBean", "ciBuildConfigEntity", ciBaseBuildConfigEntity, "err", err) - return nil, errors.New("error while parsing ci build config") - } - if !pipeline.IsExternal && pipeline.IsDockerConfigOverridden { - templateOverrideBean, err := impl.ciTemplateService.FindTemplateOverrideByCiPipelineId(pipeline.Id) - if err != nil { - return nil, err - } - ciBuildConfigBean = templateOverrideBean.CiBuildConfig - // updating args coming from ciBaseBuildConfigEntity because it is not part of Ci override - if ciBuildConfigBean != nil && ciBuildConfigBean.DockerBuildConfig != nil && ciBaseBuildConfigBean != nil && ciBaseBuildConfigBean.DockerBuildConfig != nil { - ciBuildConfigBean.DockerBuildConfig.Args = ciBaseBuildConfigBean.DockerBuildConfig.Args - } - templateOverride := templateOverrideBean.CiTemplateOverride - checkoutPath = templateOverride.GitMaterial.CheckoutPath - dockerfilePath = templateOverride.DockerfilePath - dockerRepository = templateOverride.DockerRepository - dockerRegistry = templateOverride.DockerRegistry - } else { - checkoutPath = ciTemplate.GitMaterial.CheckoutPath - dockerfilePath = ciTemplate.DockerfilePath - dockerRegistry = ciTemplate.DockerRegistry - dockerRepository = ciTemplate.DockerRepository - ciBuildConfigBean = ciBaseBuildConfigBean - if ciBuildConfigBean != nil { - ciBuildConfigBean.BuildContextGitMaterialId = ciTemplate.BuildContextGitMaterialId - } - - } - if checkoutPath == "" { - checkoutPath = "./" - } - var dockerImageTag string - customTag, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id)) - if err != nil && err != pg.ErrNoRows { - return nil, err - } - if customTag.Id != 0 && customTag.Enabled == true { - imagePathReservation, err := impl.customTagService.GenerateImagePath(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id), dockerRegistry.RegistryURL, dockerRepository) - if err != nil { - if errors.Is(err, pipelineConfigBean.ErrImagePathInUse) { - errMsg := pipelineConfigBean.ImageTagUnavailableMessage - validationErr := util.NewApiError(http.StatusConflict, errMsg, errMsg) - dbErr := impl.markCurrentCiWorkflowFailed(savedWf, validationErr) - if dbErr != nil { - impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag", "err", dbErr, "savedWf", savedWf.Id) - } - return nil, err - } - return nil, err - } - savedWf.ImagePathReservationIds = []int{imagePathReservation.Id} - // imagePath = docker.io/avd0/dashboard:fd23414b - imagePathSplit := strings.Split(imagePathReservation.ImagePath, ":") - if len(imagePathSplit) >= 1 { - dockerImageTag = imagePathSplit[len(imagePathSplit)-1] - } - } else { - dockerImageTag = impl.buildImageTag(commitHashes, pipeline.Id, savedWf.Id) - } - - // copyContainerImage plugin specific logic - var registryCredentialMap map[string]bean2.RegistryCredentials - var pluginArtifactStage string - var imageReservationIds []int - var registryDestinationImageMap map[string][]string - if !isJob { - registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imageReservationIds, err = impl.GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps, postCiSteps, dockerImageTag, customTag.Id, - fmt.Sprintf(pipelineConfigBean.ImagePathPattern, - dockerRegistry.RegistryURL, - dockerRepository, - dockerImageTag), - dockerRegistry.Id) - if err != nil { - impl.Logger.Errorw("error in getting env variables for copyContainerImage plugin") - dbErr := impl.markCurrentCiWorkflowFailed(savedWf, err) - if dbErr != nil { - impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag", "err", dbErr, "savedWf", savedWf.Id) - } - return nil, err - } - - savedWf.ImagePathReservationIds = append(savedWf.ImagePathReservationIds, imageReservationIds...) - } - // mergedArgs := string(merged) - oldArgs := ciTemplate.Args - ciBuildConfigBean, err = adapter.OverrideCiBuildConfig(dockerfilePath, oldArgs, ciLevelArgs, ciTemplate.DockerBuildOptions, ciTemplate.TargetPlatform, ciBuildConfigBean) - if err != nil { - impl.Logger.Errorw("error occurred while overriding ci build config", "oldArgs", oldArgs, "ciLevelArgs", ciLevelArgs, "error", err) - return nil, errors.New("error while parsing ci build config") - } - buildContextCheckoutPath, err := impl.ciPipelineMaterialRepository.GetCheckoutPath(ciBuildConfigBean.BuildContextGitMaterialId) - if err != nil && err != pg.ErrNoRows { - impl.Logger.Errorw("error occurred while getting checkout path from git material", "gitMaterialId", ciBuildConfigBean.BuildContextGitMaterialId, "error", err) - return nil, err - } - if buildContextCheckoutPath == "" { - buildContextCheckoutPath = checkoutPath - } - if ciBuildConfigBean.UseRootBuildContext { - // use root build context i.e '.' - buildContextCheckoutPath = "." - } - - ciBuildConfigBean.PipelineType = trigger.PipelineType - - if ciBuildConfigBean.CiBuildType == buildBean.SELF_DOCKERFILE_BUILD_TYPE || ciBuildConfigBean.CiBuildType == buildBean.MANAGED_DOCKERFILE_BUILD_TYPE { - ciBuildConfigBean.DockerBuildConfig.BuildContext = filepath.Join(buildContextCheckoutPath, ciBuildConfigBean.DockerBuildConfig.BuildContext) - dockerBuildConfig := ciBuildConfigBean.DockerBuildConfig - dockerfilePath = filepath.Join(checkoutPath, dockerBuildConfig.DockerfilePath) - dockerBuildConfig.DockerfilePath = dockerfilePath - checkoutPath = dockerfilePath[:strings.LastIndex(dockerfilePath, "/")+1] - } else if ciBuildConfigBean.CiBuildType == buildBean.BUILDPACK_BUILD_TYPE { - buildPackConfig := ciBuildConfigBean.BuildPackConfig - checkoutPath = filepath.Join(checkoutPath, buildPackConfig.ProjectPath) - } - - defaultTargetPlatform := impl.config.DefaultTargetPlatform - useBuildx := impl.config.UseBuildx - - if ciBuildConfigBean.DockerBuildConfig != nil { - if ciBuildConfigBean.DockerBuildConfig.TargetPlatform == "" && useBuildx { - ciBuildConfigBean.DockerBuildConfig.TargetPlatform = defaultTargetPlatform - ciBuildConfigBean.DockerBuildConfig.UseBuildx = useBuildx - } - ciBuildConfigBean.DockerBuildConfig.BuildxProvenanceMode = impl.config.BuildxProvenanceMode - } - - workflowRequest := &types.WorkflowRequest{ - WorkflowNamePrefix: strconv.Itoa(savedWf.Id) + "-" + savedWf.Name, - PipelineName: pipeline.Name, - PipelineId: pipeline.Id, - CiCacheFileName: pipeline.Name + "-" + strconv.Itoa(pipeline.Id) + ".tar.gz", - CiProjectDetails: ciProjectDetails, - Namespace: ciWorkflowConfigNamespace, - BlobStorageConfigured: savedWf.BlobStorageEnabled, - CiImage: ciWorkflowConfigCiImage, - WorkflowId: savedWf.Id, - TriggeredBy: savedWf.TriggeredBy, - CacheLimit: impl.config.CacheLimit, - ScanEnabled: pipeline.ScanEnabled, - CloudProvider: impl.config.CloudProvider, - DefaultAddressPoolBaseCidr: impl.config.GetDefaultAddressPoolBaseCidr(), - DefaultAddressPoolSize: impl.config.GetDefaultAddressPoolSize(), - PreCiSteps: preCiSteps, - PostCiSteps: postCiSteps, - RefPlugins: refPluginsData, - AppName: pipeline.App.AppName, - TriggerByAuthor: userEmailId, - CiBuildConfig: ciBuildConfigBean, - CiBuildDockerMtuValue: impl.config.CiRunnerDockerMTUValue, - CacheInvalidate: trigger.InvalidateCache, - SystemEnvironmentVariables: trigger.RuntimeParameters.GetSystemVariables(), - EnableBuildContext: impl.config.EnableBuildContext, - OrchestratorHost: impl.config.OrchestratorHost, - HostUrl: host.Value, - OrchestratorToken: impl.config.OrchestratorToken, - ImageRetryCount: impl.config.ImageRetryCount, - ImageRetryInterval: impl.config.ImageRetryInterval, - WorkflowExecutor: impl.config.GetWorkflowExecutorType(), - Type: pipelineConfigBean.CI_WORKFLOW_PIPELINE_TYPE, - CiArtifactLastFetch: trigger.CiArtifactLastFetch, - RegistryDestinationImageMap: registryDestinationImageMap, - RegistryCredentialMap: registryCredentialMap, - PluginArtifactStage: pluginArtifactStage, - ImageScanMaxRetries: impl.config.ImageScanMaxRetries, - ImageScanRetryDelay: impl.config.ImageScanRetryDelay, - UseDockerApiToGetDigest: impl.config.UseDockerApiToGetDigest, - } - workflowRequest.SetAwsInspectorConfig("") - //in oss, there is no pipeline level workflow cache config, so we pass inherit to get the app level config - workflowCacheConfig := impl.ciCdPipelineOrchestrator.GetWorkflowCacheConfig(pipeline.App.AppType, trigger.PipelineType, common.WorkflowCacheConfigInherit) - workflowRequest.IgnoreDockerCachePush = !workflowCacheConfig.Value - workflowRequest.IgnoreDockerCachePull = !workflowCacheConfig.Value - impl.Logger.Debugw("Ignore Cache values", "IgnoreDockerCachePush", workflowRequest.IgnoreDockerCachePush, "IgnoreDockerCachePull", workflowRequest.IgnoreDockerCachePull) - if pipeline.App.AppType == helper.Job { - workflowRequest.AppName = pipeline.App.DisplayName - } - if pipeline.ScanEnabled { - scanToolMetadata, scanVia, err := impl.fetchImageScanExecutionMedium() - if err != nil { - impl.Logger.Errorw("error occurred getting scanned via", "err", err) - return nil, err - } - workflowRequest.SetExecuteImageScanningVia(scanVia) - if scanVia.IsScanMediumExternal() { - imageScanExecutionSteps, refPlugins, err := impl.fetchImageScanExecutionStepsForWfRequest(scanToolMetadata) - if err != nil { - impl.Logger.Errorw("error occurred, fetchImageScanExecutionStepsForWfRequest", "scanToolMetadata", scanToolMetadata, "err", err) - return nil, err - } - workflowRequest.SetImageScanningSteps(imageScanExecutionSteps) - workflowRequest.RefPlugins = append(workflowRequest.RefPlugins, refPlugins...) - } - } - - if dockerRegistry != nil { - workflowRequest.DockerRegistryId = dockerRegistry.Id - workflowRequest.DockerRegistryType = string(dockerRegistry.RegistryType) - workflowRequest.DockerImageTag = dockerImageTag - workflowRequest.DockerRegistryURL = dockerRegistry.RegistryURL - workflowRequest.DockerRepository = dockerRepository - workflowRequest.CheckoutPath = checkoutPath - workflowRequest.DockerUsername = dockerRegistry.Username - workflowRequest.DockerPassword = dockerRegistry.Password - workflowRequest.AwsRegion = dockerRegistry.AWSRegion - workflowRequest.AccessKey = dockerRegistry.AWSAccessKeyId - workflowRequest.SecretKey = dockerRegistry.AWSSecretAccessKey - workflowRequest.DockerConnection = dockerRegistry.Connection - workflowRequest.DockerCert = dockerRegistry.Cert - - } - ciWorkflowConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() - - switch workflowRequest.CloudProvider { - case types.BLOB_STORAGE_S3: - // No AccessKey is used for uploading artifacts, instead IAM based auth is used - workflowRequest.CiCacheRegion = ciWorkflowConfigCiCacheRegion - workflowRequest.CiCacheLocation = ciWorkflowConfigCiCacheBucket - workflowRequest.CiArtifactLocation, workflowRequest.CiArtifactBucket, workflowRequest.CiArtifactFileName = impl.buildS3ArtifactLocation(ciWorkflowConfigLogsBucket, savedWf) - workflowRequest.BlobStorageS3Config = &blob_storage.BlobStorageS3Config{ - AccessKey: impl.config.BlobStorageS3AccessKey, - Passkey: impl.config.BlobStorageS3SecretKey, - EndpointUrl: impl.config.BlobStorageS3Endpoint, - IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - CiCacheBucketName: ciWorkflowConfigCiCacheBucket, - CiCacheRegion: ciWorkflowConfigCiCacheRegion, - CiCacheBucketVersioning: impl.config.BlobStorageS3BucketVersioned, - CiArtifactBucketName: workflowRequest.CiArtifactBucket, - CiArtifactRegion: impl.config.GetDefaultCdLogsBucketRegion(), - CiArtifactBucketVersioning: impl.config.BlobStorageS3BucketVersioned, - CiLogBucketName: impl.config.GetDefaultBuildLogsBucket(), - CiLogRegion: impl.config.GetDefaultCdLogsBucketRegion(), - CiLogBucketVersioning: impl.config.BlobStorageS3BucketVersioned, - } - case types.BLOB_STORAGE_GCP: - workflowRequest.GcpBlobConfig = &blob_storage.GcpBlobConfig{ - CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - CacheBucketName: ciWorkflowConfigCiCacheBucket, - LogBucketName: ciWorkflowConfigLogsBucket, - ArtifactBucketName: ciWorkflowConfigLogsBucket, - } - workflowRequest.CiArtifactLocation = impl.buildDefaultArtifactLocation(savedWf) - workflowRequest.CiArtifactFileName = workflowRequest.CiArtifactLocation - case types.BLOB_STORAGE_AZURE: - workflowRequest.AzureBlobConfig = &blob_storage.AzureBlobConfig{ - Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, - AccountName: impl.config.AzureAccountName, - BlobContainerCiCache: impl.config.AzureBlobContainerCiCache, - AccountKey: impl.config.AzureAccountKey, - BlobContainerCiLog: impl.config.AzureBlobContainerCiLog, - BlobContainerArtifact: impl.config.AzureBlobContainerCiLog, - } - workflowRequest.BlobStorageS3Config = &blob_storage.BlobStorageS3Config{ - EndpointUrl: impl.config.AzureGatewayUrl, - IsInSecure: impl.config.AzureGatewayConnectionInsecure, - CiLogBucketName: impl.config.AzureBlobContainerCiLog, - CiLogRegion: impl.config.DefaultCacheBucketRegion, - CiLogBucketVersioning: impl.config.BlobStorageS3BucketVersioned, - AccessKey: impl.config.AzureAccountName, - } - workflowRequest.CiArtifactLocation = impl.buildDefaultArtifactLocation(savedWf) - workflowRequest.CiArtifactFileName = workflowRequest.CiArtifactLocation - default: - if impl.config.BlobStorageEnabled { - return nil, fmt.Errorf("blob storage %s not supported", workflowRequest.CloudProvider) - } - } - return workflowRequest, nil -} - -func (impl *CiServiceImpl) GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, customTag string, customTagId int, buildImagePath string, buildImagedockerRegistryId string) (map[string][]string, map[string]bean2.RegistryCredentials, string, []int, error) { - - copyContainerImagePluginDetail, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(COPY_CONTAINER_IMAGE) - if err != nil && err != pg.ErrNoRows { - impl.Logger.Errorw("error in getting copyContainerImage plugin id", "err", err) - return nil, nil, "", nil, err - } - - pluginIdToVersionMap := make(map[int]string) - for _, p := range copyContainerImagePluginDetail { - pluginIdToVersionMap[p.Id] = p.Version - } - - for _, step := range preCiSteps { - if _, ok := pluginIdToVersionMap[step.RefPluginId]; ok { - // for copyContainerImage plugin parse destination images and save its data in image path reservation table - return nil, nil, "", nil, errors.New("copyContainerImage plugin not allowed in pre-ci step, please remove it and try again") - } - } - - registryCredentialMap := make(map[string]bean2.RegistryCredentials) - registryDestinationImageMap := make(map[string][]string) - var allDestinationImages []string //saving all images to be reserved in this array - - for _, step := range postCiSteps { - if version, ok := pluginIdToVersionMap[step.RefPluginId]; ok { - destinationImageMap, credentialMap, err := impl.pluginInputVariableParser.HandleCopyContainerImagePluginInputVariables(step.InputVars, customTag, buildImagePath, buildImagedockerRegistryId) - if err != nil { - impl.Logger.Errorw("error in parsing copyContainerImage input variable", "err", err) - return nil, nil, "", nil, err - } - if version == COPY_CONTAINER_IMAGE_VERSION_V1 { - // this is needed in ci runner only for v1 - registryDestinationImageMap = destinationImageMap - } - for _, images := range destinationImageMap { - allDestinationImages = append(allDestinationImages, images...) - } - for k, v := range credentialMap { - registryCredentialMap[k] = v - } - } - } - - pluginArtifactStage := repository5.POST_CI - for _, image := range allDestinationImages { - if image == buildImagePath { - return nil, registryCredentialMap, pluginArtifactStage, nil, - pipelineConfigBean.ErrImagePathInUse - } - } - savedCIArtifacts, err := impl.ciArtifactRepository.FindCiArtifactByImagePaths(allDestinationImages) - if err != nil { - impl.Logger.Errorw("error in fetching artifacts by image path", "err", err) - return nil, nil, pluginArtifactStage, nil, err - } - if len(savedCIArtifacts) > 0 { - // if already present in ci artifact, return "image path already in use error" - return nil, nil, pluginArtifactStage, nil, pipelineConfigBean.ErrImagePathInUse - } - imagePathReservationIds, err := impl.ReserveImagesGeneratedAtPlugin(customTagId, allDestinationImages) - if err != nil { - return nil, nil, pluginArtifactStage, imagePathReservationIds, err - } - return registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imagePathReservationIds, nil -} - -func (impl *CiServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, destinationImages []string) ([]int, error) { - var imagePathReservationIds []int - for _, image := range destinationImages { - imagePathReservationData, err := impl.customTagService.ReserveImagePath(image, customTagId) - if err != nil { - impl.Logger.Errorw("Error in marking custom tag reserved", "err", err) - return imagePathReservationIds, err - } - imagePathReservationIds = append(imagePathReservationIds, imagePathReservationData.Id) - } - return imagePathReservationIds, nil -} - -func buildCiStepsDataFromDockerBuildScripts(dockerBuildScripts []*bean.CiScript) []*pipelineConfigBean.StepObject { - // before plugin support, few variables were set as env vars in ci-runner - // these variables are now moved to global vars in plugin steps, but to avoid error in old scripts adding those variables in payload - inputVars := []*commonBean.VariableObject{ - { - Name: "DOCKER_IMAGE_TAG", - Format: "STRING", - VariableType: commonBean.VariableTypeRefGlobal, - ReferenceVariableName: "DOCKER_IMAGE_TAG", - }, - { - Name: "DOCKER_REPOSITORY", - Format: "STRING", - VariableType: commonBean.VariableTypeRefGlobal, - ReferenceVariableName: "DOCKER_REPOSITORY", - }, - { - Name: "DOCKER_REGISTRY_URL", - Format: "STRING", - VariableType: commonBean.VariableTypeRefGlobal, - ReferenceVariableName: "DOCKER_REGISTRY_URL", - }, - { - Name: "DOCKER_IMAGE", - Format: "STRING", - VariableType: commonBean.VariableTypeRefGlobal, - ReferenceVariableName: "DOCKER_IMAGE", - }, - } - var ciSteps []*pipelineConfigBean.StepObject - for _, dockerBuildScript := range dockerBuildScripts { - ciStep := &pipelineConfigBean.StepObject{ - Name: dockerBuildScript.Name, - Index: dockerBuildScript.Index, - Script: dockerBuildScript.Script, - ArtifactPaths: []string{dockerBuildScript.OutputLocation}, - StepType: string(repository.PIPELINE_STEP_TYPE_INLINE), - ExecutorType: string(repository2.SCRIPT_TYPE_SHELL), - InputVars: inputVars, - } - ciSteps = append(ciSteps, ciStep) - } - return ciSteps -} - -func (impl *CiServiceImpl) buildImageTag(commitHashes map[int]pipelineConfig.GitCommit, id int, wfId int) string { - dockerImageTag := "" - toAppendDevtronParamInTag := true - for _, v := range commitHashes { - if v.WebhookData.Id == 0 { - if v.Commit == "" { - continue - } - dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _getTruncatedImageTag(v.Commit)) - } else { - _targetCheckout := v.WebhookData.Data[bean.WEBHOOK_SELECTOR_TARGET_CHECKOUT_NAME] - if _targetCheckout == "" { - continue - } - // if not PR based then meaning tag based - isPRBasedEvent := v.WebhookData.EventActionType == bean.WEBHOOK_EVENT_MERGED_ACTION_TYPE - if !isPRBasedEvent && impl.config.CiCdConfig.UseImageTagFromGitProviderForTagBasedBuild { - dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _targetCheckout) - } else { - dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _getTruncatedImageTag(_targetCheckout)) - } - if isPRBasedEvent { - _sourceCheckout := v.WebhookData.Data[bean.WEBHOOK_SELECTOR_SOURCE_CHECKOUT_NAME] - dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _getTruncatedImageTag(_sourceCheckout)) - } else { - toAppendDevtronParamInTag = !impl.config.CiCdConfig.UseImageTagFromGitProviderForTagBasedBuild - } - } - } - toAppendDevtronParamInTag = toAppendDevtronParamInTag && dockerImageTag != "" - if toAppendDevtronParamInTag { - dockerImageTag = fmt.Sprintf("%s-%d-%d", dockerImageTag, id, wfId) - } - // replace / with underscore, as docker image tag doesn't support slash. it gives error - dockerImageTag = strings.ReplaceAll(dockerImageTag, "/", "_") - return dockerImageTag -} - -func getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, commitOrCheckOutData string) string { - if dockerImageTag == "" { - dockerImageTag = commitOrCheckOutData - } else { - if commitOrCheckOutData != "" { - dockerImageTag = fmt.Sprintf("%s-%s", dockerImageTag, commitOrCheckOutData) - } - } - return dockerImageTag -} - -func (impl *CiServiceImpl) updateCiWorkflow(request *types.WorkflowRequest, savedWf *pipelineConfig.CiWorkflow) error { - ciBuildConfig := request.CiBuildConfig - ciBuildType := string(ciBuildConfig.CiBuildType) - savedWf.CiBuildType = ciBuildType - return impl.UpdateCiWorkflowWithStage(savedWf) -} - -func _getTruncatedImageTag(imageTag string) string { - _length := len(imageTag) - if _length == 0 { - return imageTag - } - - _truncatedLength := 8 - - if _length < _truncatedLength { - return imageTag - } else { - return imageTag[:_truncatedLength] - } -} - func (impl *CiServiceImpl) WriteCIFailEvent(ciWorkflow *pipelineConfig.CiWorkflow) { event, _ := impl.eventFactory.Build(util2.Fail, &ciWorkflow.CiPipelineId, ciWorkflow.CiPipeline.AppId, nil, util2.CI) material := &buildBean.MaterialTriggerInfo{} diff --git a/pkg/pipeline/pipelineStageVariableParser.go b/pkg/pipeline/pipelineStageVariableParser.go index b97517d30b..bd0d3bce77 100644 --- a/pkg/pipeline/pipelineStageVariableParser.go +++ b/pkg/pipeline/pipelineStageVariableParser.go @@ -30,13 +30,9 @@ import ( ) type copyContainerImagePluginInputVariable = string -type RefPluginName = string const ( - COPY_CONTAINER_IMAGE RefPluginName = "Copy container image" - COPY_CONTAINER_IMAGE_VERSION_V1 = "1.0.0" - COPY_CONTAINER_IMAGE_VERSION_V2 = "2.0.0" - EMPTY_STRING = " " + EMPTY_STRING = " " ) const ( From 3bf9cd7f474e4390581c027b5413aa2113cafedb Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Wed, 2 Apr 2025 10:37:03 +0530 Subject: [PATCH 04/37] wire file missed --- pkg/build/trigger/wire_trigger.go | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 pkg/build/trigger/wire_trigger.go diff --git a/pkg/build/trigger/wire_trigger.go b/pkg/build/trigger/wire_trigger.go new file mode 100644 index 0000000000..dc462bcd88 --- /dev/null +++ b/pkg/build/trigger/wire_trigger.go @@ -0,0 +1,10 @@ +package trigger + +import ( + "github.com/google/wire" +) + +var WireSet = wire.NewSet( + NewServiceImpl, + wire.Bind(new(Service), new(*ServiceImpl)), +) From 3d23b352dcdb4d2f36555c452894258f87e9de11 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Wed, 2 Apr 2025 18:55:20 +0530 Subject: [PATCH 05/37] trigger service moved completely --- .../configure/BuildPipelineRestHandler.go | 2 +- .../configure/PipelineConfigRestHandler.go | 5 +- client/cron/CiStatusUpdateCron.go | 11 +- client/cron/CiTriggerCron.go | 11 +- pkg/build/git/gitWebhook/GitWebhookService.go | 11 +- pkg/build/trigger/Service.go | 433 ++++++++++++++ pkg/build/trigger/Service_ent.go | 10 + pkg/bulkAction/service/BulkUpdateService.go | 8 +- .../in/WorkflowEventProcessorService.go | 10 +- pkg/pipeline/CIHandler_ent.go | 10 - pkg/pipeline/CiHandler.go | 536 +----------------- pkg/workflow/dag/WorkflowDagExecutor.go | 150 +++++ wire_gen.go | 130 ++--- 13 files changed, 700 insertions(+), 627 deletions(-) diff --git a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go index 32448e0abf..9e749ec207 100644 --- a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go @@ -740,7 +740,7 @@ func (handler *PipelineConfigRestHandlerImpl) TriggerCiPipeline(w http.ResponseW ciTriggerRequest.TriggeredBy = userId handler.Logger.Infow("request payload, TriggerCiPipeline", "payload", ciTriggerRequest) response := make(map[string]string) - resp, err := handler.ciHandler.HandleCIManual(ciTriggerRequest) + resp, err := handler.ciTriggerService.HandleCIManual(ciTriggerRequest) if errors.Is(err, bean1.ErrImagePathInUse) { handler.Logger.Errorw("service err duplicate image tag, TriggerCiPipeline", "err", err, "payload", ciTriggerRequest) common.WriteJsonResp(w, err, err, http.StatusConflict) diff --git a/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go b/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go index 5e3e81defd..0980abae24 100644 --- a/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go +++ b/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go @@ -26,6 +26,7 @@ import ( read2 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/read" gitProviderRead "github.com/devtron-labs/devtron/pkg/build/git/gitProvider/read" bean3 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/build/trigger" "github.com/devtron-labs/devtron/pkg/chart/gitOpsConfig" read5 "github.com/devtron-labs/devtron/pkg/chart/read" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" @@ -138,6 +139,7 @@ type PipelineConfigRestHandlerImpl struct { teamReadService read3.TeamReadService environmentRepository repository2.EnvironmentRepository chartReadService read5.ChartReadService + ciTriggerService trigger.Service } func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger *zap.SugaredLogger, @@ -171,7 +173,8 @@ func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger gitProviderReadService gitProviderRead.GitProviderReadService, teamReadService read3.TeamReadService, EnvironmentRepository repository2.EnvironmentRepository, - chartReadService read5.ChartReadService) *PipelineConfigRestHandlerImpl { + chartReadService read5.ChartReadService, + ciTriggerService trigger.Service) *PipelineConfigRestHandlerImpl { envConfig := &PipelineRestHandlerEnvConfig{} err := env.Parse(envConfig) if err != nil { diff --git a/client/cron/CiStatusUpdateCron.go b/client/cron/CiStatusUpdateCron.go index 696feaed10..9aa4e55554 100644 --- a/client/cron/CiStatusUpdateCron.go +++ b/client/cron/CiStatusUpdateCron.go @@ -21,7 +21,7 @@ import ( "github.com/caarlos0/env" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/app" - "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/workflow/dag" cron2 "github.com/devtron-labs/devtron/util/cron" "github.com/robfig/cron/v3" "go.uber.org/zap" @@ -38,12 +38,13 @@ type CiStatusUpdateCronImpl struct { appService app.AppService ciWorkflowStatusUpdateConfig *CiWorkflowStatusUpdateConfig ciPipelineRepository pipelineConfig.CiPipelineRepository - ciHandler pipeline.CiHandler + workflowDagExecutor dag.WorkflowDagExecutor } func NewCiStatusUpdateCronImpl(logger *zap.SugaredLogger, appService app.AppService, ciWorkflowStatusUpdateConfig *CiWorkflowStatusUpdateConfig, ciPipelineRepository pipelineConfig.CiPipelineRepository, - ciHandler pipeline.CiHandler, cronLogger *cron2.CronLoggerImpl) *CiStatusUpdateCronImpl { + cronLogger *cron2.CronLoggerImpl, + workflowDagExecutor dag.WorkflowDagExecutor) *CiStatusUpdateCronImpl { cron := cron.New( cron.WithChain(cron.Recover(cronLogger))) cron.Start() @@ -53,7 +54,7 @@ func NewCiStatusUpdateCronImpl(logger *zap.SugaredLogger, appService app.AppServ appService: appService, ciWorkflowStatusUpdateConfig: ciWorkflowStatusUpdateConfig, ciPipelineRepository: ciPipelineRepository, - ciHandler: ciHandler, + workflowDagExecutor: workflowDagExecutor, } // execute periodically, update ci workflow status for failed process @@ -87,7 +88,7 @@ func (impl *CiStatusUpdateCronImpl) UpdateCiWorkflowStatusFailedCron() { impl.logger.Errorw("error in converting string to int", "err", err) return } - err = impl.ciHandler.UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild) + err = impl.workflowDagExecutor.UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild) if err != nil { impl.logger.Errorw("error in updating ci workflow status for failed workflows", "err", err) return diff --git a/client/cron/CiTriggerCron.go b/client/cron/CiTriggerCron.go index 35f1e9876a..b5b9305afa 100644 --- a/client/cron/CiTriggerCron.go +++ b/client/cron/CiTriggerCron.go @@ -23,7 +23,7 @@ import ( bean2 "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" - "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/build/trigger" "github.com/devtron-labs/devtron/pkg/pipeline/repository" repository3 "github.com/devtron-labs/devtron/pkg/plugin/repository" cron2 "github.com/devtron-labs/devtron/util/cron" @@ -40,13 +40,14 @@ type CiTriggerCronImpl struct { cron *cron.Cron cfg *CiTriggerCronConfig pipelineStageRepository repository.PipelineStageRepository - ciHandler pipeline.CiHandler ciArtifactRepository repository2.CiArtifactRepository globalPluginRepository repository3.GlobalPluginRepository + ciTriggerService trigger.Service } func NewCiTriggerCronImpl(logger *zap.SugaredLogger, cfg *CiTriggerCronConfig, pipelineStageRepository repository.PipelineStageRepository, - ciHandler pipeline.CiHandler, ciArtifactRepository repository2.CiArtifactRepository, globalPluginRepository repository3.GlobalPluginRepository, cronLogger *cron2.CronLoggerImpl) *CiTriggerCronImpl { + ciArtifactRepository repository2.CiArtifactRepository, globalPluginRepository repository3.GlobalPluginRepository, cronLogger *cron2.CronLoggerImpl, + ciTriggerService trigger.Service) *CiTriggerCronImpl { cron := cron.New( cron.WithChain(cron.Recover(cronLogger))) cron.Start() @@ -54,10 +55,10 @@ func NewCiTriggerCronImpl(logger *zap.SugaredLogger, cfg *CiTriggerCronConfig, p logger: logger, cron: cron, pipelineStageRepository: pipelineStageRepository, - ciHandler: ciHandler, cfg: cfg, ciArtifactRepository: ciArtifactRepository, globalPluginRepository: globalPluginRepository, + ciTriggerService: ciTriggerService, } _, err := cron.AddFunc(fmt.Sprintf("@every %dm", cfg.SourceControllerCronTime), impl.TriggerCiCron) @@ -103,7 +104,7 @@ func (impl *CiTriggerCronImpl) TriggerCiCron() { InvalidateCache: false, PipelineType: string(common.CI_JOB), } - _, err = impl.ciHandler.HandleCIManual(ciTriggerRequest) + _, err = impl.ciTriggerService.HandleCIManual(ciTriggerRequest) if err != nil { return } diff --git a/pkg/build/git/gitWebhook/GitWebhookService.go b/pkg/build/git/gitWebhook/GitWebhookService.go index 3f301db311..8b4405a68b 100644 --- a/pkg/build/git/gitWebhook/GitWebhookService.go +++ b/pkg/build/git/gitWebhook/GitWebhookService.go @@ -23,7 +23,7 @@ import ( bean2 "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/build/git/gitWebhook/repository" - "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/build/trigger" "go.uber.org/zap" ) @@ -33,15 +33,16 @@ type GitWebhookService interface { type GitWebhookServiceImpl struct { logger *zap.SugaredLogger - ciHandler pipeline.CiHandler gitWebhookRepository repository.GitWebhookRepository + ciTriggerService trigger.Service } -func NewGitWebhookServiceImpl(Logger *zap.SugaredLogger, ciHandler pipeline.CiHandler, gitWebhookRepository repository.GitWebhookRepository) *GitWebhookServiceImpl { +func NewGitWebhookServiceImpl(Logger *zap.SugaredLogger, gitWebhookRepository repository.GitWebhookRepository, + ciTriggerService trigger.Service) *GitWebhookServiceImpl { return &GitWebhookServiceImpl{ logger: Logger, - ciHandler: ciHandler, gitWebhookRepository: gitWebhookRepository, + ciTriggerService: ciTriggerService, } } @@ -70,7 +71,7 @@ func (impl *GitWebhookServiceImpl) HandleGitWebhook(gitWebhookRequest gitSensor. } } - resp, err := impl.ciHandler.HandleCIWebhook(bean.GitCiTriggerRequest{ + resp, err := impl.ciTriggerService.HandleCIWebhook(bean.GitCiTriggerRequest{ CiPipelineMaterial: ciPipelineMaterial, TriggeredBy: bean2.SYSTEM_USER_ID, // Automatic trigger, system user ExtraEnvironmentVariables: gitWebhookRequest.ExtraEnvironmentVariables, diff --git a/pkg/build/trigger/Service.go b/pkg/build/trigger/Service.go index 67cff996ab..48f217c40e 100644 --- a/pkg/build/trigger/Service.go +++ b/pkg/build/trigger/Service.go @@ -1,15 +1,18 @@ package trigger import ( + "context" "encoding/json" "errors" "fmt" + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/caarlos0/env" blob_storage "github.com/devtron-labs/common-lib/blob-storage" "github.com/devtron-labs/common-lib/utils" bean4 "github.com/devtron-labs/common-lib/utils/bean" commonBean "github.com/devtron-labs/common-lib/workflow" client "github.com/devtron-labs/devtron/client/events" + "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/middleware" "github.com/devtron-labs/devtron/internal/sql/constants" repository5 "github.com/devtron-labs/devtron/internal/sql/repository" @@ -23,6 +26,7 @@ import ( "github.com/devtron-labs/devtron/pkg/attributes" bean3 "github.com/devtron-labs/devtron/pkg/attributes/bean" "github.com/devtron-labs/devtron/pkg/auth/user" + bean6 "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/bean/common" "github.com/devtron-labs/devtron/pkg/build/pipeline" @@ -32,6 +36,7 @@ import ( pipeline2 "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/adapter" pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/plugin" @@ -53,6 +58,10 @@ import ( ) type Service interface { + HandlePodDeleted(ciWorkflow *pipelineConfig.CiWorkflow) + CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error + HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) + HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) TriggerCiPipeline(trigger types.Trigger) (int, error) } @@ -82,6 +91,8 @@ type ServiceImpl struct { pluginInputVariableParser pipeline2.PluginInputVariableParser globalPluginService plugin.GlobalPluginService ciService pipeline2.CiService + ciWorkflowRepository pipelineConfig.CiWorkflowRepository + gitSensorClient gitSensor.Client } func NewServiceImpl(Logger *zap.SugaredLogger, workflowService pipeline2.WorkflowService, @@ -100,6 +111,8 @@ func NewServiceImpl(Logger *zap.SugaredLogger, workflowService pipeline2.Workflo pluginInputVariableParser pipeline2.PluginInputVariableParser, globalPluginService plugin.GlobalPluginService, ciService pipeline2.CiService, + ciWorkflowRepository pipelineConfig.CiWorkflowRepository, + gitSensorClient gitSensor.Client, ) *ServiceImpl { buildxCacheFlags := &BuildxCacheFlags{} err := env.Parse(buildxCacheFlags) @@ -126,6 +139,8 @@ func NewServiceImpl(Logger *zap.SugaredLogger, workflowService pipeline2.Workflo pluginInputVariableParser: pluginInputVariableParser, globalPluginService: globalPluginService, ciService: ciService, + ciWorkflowRepository: ciWorkflowRepository, + gitSensorClient: gitSensorClient, } config, err := types.GetCiConfig() if err != nil { @@ -135,6 +150,424 @@ func NewServiceImpl(Logger *zap.SugaredLogger, workflowService pipeline2.Workflo return cis } +func (impl *ServiceImpl) HandlePodDeleted(ciWorkflow *pipelineConfig.CiWorkflow) { + if !impl.config.WorkflowRetriesEnabled() { + impl.Logger.Debug("ci workflow retry feature disabled") + return + } + retryCount, refCiWorkflow, err := impl.getRefWorkflowAndCiRetryCount(ciWorkflow) + if err != nil { + impl.Logger.Errorw("error in getRefWorkflowAndCiRetryCount", "ciWorkflowId", ciWorkflow.Id, "err", err) + } + impl.Logger.Infow("re-triggering ci by UpdateCiWorkflowStatusFailedCron", "refCiWorkflowId", refCiWorkflow.Id, "ciWorkflow.Status", ciWorkflow.Status, "ciWorkflow.Message", ciWorkflow.Message, "retryCount", retryCount) + err = impl.reTriggerCi(retryCount, refCiWorkflow) + if err != nil { + impl.Logger.Errorw("error in reTriggerCi", "ciWorkflowId", refCiWorkflow.Id, "workflowStatus", ciWorkflow.Status, "ciWorkflowMessage", "ciWorkflow.Message", "retryCount", retryCount, "err", err) + } +} + +func (impl *ServiceImpl) CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error { + + // return if re-trigger feature is disabled + if !impl.config.WorkflowRetriesEnabled() { + impl.Logger.Debug("CI re-trigger is disabled") + return nil + } + + status, message, ciWorkFlow, err := impl.extractPodStatusAndWorkflow(workflowStatus) + if err != nil { + impl.Logger.Errorw("error in extractPodStatusAndWorkflow", "err", err) + return err + } + + if !executors.CheckIfReTriggerRequired(status, message, ciWorkFlow.Status) { + impl.Logger.Debugw("not re-triggering ci", "status", status, "message", message, "ciWorkflowStatus", ciWorkFlow.Status) + return nil + } + + impl.Logger.Debugw("re-triggering ci", "status", status, "message", message, "ciWorkflowStatus", ciWorkFlow.Status, "ciWorkFlowId", ciWorkFlow.Id) + + retryCount, refCiWorkflow, err := impl.getRefWorkflowAndCiRetryCount(ciWorkFlow) + if err != nil { + impl.Logger.Errorw("error while getting retry count value for a ciWorkflow", "ciWorkFlowId", ciWorkFlow.Id) + return err + } + + err = impl.reTriggerCi(retryCount, refCiWorkflow) + if err != nil { + impl.Logger.Errorw("error in reTriggerCi", "err", err, "status", status, "message", message, "retryCount", retryCount, "ciWorkFlowId", ciWorkFlow.Id) + } + return err +} + +func (impl *ServiceImpl) reTriggerCi(retryCount int, refCiWorkflow *pipelineConfig.CiWorkflow) error { + if retryCount >= impl.config.MaxCiWorkflowRetries { + impl.Logger.Infow("maximum retries exhausted for this ciWorkflow", "ciWorkflowId", refCiWorkflow.Id, "retries", retryCount, "configuredRetries", impl.config.MaxCiWorkflowRetries) + return nil + } + impl.Logger.Infow("re-triggering ci for a ci workflow", "ReferenceCiWorkflowId", refCiWorkflow.Id) + ciPipelineMaterialIds := make([]int, 0, len(refCiWorkflow.GitTriggers)) + for id, _ := range refCiWorkflow.GitTriggers { + ciPipelineMaterialIds = append(ciPipelineMaterialIds, id) + } + ciMaterials, err := impl.ciPipelineMaterialRepository.GetByIdsIncludeDeleted(ciPipelineMaterialIds) + if err != nil { + impl.Logger.Errorw("error in getting ci Pipeline Materials using ciPipeline Material Ids", "ciPipelineMaterialIds", ciPipelineMaterialIds, "err", err) + return err + } + + trigger := types.Trigger{} + trigger.BuildTriggerObject(refCiWorkflow, ciMaterials, bean6.SYSTEM_USER_ID, true, nil, "") + + // updating runtime params + trigger.RuntimeParameters, err = impl.updateRuntimeParamsForAutoCI(trigger.PipelineId, trigger.RuntimeParameters) + if err != nil { + impl.Logger.Errorw("err, updateRuntimeParamsForAutoCI", "ciPipelineId", trigger.PipelineId, + "runtimeParameters", trigger.RuntimeParameters, "err", err) + return err + } + _, err = impl.TriggerCiPipeline(trigger) + + if err != nil { + impl.Logger.Errorw("error occurred in re-triggering ciWorkflow", "triggerDetails", trigger, "err", err) + return err + } + return nil +} + +func (impl *ServiceImpl) HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) { + impl.Logger.Debugw("HandleCIManual for pipeline ", "PipelineId", ciTriggerRequest.PipelineId) + commitHashes, runtimeParams, err := impl.buildManualTriggerCommitHashes(ciTriggerRequest) + if err != nil { + return 0, err + } + + ciArtifact, err := impl.ciArtifactRepository.GetLatestArtifactTimeByCiPipelineId(ciTriggerRequest.PipelineId) + if err != nil && err != pg.ErrNoRows { + impl.Logger.Errorw("Error in GetLatestArtifactTimeByCiPipelineId", "err", err, "pipelineId", ciTriggerRequest.PipelineId) + return 0, err + } + + createdOn := time.Time{} + if err != pg.ErrNoRows { + createdOn = ciArtifact.CreatedOn + } + + trigger := types.Trigger{ + PipelineId: ciTriggerRequest.PipelineId, + CommitHashes: commitHashes, + CiMaterials: nil, + TriggeredBy: ciTriggerRequest.TriggeredBy, + InvalidateCache: ciTriggerRequest.InvalidateCache, + RuntimeParameters: runtimeParams, + EnvironmentId: ciTriggerRequest.EnvironmentId, + PipelineType: ciTriggerRequest.PipelineType, + CiArtifactLastFetch: createdOn, + } + id, err := impl.TriggerCiPipeline(trigger) + + if err != nil { + return 0, err + } + return id, nil +} + +func (impl *ServiceImpl) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) { + impl.Logger.Debugw("HandleCIWebhook for material ", "material", gitCiTriggerRequest.CiPipelineMaterial) + ciPipeline, err := impl.GetCiPipeline(gitCiTriggerRequest.CiPipelineMaterial.Id) + if err != nil { + impl.Logger.Errorw("err in getting ci_pipeline by ciPipelineMaterialId", "ciPipelineMaterialId", gitCiTriggerRequest.CiPipelineMaterial.Id, "err", err) + return 0, err + } + if ciPipeline.IsManual || ciPipeline.PipelineType == buildCommonBean.LINKED_CD.ToString() { + impl.Logger.Debugw("not handling for manual pipeline or in case of linked cd", "pipelineId", ciPipeline.Id) + return 0, err + } + + ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(ciPipeline.Id) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return 0, err + } + isValidBuildSequence, err := impl.validateBuildSequence(gitCiTriggerRequest, ciPipeline.Id) + if !isValidBuildSequence { + return 0, errors.New("ignoring older build for ciMaterial " + strconv.Itoa(gitCiTriggerRequest.CiPipelineMaterial.Id) + + " commit " + gitCiTriggerRequest.CiPipelineMaterial.GitCommit.Commit) + } + // updating runtime params + runtimeParams := common.NewRuntimeParameters() + for k, v := range gitCiTriggerRequest.ExtraEnvironmentVariables { + runtimeParams = runtimeParams.AddSystemVariable(k, v) + } + runtimeParams, err = impl.updateRuntimeParamsForAutoCI(ciPipeline.Id, runtimeParams) + if err != nil { + impl.Logger.Errorw("err, updateRuntimeParamsForAutoCI", "ciPipelineId", ciPipeline.Id, + "runtimeParameters", runtimeParams, "err", err) + return 0, err + } + commitHashes, err := impl.buildAutomaticTriggerCommitHashes(ciMaterials, gitCiTriggerRequest) + if err != nil { + return 0, err + } + + trigger := types.Trigger{ + PipelineId: ciPipeline.Id, + CommitHashes: commitHashes, + CiMaterials: ciMaterials, + TriggeredBy: gitCiTriggerRequest.TriggeredBy, + RuntimeParameters: runtimeParams, + } + id, err := impl.TriggerCiPipeline(trigger) + if err != nil { + return 0, err + } + return id, nil +} + +func (impl *ServiceImpl) extractPodStatusAndWorkflow(workflowStatus v1alpha1.WorkflowStatus) (string, string, *pipelineConfig.CiWorkflow, error) { + workflowName, status, _, message, _, _ := pipeline2.ExtractWorkflowStatus(workflowStatus) + if workflowName == "" { + impl.Logger.Errorw("extract workflow status, invalid wf name", "workflowName", workflowName, "status", status, "message", message) + return status, message, nil, errors.New("invalid wf name") + } + workflowId, err := strconv.Atoi(workflowName[:strings.Index(workflowName, "-")]) + if err != nil { + impl.Logger.Errorw("extract workflowId, invalid wf name", "workflowName", workflowName, "err", err) + return status, message, nil, err + } + + savedWorkflow, err := impl.ciWorkflowRepository.FindById(workflowId) + if err != nil { + impl.Logger.Errorw("cannot get saved wf", "workflowId", workflowId, "err", err) + return status, message, nil, err + } + + return status, message, savedWorkflow, err + +} + +func (impl *ServiceImpl) getRefWorkflowAndCiRetryCount(savedWorkflow *pipelineConfig.CiWorkflow) (int, *pipelineConfig.CiWorkflow, error) { + var err error + + if savedWorkflow.ReferenceCiWorkflowId != 0 { + savedWorkflow, err = impl.ciWorkflowRepository.FindById(savedWorkflow.ReferenceCiWorkflowId) + } + if err != nil { + impl.Logger.Errorw("cannot get saved wf", "err", err) + return 0, savedWorkflow, err + } + retryCount, err := impl.ciWorkflowRepository.FindRetriedWorkflowCountByReferenceId(savedWorkflow.Id) + return retryCount, savedWorkflow, err +} + +func (impl *ServiceImpl) validateBuildSequence(gitCiTriggerRequest bean.GitCiTriggerRequest, pipelineId int) (bool, error) { + isValid := true + lastTriggeredBuild, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflow(pipelineId) + if !(lastTriggeredBuild.Status == string(v1alpha1.NodePending) || lastTriggeredBuild.Status == string(v1alpha1.NodeRunning)) { + return true, nil + } + if err != nil && !util.IsErrNoRows(err) { + impl.Logger.Errorw("cannot get last build for pipeline", "pipelineId", pipelineId) + return false, err + } + + ciPipelineMaterial := gitCiTriggerRequest.CiPipelineMaterial + + if ciPipelineMaterial.Type == string(constants.SOURCE_TYPE_BRANCH_FIXED) { + if ciPipelineMaterial.GitCommit.Date.Before(lastTriggeredBuild.GitTriggers[ciPipelineMaterial.Id].Date) { + impl.Logger.Warnw("older commit cannot be built for pipeline", "pipelineId", pipelineId, "ciMaterial", gitCiTriggerRequest.CiPipelineMaterial.Id) + isValid = false + } + } + + return isValid, nil +} + +func (impl *ServiceImpl) buildAutomaticTriggerCommitHashes(ciMaterials []*pipelineConfig.CiPipelineMaterial, request bean.GitCiTriggerRequest) (map[int]pipelineConfig.GitCommit, error) { + commitHashes := map[int]pipelineConfig.GitCommit{} + for _, ciMaterial := range ciMaterials { + if ciMaterial.Id == request.CiPipelineMaterial.Id || len(ciMaterials) == 1 { + request.CiPipelineMaterial.GitCommit = SetGitCommitValuesForBuildingCommitHash(ciMaterial, request.CiPipelineMaterial.GitCommit) + commitHashes[ciMaterial.Id] = request.CiPipelineMaterial.GitCommit + } else { + // this is possible in case of non Webhook, as there would be only one pipeline material per git material in case of PR + lastCommit, err := impl.getLastSeenCommit(ciMaterial.Id) + if err != nil { + return map[int]pipelineConfig.GitCommit{}, err + } + lastCommit = SetGitCommitValuesForBuildingCommitHash(ciMaterial, lastCommit) + commitHashes[ciMaterial.Id] = lastCommit + } + } + return commitHashes, nil +} + +func (impl *ServiceImpl) GetCiPipeline(ciMaterialId int) (*pipelineConfig.CiPipeline, error) { + ciMaterial, err := impl.ciPipelineMaterialRepository.GetById(ciMaterialId) + if err != nil { + return nil, err + } + ciPipeline := ciMaterial.CiPipeline + return ciPipeline, nil +} + +func (impl *ServiceImpl) buildManualTriggerCommitHashes(ciTriggerRequest bean.CiTriggerRequest) (map[int]pipelineConfig.GitCommit, *common.RuntimeParameters, error) { + commitHashes := map[int]pipelineConfig.GitCommit{} + runtimeParams := impl.getRuntimeParamsForBuildingManualTriggerHashes(ciTriggerRequest) + for _, ciPipelineMaterial := range ciTriggerRequest.CiPipelineMaterial { + + pipeLineMaterialFromDb, err := impl.ciPipelineMaterialRepository.GetById(ciPipelineMaterial.Id) + if err != nil { + impl.Logger.Errorw("err in fetching pipeline material by id", "err", err) + return map[int]pipelineConfig.GitCommit{}, nil, err + } + + pipelineType := pipeLineMaterialFromDb.Type + if pipelineType == constants.SOURCE_TYPE_BRANCH_FIXED { + gitCommit, err := impl.BuildManualTriggerCommitHashesForSourceTypeBranchFix(ciPipelineMaterial, pipeLineMaterialFromDb) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return map[int]pipelineConfig.GitCommit{}, nil, err + } + commitHashes[ciPipelineMaterial.Id] = gitCommit + + } else if pipelineType == constants.SOURCE_TYPE_WEBHOOK { + gitCommit, extraEnvVariables, err := impl.BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial, pipeLineMaterialFromDb) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return map[int]pipelineConfig.GitCommit{}, nil, err + } + commitHashes[ciPipelineMaterial.Id] = gitCommit + for key, value := range extraEnvVariables { + runtimeParams = runtimeParams.AddSystemVariable(key, value) + } + } + } + return commitHashes, runtimeParams, nil +} + +func (impl *ServiceImpl) BuildManualTriggerCommitHashesForSourceTypeBranchFix(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (pipelineConfig.GitCommit, error) { + commitMetadataRequest := &gitSensor.CommitMetadataRequest{ + PipelineMaterialId: ciPipelineMaterial.Id, + GitHash: ciPipelineMaterial.GitCommit.Commit, + GitTag: ciPipelineMaterial.GitTag, + } + gitCommitResponse, err := impl.gitSensorClient.GetCommitMetadataForPipelineMaterial(context.Background(), commitMetadataRequest) + if err != nil { + impl.Logger.Errorw("err in fetching commit metadata", "commitMetadataRequest", commitMetadataRequest, "err", err) + return pipelineConfig.GitCommit{}, err + } + if gitCommitResponse == nil { + return pipelineConfig.GitCommit{}, errors.New("commit not found") + } + + gitCommit := pipelineConfig.GitCommit{ + Commit: gitCommitResponse.Commit, + Author: gitCommitResponse.Author, + Date: gitCommitResponse.Date, + Message: gitCommitResponse.Message, + Changes: gitCommitResponse.Changes, + GitRepoName: pipeLineMaterialFromDb.GitMaterial.Name[strings.Index(pipeLineMaterialFromDb.GitMaterial.Name, "-")+1:], + GitRepoUrl: pipeLineMaterialFromDb.GitMaterial.Url, + CiConfigureSourceValue: pipeLineMaterialFromDb.Value, + CiConfigureSourceType: pipeLineMaterialFromDb.Type, + } + + return gitCommit, nil +} + +func (impl *ServiceImpl) BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (pipelineConfig.GitCommit, map[string]string, error) { + webhookDataInput := ciPipelineMaterial.GitCommit.WebhookData + + // fetch webhook data on the basis of Id + webhookDataRequest := &gitSensor.WebhookDataRequest{ + Id: webhookDataInput.Id, + CiPipelineMaterialId: ciPipelineMaterial.Id, + } + + webhookAndCiData, err := impl.gitSensorClient.GetWebhookData(context.Background(), webhookDataRequest) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return pipelineConfig.GitCommit{}, nil, err + } + webhookData := webhookAndCiData.WebhookData + + // if webhook event is of merged type, then fetch latest commit for target branch + if webhookData.EventActionType == bean.WEBHOOK_EVENT_MERGED_ACTION_TYPE { + + // get target branch name from webhook + targetBranchName := webhookData.Data[bean.WEBHOOK_SELECTOR_TARGET_BRANCH_NAME_NAME] + if targetBranchName == "" { + impl.Logger.Error("target branch not found from webhook data") + return pipelineConfig.GitCommit{}, nil, err + } + + // get latest commit hash for target branch + latestCommitMetadataRequest := &gitSensor.CommitMetadataRequest{ + PipelineMaterialId: ciPipelineMaterial.Id, + BranchName: targetBranchName, + } + + latestCommit, err := impl.gitSensorClient.GetCommitMetadata(context.Background(), latestCommitMetadataRequest) + + if err != nil { + impl.Logger.Errorw("err", "err", err) + return pipelineConfig.GitCommit{}, nil, err + } + + // update webhookData (local) with target latest hash + webhookData.Data[bean.WEBHOOK_SELECTOR_TARGET_CHECKOUT_NAME] = latestCommit.Commit + + } + + // build git commit + gitCommit := pipelineConfig.GitCommit{ + GitRepoName: pipeLineMaterialFromDb.GitMaterial.Name[strings.Index(pipeLineMaterialFromDb.GitMaterial.Name, "-")+1:], + GitRepoUrl: pipeLineMaterialFromDb.GitMaterial.Url, + CiConfigureSourceValue: pipeLineMaterialFromDb.Value, + CiConfigureSourceType: pipeLineMaterialFromDb.Type, + WebhookData: pipelineConfig.WebhookData{ + Id: int(webhookData.Id), + EventActionType: webhookData.EventActionType, + Data: webhookData.Data, + }, + } + + return gitCommit, webhookAndCiData.ExtraEnvironmentVariables, nil +} + +func (impl *ServiceImpl) getLastSeenCommit(ciMaterialId int) (pipelineConfig.GitCommit, error) { + var materialIds []int + materialIds = append(materialIds, ciMaterialId) + headReq := &gitSensor.HeadRequest{ + MaterialIds: materialIds, + } + res, err := impl.gitSensorClient.GetHeadForPipelineMaterials(context.Background(), headReq) + if err != nil { + return pipelineConfig.GitCommit{}, err + } + if len(res) == 0 { + return pipelineConfig.GitCommit{}, errors.New("received empty response") + } + gitCommit := pipelineConfig.GitCommit{ + Commit: res[0].GitCommit.Commit, + Author: res[0].GitCommit.Author, + Date: res[0].GitCommit.Date, + Message: res[0].GitCommit.Message, + Changes: res[0].GitCommit.Changes, + } + return gitCommit, nil +} + +func SetGitCommitValuesForBuildingCommitHash(ciMaterial *pipelineConfig.CiPipelineMaterial, oldGitCommit pipelineConfig.GitCommit) pipelineConfig.GitCommit { + newGitCommit := oldGitCommit + newGitCommit.CiConfigureSourceType = ciMaterial.Type + newGitCommit.CiConfigureSourceValue = ciMaterial.Value + newGitCommit.GitRepoUrl = ciMaterial.GitMaterial.Url + newGitCommit.GitRepoName = ciMaterial.GitMaterial.Name[strings.Index(ciMaterial.GitMaterial.Name, "-")+1:] + return newGitCommit +} + func (impl *ServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { impl.Logger.Debug("ci pipeline manual trigger") ciMaterials, err := impl.GetCiMaterials(trigger.PipelineId, trigger.CiMaterials) diff --git a/pkg/build/trigger/Service_ent.go b/pkg/build/trigger/Service_ent.go index b20094eca7..cd22daa921 100644 --- a/pkg/build/trigger/Service_ent.go +++ b/pkg/build/trigger/Service_ent.go @@ -2,11 +2,21 @@ package trigger import ( "github.com/devtron-labs/common-lib/imageScan/bean" + bean3 "github.com/devtron-labs/devtron/pkg/bean" + "github.com/devtron-labs/devtron/pkg/bean/common" bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" ) +func (impl *ServiceImpl) updateRuntimeParamsForAutoCI(ciPipelineId int, runtimeParameters *common.RuntimeParameters) (*common.RuntimeParameters, error) { + return runtimeParameters, nil +} + +func (impl *ServiceImpl) getRuntimeParamsForBuildingManualTriggerHashes(ciTriggerRequest bean3.CiTriggerRequest) *common.RuntimeParameters { + return common.NewRuntimeParameters() +} + func (impl *ServiceImpl) fetchImageScanExecutionMedium() (*repository.ScanToolMetadata, bean.ScanExecutionMedium, error) { return &repository.ScanToolMetadata{}, "", nil } diff --git a/pkg/bulkAction/service/BulkUpdateService.go b/pkg/bulkAction/service/BulkUpdateService.go index 3b94648b69..afd8efeb41 100644 --- a/pkg/bulkAction/service/BulkUpdateService.go +++ b/pkg/bulkAction/service/BulkUpdateService.go @@ -35,6 +35,7 @@ import ( "github.com/devtron-labs/devtron/internal/util" appWorkflow2 "github.com/devtron-labs/devtron/pkg/appWorkflow" bean2 "github.com/devtron-labs/devtron/pkg/bean" + "github.com/devtron-labs/devtron/pkg/build/trigger" bean4 "github.com/devtron-labs/devtron/pkg/bulkAction/bean" "github.com/devtron-labs/devtron/pkg/bulkAction/utils" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" @@ -102,6 +103,7 @@ type BulkUpdateServiceImpl struct { chartRefService chartRef.ChartRefService deployedAppService deployedApp.DeployedAppService cdPipelineEventPublishService out.CDPipelineEventPublishService + ciTriggerService trigger.Service } func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateRepository, @@ -121,7 +123,8 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateReposito deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, chartRefService chartRef.ChartRefService, deployedAppService deployedApp.DeployedAppService, - cdPipelineEventPublishService out.CDPipelineEventPublishService) *BulkUpdateServiceImpl { + cdPipelineEventPublishService out.CDPipelineEventPublishService, + ciTriggerService trigger.Service) *BulkUpdateServiceImpl { return &BulkUpdateServiceImpl{ bulkUpdateRepository: bulkUpdateRepository, logger: logger, @@ -141,6 +144,7 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateReposito chartRefService: chartRefService, deployedAppService: deployedAppService, cdPipelineEventPublishService: cdPipelineEventPublishService, + ciTriggerService: ciTriggerService, } } @@ -1467,7 +1471,7 @@ func (impl BulkUpdateServiceImpl) BulkBuildTrigger(request *bean4.BulkApplicatio } ciTriggerRequest := latestCommitsMap[pipeline.CiPipelineId] - _, err = impl.ciHandler.HandleCIManual(ciTriggerRequest) + _, err = impl.ciTriggerService.HandleCIManual(ciTriggerRequest) if err != nil { impl.logger.Errorw("service err, HandleCIManual", "err", err, "ciTriggerRequest", ciTriggerRequest) //return nil, err diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index 2800e9a3bb..5e97d5d99a 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -35,6 +35,7 @@ import ( util3 "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" userBean "github.com/devtron-labs/devtron/pkg/auth/user/bean" + "github.com/devtron-labs/devtron/pkg/build/trigger" "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/deployedApp" deploymentBean "github.com/devtron-labs/devtron/pkg/deployment/deployedApp/bean" @@ -91,6 +92,9 @@ type WorkflowEventProcessorImpl struct { devtronAppReleaseContextMapLock *sync.Mutex appServiceConfig *app.AppServiceConfig + //ent only + ciTriggerService trigger.Service + // repositories import to be removed pipelineRepository pipelineConfig.PipelineRepository ciArtifactRepository repository.CiArtifactRepository @@ -118,7 +122,8 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, ciArtifactRepository repository.CiArtifactRepository, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, - deploymentConfigService common.DeploymentConfigService) (*WorkflowEventProcessorImpl, error) { + deploymentConfigService common.DeploymentConfigService, + ciTriggerService trigger.Service) (*WorkflowEventProcessorImpl, error) { impl := &WorkflowEventProcessorImpl{ logger: logger, pubSubClient: pubSubClient, @@ -145,6 +150,7 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, ciArtifactRepository: ciArtifactRepository, cdWorkflowRepository: cdWorkflowRepository, deploymentConfigService: deploymentConfigService, + ciTriggerService: ciTriggerService, } appServiceConfig, err := app.GetAppServiceConfig() if err != nil { @@ -394,7 +400,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCIWorkflowStatusUpdate() error return } - err = impl.ciHandler.CheckAndReTriggerCI(wfStatus) + err = impl.ciTriggerService.CheckAndReTriggerCI(wfStatus) if err != nil { impl.logger.Errorw("error in checking and re triggering ci", "err", err) //don't return as we have to update the workflow status diff --git a/pkg/pipeline/CIHandler_ent.go b/pkg/pipeline/CIHandler_ent.go index 7e1986b518..4d41cf80a3 100644 --- a/pkg/pipeline/CIHandler_ent.go +++ b/pkg/pipeline/CIHandler_ent.go @@ -2,25 +2,15 @@ package pipeline import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/pkg/bean" - "github.com/devtron-labs/devtron/pkg/bean/common" ) type CiHandlerEnt interface { } -func (impl *CiHandlerImpl) updateRuntimeParamsForAutoCI(ciPipelineId int, runtimeParameters *common.RuntimeParameters) (*common.RuntimeParameters, error) { - return runtimeParameters, nil -} - func (impl *CiHandlerImpl) updateResourceStatusInCache(ciWorkflowId int, podName string, namespace string, status string) { //do nothing } -func (impl *CiHandlerImpl) getRuntimeParamsForBuildingManualTriggerHashes(ciTriggerRequest bean.CiTriggerRequest) *common.RuntimeParameters { - return common.NewRuntimeParameters() -} - func (impl *CiHandlerImpl) getPipelineIdForTriggerView(pipeline *pipelineConfig.CiPipeline) (pipelineId int) { if pipeline.ParentCiPipeline == 0 { pipelineId = pipeline.Id diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index aa018f6b96..20d18c93eb 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -23,13 +23,9 @@ import ( "fmt" "github.com/devtron-labs/common-lib/utils" "github.com/devtron-labs/common-lib/utils/workFlow" - constants2 "github.com/devtron-labs/devtron/internal/sql/constants" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" - bean6 "github.com/devtron-labs/devtron/pkg/auth/user/bean" - "github.com/devtron-labs/devtron/pkg/bean/common" "github.com/devtron-labs/devtron/pkg/build/artifacts/imageTagging" buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" - buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" "github.com/devtron-labs/devtron/pkg/cluster/adapter" clusterBean "github.com/devtron-labs/devtron/pkg/cluster/bean" "github.com/devtron-labs/devtron/pkg/cluster/environment" @@ -59,7 +55,6 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/util/rbac" - errors2 "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/rest" "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" @@ -73,9 +68,9 @@ import ( ) type CiHandler interface { - HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) - HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) - CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error + // HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) + //HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) + //CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error FetchMaterialsByPipelineId(pipelineId int, showAll bool) ([]buildBean.CiPipelineMaterialResponse, error) FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId int, gitMaterialId int, showAll bool) ([]buildBean.CiPipelineMaterialResponse, error) FetchWorkflowDetails(appId int, pipelineId int, buildId int) (types.WorkflowResponse, error) @@ -93,7 +88,7 @@ type CiHandler interface { FetchCiStatusForTriggerViewV1(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) RefreshMaterialByCiPipelineMaterialId(gitMaterialId int) (refreshRes *gitSensor.RefreshGitMaterialResponse, err error) FetchMaterialInfoByArtifactId(ciArtifactId int, envId int) (*types.GitTriggerInfoResponse, error) - UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error + //UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error FetchCiStatusForTriggerViewForEnvironment(request resourceGroup.ResourceGroupingRequest, token string) ([]*pipelineConfig.CiWorkflowStatus, error) CiHandlerEnt } @@ -172,187 +167,6 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline return cih } -func (impl *CiHandlerImpl) CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error { - - // return if re-trigger feature is disabled - if !impl.config.WorkflowRetriesEnabled() { - impl.Logger.Debug("CI re-trigger is disabled") - return nil - } - - status, message, ciWorkFlow, err := impl.extractPodStatusAndWorkflow(workflowStatus) - if err != nil { - impl.Logger.Errorw("error in extractPodStatusAndWorkflow", "err", err) - return err - } - - if !executors.CheckIfReTriggerRequired(status, message, ciWorkFlow.Status) { - impl.Logger.Debugw("not re-triggering ci", "status", status, "message", message, "ciWorkflowStatus", ciWorkFlow.Status) - return nil - } - - impl.Logger.Debugw("re-triggering ci", "status", status, "message", message, "ciWorkflowStatus", ciWorkFlow.Status, "ciWorkFlowId", ciWorkFlow.Id) - - retryCount, refCiWorkflow, err := impl.getRefWorkflowAndCiRetryCount(ciWorkFlow) - if err != nil { - impl.Logger.Errorw("error while getting retry count value for a ciWorkflow", "ciWorkFlowId", ciWorkFlow.Id) - return err - } - - err = impl.reTriggerCi(retryCount, refCiWorkflow) - if err != nil { - impl.Logger.Errorw("error in reTriggerCi", "err", err, "status", status, "message", message, "retryCount", retryCount, "ciWorkFlowId", ciWorkFlow.Id) - } - return err -} - -func (impl *CiHandlerImpl) reTriggerCi(retryCount int, refCiWorkflow *pipelineConfig.CiWorkflow) error { - if retryCount >= impl.config.MaxCiWorkflowRetries { - impl.Logger.Infow("maximum retries exhausted for this ciWorkflow", "ciWorkflowId", refCiWorkflow.Id, "retries", retryCount, "configuredRetries", impl.config.MaxCiWorkflowRetries) - return nil - } - impl.Logger.Infow("re-triggering ci for a ci workflow", "ReferenceCiWorkflowId", refCiWorkflow.Id) - ciPipelineMaterialIds := make([]int, 0, len(refCiWorkflow.GitTriggers)) - for id, _ := range refCiWorkflow.GitTriggers { - ciPipelineMaterialIds = append(ciPipelineMaterialIds, id) - } - ciMaterials, err := impl.ciPipelineMaterialRepository.GetByIdsIncludeDeleted(ciPipelineMaterialIds) - if err != nil { - impl.Logger.Errorw("error in getting ci Pipeline Materials using ciPipeline Material Ids", "ciPipelineMaterialIds", ciPipelineMaterialIds, "err", err) - return err - } - - trigger := types.Trigger{} - trigger.BuildTriggerObject(refCiWorkflow, ciMaterials, bean6.SYSTEM_USER_ID, true, nil, "") - - // updating runtime params - trigger.RuntimeParameters, err = impl.updateRuntimeParamsForAutoCI(trigger.PipelineId, trigger.RuntimeParameters) - if err != nil { - impl.Logger.Errorw("err, updateRuntimeParamsForAutoCI", "ciPipelineId", trigger.PipelineId, - "runtimeParameters", trigger.RuntimeParameters, "err", err) - return err - } - _, err = impl.ciService.TriggerCiPipeline(trigger) - - if err != nil { - impl.Logger.Errorw("error occurred in re-triggering ciWorkflow", "triggerDetails", trigger, "err", err) - return err - } - return nil -} - -func (impl *CiHandlerImpl) HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) { - impl.Logger.Debugw("HandleCIManual for pipeline ", "PipelineId", ciTriggerRequest.PipelineId) - commitHashes, runtimeParams, err := impl.buildManualTriggerCommitHashes(ciTriggerRequest) - if err != nil { - return 0, err - } - - ciArtifact, err := impl.ciArtifactRepository.GetLatestArtifactTimeByCiPipelineId(ciTriggerRequest.PipelineId) - if err != nil && err != pg.ErrNoRows { - impl.Logger.Errorw("Error in GetLatestArtifactTimeByCiPipelineId", "err", err, "pipelineId", ciTriggerRequest.PipelineId) - return 0, err - } - - createdOn := time.Time{} - if err != pg.ErrNoRows { - createdOn = ciArtifact.CreatedOn - } - - trigger := types.Trigger{ - PipelineId: ciTriggerRequest.PipelineId, - CommitHashes: commitHashes, - CiMaterials: nil, - TriggeredBy: ciTriggerRequest.TriggeredBy, - InvalidateCache: ciTriggerRequest.InvalidateCache, - RuntimeParameters: runtimeParams, - EnvironmentId: ciTriggerRequest.EnvironmentId, - PipelineType: ciTriggerRequest.PipelineType, - CiArtifactLastFetch: createdOn, - } - id, err := impl.ciService.TriggerCiPipeline(trigger) - - if err != nil { - return 0, err - } - return id, nil -} - -func (impl *CiHandlerImpl) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) { - impl.Logger.Debugw("HandleCIWebhook for material ", "material", gitCiTriggerRequest.CiPipelineMaterial) - ciPipeline, err := impl.GetCiPipeline(gitCiTriggerRequest.CiPipelineMaterial.Id) - if err != nil { - impl.Logger.Errorw("err in getting ci_pipeline by ciPipelineMaterialId", "ciPipelineMaterialId", gitCiTriggerRequest.CiPipelineMaterial.Id, "err", err) - return 0, err - } - if ciPipeline.IsManual || ciPipeline.PipelineType == buildCommonBean.LINKED_CD.ToString() { - impl.Logger.Debugw("not handling for manual pipeline or in case of linked cd", "pipelineId", ciPipeline.Id) - return 0, err - } - - ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(ciPipeline.Id) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return 0, err - } - isValidBuildSequence, err := impl.validateBuildSequence(gitCiTriggerRequest, ciPipeline.Id) - if !isValidBuildSequence { - return 0, errors.New("ignoring older build for ciMaterial " + strconv.Itoa(gitCiTriggerRequest.CiPipelineMaterial.Id) + - " commit " + gitCiTriggerRequest.CiPipelineMaterial.GitCommit.Commit) - } - // updating runtime params - runtimeParams := common.NewRuntimeParameters() - for k, v := range gitCiTriggerRequest.ExtraEnvironmentVariables { - runtimeParams = runtimeParams.AddSystemVariable(k, v) - } - runtimeParams, err = impl.updateRuntimeParamsForAutoCI(ciPipeline.Id, runtimeParams) - if err != nil { - impl.Logger.Errorw("err, updateRuntimeParamsForAutoCI", "ciPipelineId", ciPipeline.Id, - "runtimeParameters", runtimeParams, "err", err) - return 0, err - } - commitHashes, err := impl.buildAutomaticTriggerCommitHashes(ciMaterials, gitCiTriggerRequest) - if err != nil { - return 0, err - } - - trigger := types.Trigger{ - PipelineId: ciPipeline.Id, - CommitHashes: commitHashes, - CiMaterials: ciMaterials, - TriggeredBy: gitCiTriggerRequest.TriggeredBy, - RuntimeParameters: runtimeParams, - } - id, err := impl.ciService.TriggerCiPipeline(trigger) - if err != nil { - return 0, err - } - return id, nil -} - -func (impl *CiHandlerImpl) validateBuildSequence(gitCiTriggerRequest bean.GitCiTriggerRequest, pipelineId int) (bool, error) { - isValid := true - lastTriggeredBuild, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflow(pipelineId) - if !(lastTriggeredBuild.Status == string(v1alpha1.NodePending) || lastTriggeredBuild.Status == string(v1alpha1.NodeRunning)) { - return true, nil - } - if err != nil && !util.IsErrNoRows(err) { - impl.Logger.Errorw("cannot get last build for pipeline", "pipelineId", pipelineId) - return false, err - } - - ciPipelineMaterial := gitCiTriggerRequest.CiPipelineMaterial - - if ciPipelineMaterial.Type == string(constants2.SOURCE_TYPE_BRANCH_FIXED) { - if ciPipelineMaterial.GitCommit.Date.Before(lastTriggeredBuild.GitTriggers[ciPipelineMaterial.Id].Date) { - impl.Logger.Warnw("older commit cannot be built for pipeline", "pipelineId", pipelineId, "ciMaterial", gitCiTriggerRequest.CiPipelineMaterial.Id) - isValid = false - } - } - - return isValid, nil -} - func (impl *CiHandlerImpl) RefreshMaterialByCiPipelineMaterialId(gitMaterialId int) (refreshRes *gitSensor.RefreshGitMaterialResponse, err error) { impl.Logger.Debugw("refreshing git material", "id", gitMaterialId) refreshRes, err = impl.gitSensorClient.RefreshGitMaterial(context.Background(), @@ -1157,42 +971,6 @@ func ExtractWorkflowStatus(workflowStatus v1alpha1.WorkflowStatus) (string, stri return workflowName, status, podStatus, message, logLocation, podName } -func (impl *CiHandlerImpl) extractPodStatusAndWorkflow(workflowStatus v1alpha1.WorkflowStatus) (string, string, *pipelineConfig.CiWorkflow, error) { - workflowName, status, _, message, _, _ := ExtractWorkflowStatus(workflowStatus) - if workflowName == "" { - impl.Logger.Errorw("extract workflow status, invalid wf name", "workflowName", workflowName, "status", status, "message", message) - return status, message, nil, errors.New("invalid wf name") - } - workflowId, err := strconv.Atoi(workflowName[:strings.Index(workflowName, "-")]) - if err != nil { - impl.Logger.Errorw("extract workflowId, invalid wf name", "workflowName", workflowName, "err", err) - return status, message, nil, err - } - - savedWorkflow, err := impl.ciWorkflowRepository.FindById(workflowId) - if err != nil { - impl.Logger.Errorw("cannot get saved wf", "workflowId", workflowId, "err", err) - return status, message, nil, err - } - - return status, message, savedWorkflow, err - -} - -func (impl *CiHandlerImpl) getRefWorkflowAndCiRetryCount(savedWorkflow *pipelineConfig.CiWorkflow) (int, *pipelineConfig.CiWorkflow, error) { - var err error - - if savedWorkflow.ReferenceCiWorkflowId != 0 { - savedWorkflow, err = impl.ciWorkflowRepository.FindById(savedWorkflow.ReferenceCiWorkflowId) - } - if err != nil { - impl.Logger.Errorw("cannot get saved wf", "err", err) - return 0, savedWorkflow, err - } - retryCount, err := impl.ciWorkflowRepository.FindRetriedWorkflowCountByReferenceId(savedWorkflow.Id) - return retryCount, savedWorkflow, err -} - func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, error) { workflowName, status, podStatus, message, _, podName := ExtractWorkflowStatus(workflowStatus) if workflowName == "" { @@ -1285,191 +1063,6 @@ func (impl *CiHandlerImpl) stateChanged(status string, podStatus string, msg str return savedWorkflow.Status != status || savedWorkflow.PodStatus != podStatus || savedWorkflow.Message != msg || savedWorkflow.FinishedOn != finishedAt } -func (impl *CiHandlerImpl) GetCiPipeline(ciMaterialId int) (*pipelineConfig.CiPipeline, error) { - ciMaterial, err := impl.ciPipelineMaterialRepository.GetById(ciMaterialId) - if err != nil { - return nil, err - } - ciPipeline := ciMaterial.CiPipeline - return ciPipeline, nil -} - -func (impl *CiHandlerImpl) buildAutomaticTriggerCommitHashes(ciMaterials []*pipelineConfig.CiPipelineMaterial, request bean.GitCiTriggerRequest) (map[int]pipelineConfig.GitCommit, error) { - commitHashes := map[int]pipelineConfig.GitCommit{} - for _, ciMaterial := range ciMaterials { - if ciMaterial.Id == request.CiPipelineMaterial.Id || len(ciMaterials) == 1 { - request.CiPipelineMaterial.GitCommit = SetGitCommitValuesForBuildingCommitHash(ciMaterial, request.CiPipelineMaterial.GitCommit) - commitHashes[ciMaterial.Id] = request.CiPipelineMaterial.GitCommit - } else { - // this is possible in case of non Webhook, as there would be only one pipeline material per git material in case of PR - lastCommit, err := impl.getLastSeenCommit(ciMaterial.Id) - if err != nil { - return map[int]pipelineConfig.GitCommit{}, err - } - lastCommit = SetGitCommitValuesForBuildingCommitHash(ciMaterial, lastCommit) - commitHashes[ciMaterial.Id] = lastCommit - } - } - return commitHashes, nil -} - -func SetGitCommitValuesForBuildingCommitHash(ciMaterial *pipelineConfig.CiPipelineMaterial, oldGitCommit pipelineConfig.GitCommit) pipelineConfig.GitCommit { - newGitCommit := oldGitCommit - newGitCommit.CiConfigureSourceType = ciMaterial.Type - newGitCommit.CiConfigureSourceValue = ciMaterial.Value - newGitCommit.GitRepoUrl = ciMaterial.GitMaterial.Url - newGitCommit.GitRepoName = ciMaterial.GitMaterial.Name[strings.Index(ciMaterial.GitMaterial.Name, "-")+1:] - return newGitCommit -} - -func (impl *CiHandlerImpl) buildManualTriggerCommitHashes(ciTriggerRequest bean.CiTriggerRequest) (map[int]pipelineConfig.GitCommit, *common.RuntimeParameters, error) { - commitHashes := map[int]pipelineConfig.GitCommit{} - runtimeParams := impl.getRuntimeParamsForBuildingManualTriggerHashes(ciTriggerRequest) - for _, ciPipelineMaterial := range ciTriggerRequest.CiPipelineMaterial { - - pipeLineMaterialFromDb, err := impl.ciPipelineMaterialRepository.GetById(ciPipelineMaterial.Id) - if err != nil { - impl.Logger.Errorw("err in fetching pipeline material by id", "err", err) - return map[int]pipelineConfig.GitCommit{}, nil, err - } - - pipelineType := pipeLineMaterialFromDb.Type - if pipelineType == constants2.SOURCE_TYPE_BRANCH_FIXED { - gitCommit, err := impl.BuildManualTriggerCommitHashesForSourceTypeBranchFix(ciPipelineMaterial, pipeLineMaterialFromDb) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return map[int]pipelineConfig.GitCommit{}, nil, err - } - commitHashes[ciPipelineMaterial.Id] = gitCommit - - } else if pipelineType == constants2.SOURCE_TYPE_WEBHOOK { - gitCommit, extraEnvVariables, err := impl.BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial, pipeLineMaterialFromDb) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return map[int]pipelineConfig.GitCommit{}, nil, err - } - commitHashes[ciPipelineMaterial.Id] = gitCommit - for key, value := range extraEnvVariables { - runtimeParams = runtimeParams.AddSystemVariable(key, value) - } - } - } - return commitHashes, runtimeParams, nil -} - -func (impl *CiHandlerImpl) BuildManualTriggerCommitHashesForSourceTypeBranchFix(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (pipelineConfig.GitCommit, error) { - commitMetadataRequest := &gitSensor.CommitMetadataRequest{ - PipelineMaterialId: ciPipelineMaterial.Id, - GitHash: ciPipelineMaterial.GitCommit.Commit, - GitTag: ciPipelineMaterial.GitTag, - } - gitCommitResponse, err := impl.gitSensorClient.GetCommitMetadataForPipelineMaterial(context.Background(), commitMetadataRequest) - if err != nil { - impl.Logger.Errorw("err in fetching commit metadata", "commitMetadataRequest", commitMetadataRequest, "err", err) - return pipelineConfig.GitCommit{}, err - } - if gitCommitResponse == nil { - return pipelineConfig.GitCommit{}, errors.New("commit not found") - } - - gitCommit := pipelineConfig.GitCommit{ - Commit: gitCommitResponse.Commit, - Author: gitCommitResponse.Author, - Date: gitCommitResponse.Date, - Message: gitCommitResponse.Message, - Changes: gitCommitResponse.Changes, - GitRepoName: pipeLineMaterialFromDb.GitMaterial.Name[strings.Index(pipeLineMaterialFromDb.GitMaterial.Name, "-")+1:], - GitRepoUrl: pipeLineMaterialFromDb.GitMaterial.Url, - CiConfigureSourceValue: pipeLineMaterialFromDb.Value, - CiConfigureSourceType: pipeLineMaterialFromDb.Type, - } - - return gitCommit, nil -} - -func (impl *CiHandlerImpl) BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (pipelineConfig.GitCommit, map[string]string, error) { - webhookDataInput := ciPipelineMaterial.GitCommit.WebhookData - - // fetch webhook data on the basis of Id - webhookDataRequest := &gitSensor.WebhookDataRequest{ - Id: webhookDataInput.Id, - CiPipelineMaterialId: ciPipelineMaterial.Id, - } - - webhookAndCiData, err := impl.gitSensorClient.GetWebhookData(context.Background(), webhookDataRequest) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return pipelineConfig.GitCommit{}, nil, err - } - webhookData := webhookAndCiData.WebhookData - - // if webhook event is of merged type, then fetch latest commit for target branch - if webhookData.EventActionType == bean.WEBHOOK_EVENT_MERGED_ACTION_TYPE { - - // get target branch name from webhook - targetBranchName := webhookData.Data[bean.WEBHOOK_SELECTOR_TARGET_BRANCH_NAME_NAME] - if targetBranchName == "" { - impl.Logger.Error("target branch not found from webhook data") - return pipelineConfig.GitCommit{}, nil, err - } - - // get latest commit hash for target branch - latestCommitMetadataRequest := &gitSensor.CommitMetadataRequest{ - PipelineMaterialId: ciPipelineMaterial.Id, - BranchName: targetBranchName, - } - - latestCommit, err := impl.gitSensorClient.GetCommitMetadata(context.Background(), latestCommitMetadataRequest) - - if err != nil { - impl.Logger.Errorw("err", "err", err) - return pipelineConfig.GitCommit{}, nil, err - } - - // update webhookData (local) with target latest hash - webhookData.Data[bean.WEBHOOK_SELECTOR_TARGET_CHECKOUT_NAME] = latestCommit.Commit - - } - - // build git commit - gitCommit := pipelineConfig.GitCommit{ - GitRepoName: pipeLineMaterialFromDb.GitMaterial.Name[strings.Index(pipeLineMaterialFromDb.GitMaterial.Name, "-")+1:], - GitRepoUrl: pipeLineMaterialFromDb.GitMaterial.Url, - CiConfigureSourceValue: pipeLineMaterialFromDb.Value, - CiConfigureSourceType: pipeLineMaterialFromDb.Type, - WebhookData: pipelineConfig.WebhookData{ - Id: int(webhookData.Id), - EventActionType: webhookData.EventActionType, - Data: webhookData.Data, - }, - } - - return gitCommit, webhookAndCiData.ExtraEnvironmentVariables, nil -} - -func (impl *CiHandlerImpl) getLastSeenCommit(ciMaterialId int) (pipelineConfig.GitCommit, error) { - var materialIds []int - materialIds = append(materialIds, ciMaterialId) - headReq := &gitSensor.HeadRequest{ - MaterialIds: materialIds, - } - res, err := impl.gitSensorClient.GetHeadForPipelineMaterials(context.Background(), headReq) - if err != nil { - return pipelineConfig.GitCommit{}, err - } - if len(res) == 0 { - return pipelineConfig.GitCommit{}, errors.New("received empty response") - } - gitCommit := pipelineConfig.GitCommit{ - Commit: res[0].GitCommit.Commit, - Author: res[0].GitCommit.Author, - Date: res[0].GitCommit.Date, - Message: res[0].GitCommit.Message, - Changes: res[0].GitCommit.Changes, - } - return gitCommit, nil -} - func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewV1(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) { ciWorkflowStatuses, err := impl.ciWorkflowRepository.FIndCiWorkflowStatusesByAppId(appId) if err != nil && !util.IsErrNoRows(err) { @@ -1626,127 +1219,6 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId return gitTriggerInfoResponse, nil } -func (impl *CiHandlerImpl) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error { - ciWorkflows, err := impl.ciWorkflowRepository.FindByStatusesIn([]string{constants.Starting, constants.Running}) - if err != nil { - impl.Logger.Errorw("error on fetching ci workflows", "err", err) - return err - } - client, err := impl.K8sUtil.GetClientForInCluster() - if err != nil { - impl.Logger.Errorw("error while fetching k8s client", "error", err) - return err - } - - for _, ciWorkflow := range ciWorkflows { - var isExt bool - var env *repository2.Environment - var restConfig *rest.Config - if ciWorkflow.Namespace != constants.DefaultCiWorkflowNamespace { - isExt = true - env, err = impl.envRepository.FindById(ciWorkflow.EnvironmentId) - if err != nil { - impl.Logger.Errorw("could not fetch stage env", "err", err) - return err - } - restConfig, err = impl.getRestConfig(ciWorkflow) - if err != nil { - return err - } - } - - isEligibleToMarkFailed := false - isPodDeleted := false - if time.Since(ciWorkflow.StartedOn) > (time.Minute * time.Duration(timeoutForFailureCiBuild)) { - - //check weather pod is exists or not, if exits check its status - wf, err := impl.workflowService.GetWorkflowStatus(ciWorkflow.ExecutorType, ciWorkflow.Name, ciWorkflow.Namespace, restConfig) - if err != nil { - impl.Logger.Warnw("unable to fetch ci workflow", "err", err) - statusError, ok := err.(*errors2.StatusError) - if ok && statusError.Status().Code == http.StatusNotFound { - impl.Logger.Warnw("ci workflow not found", "err", err) - isEligibleToMarkFailed = true - } else { - continue - // skip this and process for next ci workflow - } - } - - //if ci workflow is exists, check its pod - if !isEligibleToMarkFailed { - ns := constants.DefaultCiWorkflowNamespace - if isExt { - _, client, err = impl.k8sCommonService.GetCoreClientByClusterId(env.ClusterId) - if err != nil { - impl.Logger.Warnw("error in getting core v1 client using GetCoreClientByClusterId", "err", err, "clusterId", env.Cluster.Id) - continue - } - ns = env.Namespace - } - _, err := impl.K8sUtil.GetPodByName(ns, ciWorkflow.PodName, client) - if err != nil { - impl.Logger.Warnw("unable to fetch ci workflow - pod", "err", err) - statusError, ok := err.(*errors2.StatusError) - if ok && statusError.Status().Code == http.StatusNotFound { - impl.Logger.Warnw("pod not found", "err", err) - isEligibleToMarkFailed = true - } else { - continue - // skip this and process for next ci workflow - } - } - if ciWorkflow.ExecutorType == cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM { - if wf.Status == string(v1alpha1.WorkflowFailed) { - isPodDeleted = true - } - } else { - // check workflow status,get the status - if wf.Status == string(v1alpha1.WorkflowFailed) && wf.Message == cdWorkflow.POD_DELETED_MESSAGE { - isPodDeleted = true - } - } - } - } - if isEligibleToMarkFailed { - ciWorkflow.Status = "Failed" - ciWorkflow.PodStatus = "Failed" - if isPodDeleted { - ciWorkflow.Message = cdWorkflow.POD_DELETED_MESSAGE - // error logging handled inside handlePodDeleted - impl.handlePodDeleted(ciWorkflow) - } else { - ciWorkflow.Message = "marked failed by job" - } - err := impl.ciService.UpdateCiWorkflowWithStage(ciWorkflow) - if err != nil { - impl.Logger.Errorw("unable to update ci workflow, its eligible to mark failed", "err", err) - // skip this and process for next ci workflow - } - err = impl.customTagService.DeactivateImagePathReservation(ciWorkflow.ImagePathReservationId) - if err != nil { - impl.Logger.Errorw("unable to update ci workflow, its eligible to mark failed", "err", err) - } - } - } - return nil -} - -func (impl *CiHandlerImpl) handlePodDeleted(ciWorkflow *pipelineConfig.CiWorkflow) { - if !impl.config.WorkflowRetriesEnabled() { - impl.Logger.Debug("ci workflow retry feature disabled") - return - } - retryCount, refCiWorkflow, err := impl.getRefWorkflowAndCiRetryCount(ciWorkflow) - if err != nil { - impl.Logger.Errorw("error in getRefWorkflowAndCiRetryCount", "ciWorkflowId", ciWorkflow.Id, "err", err) - } - impl.Logger.Infow("re-triggering ci by UpdateCiWorkflowStatusFailedCron", "refCiWorkflowId", refCiWorkflow.Id, "ciWorkflow.Status", ciWorkflow.Status, "ciWorkflow.Message", ciWorkflow.Message, "retryCount", retryCount) - err = impl.reTriggerCi(retryCount, refCiWorkflow) - if err != nil { - impl.Logger.Errorw("error in reTriggerCi", "ciWorkflowId", refCiWorkflow.Id, "workflowStatus", ciWorkflow.Status, "ciWorkflowMessage", "ciWorkflow.Message", "retryCount", retryCount, "err", err) - } -} func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewForEnvironment(request resourceGroup.ResourceGroupingRequest, token string) ([]*pipelineConfig.CiWorkflowStatus, error) { ciWorkflowStatuses := make([]*pipelineConfig.CiWorkflowStatus, 0) var cdPipelines []*pipelineConfig.Pipeline diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index ce94737490..2a70ebe51e 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -24,6 +24,7 @@ import ( "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/devtron-labs/common-lib/async" "github.com/devtron-labs/common-lib/utils" + "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/common-lib/utils/workFlow" bean6 "github.com/devtron-labs/devtron/api/helm-app/bean" client2 "github.com/devtron-labs/devtron/api/helm-app/service" @@ -39,6 +40,9 @@ import ( "github.com/devtron-labs/devtron/pkg/build/artifacts" bean5 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" + "github.com/devtron-labs/devtron/pkg/build/trigger" + "github.com/devtron-labs/devtron/pkg/cluster/adapter" + repository5 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/manifest" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" @@ -46,7 +50,9 @@ import ( triggerBean "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/userDeploymentRequest/service" eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" + k8sPkg "github.com/devtron-labs/devtron/pkg/k8s" "github.com/devtron-labs/devtron/pkg/pipeline" + constants2 "github.com/devtron-labs/devtron/pkg/pipeline/constants" repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning" repository3 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/repository" @@ -58,6 +64,9 @@ import ( "github.com/devtron-labs/devtron/pkg/workflow/dag/helper" error2 "github.com/devtron-labs/devtron/util/error" util2 "github.com/devtron-labs/devtron/util/event" + errors2 "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/rest" + "net/http" "strings" "sync" "time" @@ -83,6 +92,7 @@ import ( ) type WorkflowDagExecutor interface { + UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error UpdateCiWorkflowForCiSuccess(request *bean2.CiArtifactWebhookRequest) (err error) HandleCiSuccessEvent(triggerContext triggerBean.TriggerContext, ciPipelineId int, request *bean2.CiArtifactWebhookRequest, imagePushedAt time.Time) (id int, err error) HandlePreStageSuccessEvent(triggerContext triggerBean.TriggerContext, cdStageCompleteEvent eventProcessorBean.CdStageCompleteEvent) error @@ -134,6 +144,12 @@ type WorkflowDagExecutorImpl struct { asyncRunnable *async.Runnable scanHistoryRepository repository3.ImageScanHistoryRepository imageScanService imageScanning.ImageScanService + + K8sUtil *k8s.K8sServiceImpl + envRepository repository5.EnvironmentRepository + k8sCommonService k8sPkg.K8sCommonService + workflowService pipeline.WorkflowService + ciTriggerService trigger.Service } func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, @@ -162,6 +178,11 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi asyncRunnable *async.Runnable, scanHistoryRepository repository3.ImageScanHistoryRepository, imageScanService imageScanning.ImageScanService, + K8sUtil *k8s.K8sServiceImpl, + envRepository repository5.EnvironmentRepository, + k8sCommonService k8sPkg.K8sCommonService, + workflowService pipeline.WorkflowService, + ciTriggerService trigger.Service, ) *WorkflowDagExecutorImpl { wde := &WorkflowDagExecutorImpl{logger: Logger, pipelineRepository: pipelineRepository, @@ -190,6 +211,11 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi imageScanService: imageScanService, cdWorkflowRunnerService: cdWorkflowRunnerService, ciService: ciService, + K8sUtil: K8sUtil, + envRepository: envRepository, + k8sCommonService: k8sCommonService, + workflowService: workflowService, + ciTriggerService: ciTriggerService, } config, err := types.GetCdConfig() if err != nil { @@ -209,6 +235,130 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi return wde } +func (impl *WorkflowDagExecutorImpl) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error { + ciWorkflows, err := impl.ciWorkflowRepository.FindByStatusesIn([]string{constants2.Starting, constants2.Running}) + if err != nil { + impl.logger.Errorw("error on fetching ci workflows", "err", err) + return err + } + client, err := impl.K8sUtil.GetClientForInCluster() + if err != nil { + impl.logger.Errorw("error while fetching k8s client", "error", err) + return err + } + + for _, ciWorkflow := range ciWorkflows { + var isExt bool + var env *repository5.Environment + var restConfig *rest.Config + if ciWorkflow.Namespace != constants2.DefaultCiWorkflowNamespace { + isExt = true + env, err = impl.envRepository.FindById(ciWorkflow.EnvironmentId) + if err != nil { + impl.logger.Errorw("could not fetch stage env", "err", err) + return err + } + restConfig, err = impl.getRestConfig(ciWorkflow) + if err != nil { + return err + } + } + + isEligibleToMarkFailed := false + isPodDeleted := false + if time.Since(ciWorkflow.StartedOn) > (time.Minute * time.Duration(timeoutForFailureCiBuild)) { + + //check weather pod is exists or not, if exits check its status + wf, err := impl.workflowService.GetWorkflowStatus(ciWorkflow.ExecutorType, ciWorkflow.Name, ciWorkflow.Namespace, restConfig) + if err != nil { + impl.logger.Warnw("unable to fetch ci workflow", "err", err) + statusError, ok := err.(*errors2.StatusError) + if ok && statusError.Status().Code == http.StatusNotFound { + impl.logger.Warnw("ci workflow not found", "err", err) + isEligibleToMarkFailed = true + } else { + continue + // skip this and process for next ci workflow + } + } + + //if ci workflow is exists, check its pod + if !isEligibleToMarkFailed { + ns := constants2.DefaultCiWorkflowNamespace + if isExt { + _, client, err = impl.k8sCommonService.GetCoreClientByClusterId(env.ClusterId) + if err != nil { + impl.logger.Warnw("error in getting core v1 client using GetCoreClientByClusterId", "err", err, "clusterId", env.Cluster.Id) + continue + } + ns = env.Namespace + } + _, err := impl.K8sUtil.GetPodByName(ns, ciWorkflow.PodName, client) + if err != nil { + impl.logger.Warnw("unable to fetch ci workflow - pod", "err", err) + statusError, ok := err.(*errors2.StatusError) + if ok && statusError.Status().Code == http.StatusNotFound { + impl.logger.Warnw("pod not found", "err", err) + isEligibleToMarkFailed = true + } else { + continue + // skip this and process for next ci workflow + } + } + if ciWorkflow.ExecutorType == cdWorkflow2.WORKFLOW_EXECUTOR_TYPE_SYSTEM { + if wf.Status == string(v1alpha1.WorkflowFailed) { + isPodDeleted = true + } + } else { + // check workflow status,get the status + if wf.Status == string(v1alpha1.WorkflowFailed) && wf.Message == cdWorkflow2.POD_DELETED_MESSAGE { + isPodDeleted = true + } + } + } + } + if isEligibleToMarkFailed { + ciWorkflow.Status = "Failed" + ciWorkflow.PodStatus = "Failed" + if isPodDeleted { + ciWorkflow.Message = cdWorkflow2.POD_DELETED_MESSAGE + // error logging handled inside handlePodDeleted + impl.ciTriggerService.HandlePodDeleted(ciWorkflow) + } else { + ciWorkflow.Message = "marked failed by job" + } + err := impl.ciService.UpdateCiWorkflowWithStage(ciWorkflow) + if err != nil { + impl.logger.Errorw("unable to update ci workflow, its eligible to mark failed", "err", err) + // skip this and process for next ci workflow + } + err = impl.customTagService.DeactivateImagePathReservation(ciWorkflow.ImagePathReservationId) + if err != nil { + impl.logger.Errorw("unable to update ci workflow, its eligible to mark failed", "err", err) + } + } + } + return nil +} + +func (impl *WorkflowDagExecutorImpl) getRestConfig(workflow *pipelineConfig.CiWorkflow) (*rest.Config, error) { + env, err := impl.envRepository.FindById(workflow.EnvironmentId) + if err != nil { + impl.logger.Errorw("could not fetch stage env", "err", err) + return nil, err + } + + clusterBean := adapter.GetClusterBean(*env.Cluster) + + clusterConfig := clusterBean.GetClusterConfig() + restConfig, err := impl.K8sUtil.GetRestConfigByCluster(clusterConfig) + if err != nil { + impl.logger.Errorw("error in getting rest config by cluster id", "err", err) + return nil, err + } + return restConfig, nil +} + func (impl *WorkflowDagExecutorImpl) HandleCdStageReTrigger(runner *pipelineConfig.CdWorkflowRunner) error { // do not re-trigger if retries = 0 if !impl.config.WorkflowRetriesEnabled() { diff --git a/wire_gen.go b/wire_gen.go index ff2d593a71..a59599dfb9 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -49,7 +49,7 @@ import ( "github.com/devtron-labs/devtron/api/restHandler/app/pipeline/configure" history2 "github.com/devtron-labs/devtron/api/restHandler/app/pipeline/history" status3 "github.com/devtron-labs/devtron/api/restHandler/app/pipeline/status" - "github.com/devtron-labs/devtron/api/restHandler/app/pipeline/trigger" + trigger2 "github.com/devtron-labs/devtron/api/restHandler/app/pipeline/trigger" "github.com/devtron-labs/devtron/api/restHandler/app/pipeline/webhook" "github.com/devtron-labs/devtron/api/restHandler/app/workflow" "github.com/devtron-labs/devtron/api/restHandler/scopedVariable" @@ -62,7 +62,7 @@ import ( configure2 "github.com/devtron-labs/devtron/api/router/app/pipeline/configure" history3 "github.com/devtron-labs/devtron/api/router/app/pipeline/history" status4 "github.com/devtron-labs/devtron/api/router/app/pipeline/status" - trigger2 "github.com/devtron-labs/devtron/api/router/app/pipeline/trigger" + trigger3 "github.com/devtron-labs/devtron/api/router/app/pipeline/trigger" workflow2 "github.com/devtron-labs/devtron/api/router/app/workflow" server2 "github.com/devtron-labs/devtron/api/server" "github.com/devtron-labs/devtron/api/sse" @@ -100,7 +100,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/deploymentConfig" repository9 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/helper" - repository22 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" + repository23 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/resourceGroup" "github.com/devtron-labs/devtron/internal/util" @@ -147,14 +147,15 @@ import ( read21 "github.com/devtron-labs/devtron/pkg/build/git/gitHost/read" repository26 "github.com/devtron-labs/devtron/pkg/build/git/gitHost/repository" read15 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/read" - repository20 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/repository" + repository21 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/repository" "github.com/devtron-labs/devtron/pkg/build/git/gitProvider" read9 "github.com/devtron-labs/devtron/pkg/build/git/gitProvider/read" - repository11 "github.com/devtron-labs/devtron/pkg/build/git/gitProvider/repository" + repository12 "github.com/devtron-labs/devtron/pkg/build/git/gitProvider/repository" "github.com/devtron-labs/devtron/pkg/build/git/gitWebhook" - repository23 "github.com/devtron-labs/devtron/pkg/build/git/gitWebhook/repository" + repository11 "github.com/devtron-labs/devtron/pkg/build/git/gitWebhook/repository" pipeline2 "github.com/devtron-labs/devtron/pkg/build/pipeline" read14 "github.com/devtron-labs/devtron/pkg/build/pipeline/read" + "github.com/devtron-labs/devtron/pkg/build/trigger" service7 "github.com/devtron-labs/devtron/pkg/bulkAction/service" "github.com/devtron-labs/devtron/pkg/chart" "github.com/devtron-labs/devtron/pkg/chart/gitOpsConfig" @@ -184,7 +185,7 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/manifest/configMapAndSecret" read19 "github.com/devtron-labs/devtron/pkg/deployment/manifest/configMapAndSecret/read" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" - repository16 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" + repository17 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" read12 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/read" @@ -198,7 +199,7 @@ import ( "github.com/devtron-labs/devtron/pkg/devtronResource" "github.com/devtron-labs/devtron/pkg/devtronResource/history/deployment/cdPipeline" read10 "github.com/devtron-labs/devtron/pkg/devtronResource/read" - repository13 "github.com/devtron-labs/devtron/pkg/devtronResource/repository" + repository14 "github.com/devtron-labs/devtron/pkg/devtronResource/repository" "github.com/devtron-labs/devtron/pkg/dockerRegistry" "github.com/devtron-labs/devtron/pkg/eventProcessor" "github.com/devtron-labs/devtron/pkg/eventProcessor/celEvaluator" @@ -212,7 +213,7 @@ import ( "github.com/devtron-labs/devtron/pkg/gitops" "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" config4 "github.com/devtron-labs/devtron/pkg/infraConfig/config" - repository14 "github.com/devtron-labs/devtron/pkg/infraConfig/repository" + repository15 "github.com/devtron-labs/devtron/pkg/infraConfig/repository" "github.com/devtron-labs/devtron/pkg/infraConfig/repository/audit" service2 "github.com/devtron-labs/devtron/pkg/infraConfig/service" audit2 "github.com/devtron-labs/devtron/pkg/infraConfig/service/audit" @@ -231,21 +232,21 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/history" - repository21 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" + repository22 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders" "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders/infraGetters/ci" "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders/infraGetters/job" - repository18 "github.com/devtron-labs/devtron/pkg/pipeline/repository" + repository19 "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" - repository17 "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus/repository" + repository18 "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus/repository" "github.com/devtron-labs/devtron/pkg/plugin" - repository19 "github.com/devtron-labs/devtron/pkg/plugin/repository" + repository20 "github.com/devtron-labs/devtron/pkg/plugin/repository" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning" read18 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/read" repository24 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/repository" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool" - repository15 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" + repository16 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" "github.com/devtron-labs/devtron/pkg/server" @@ -260,7 +261,7 @@ import ( util3 "github.com/devtron-labs/devtron/pkg/util" "github.com/devtron-labs/devtron/pkg/variables" "github.com/devtron-labs/devtron/pkg/variables/parsers" - repository12 "github.com/devtron-labs/devtron/pkg/variables/repository" + repository13 "github.com/devtron-labs/devtron/pkg/variables/repository" "github.com/devtron-labs/devtron/pkg/webhook/helm" "github.com/devtron-labs/devtron/pkg/workflow/cd" read20 "github.com/devtron-labs/devtron/pkg/workflow/cd/read" @@ -485,11 +486,12 @@ func InitializeApp() (*App, error) { clusterRbacServiceImpl := rbac2.NewClusterRbacServiceImpl(environmentServiceImpl, enforcerImpl, enforcerUtilImpl, clusterServiceImplExtended, sugaredLogger, userServiceImpl, clusterReadServiceImpl) clusterRestHandlerImpl := cluster3.NewClusterRestHandlerImpl(clusterServiceImplExtended, genericNoteServiceImpl, clusterDescriptionServiceImpl, sugaredLogger, userServiceImpl, validate, enforcerImpl, deleteServiceExtendedImpl, environmentServiceImpl, clusterRbacServiceImpl) clusterRouterImpl := cluster3.NewClusterRouterImpl(clusterRestHandlerImpl) + gitWebhookRepositoryImpl := repository11.NewGitWebhookRepositoryImpl(db) ciCdConfig, err := types.GetCiCdConfig() if err != nil { return nil, err } - gitProviderRepositoryImpl := repository11.NewGitProviderRepositoryImpl(db) + gitProviderRepositoryImpl := repository12.NewGitProviderRepositoryImpl(db) gitProviderReadServiceImpl := read9.NewGitProviderReadService(sugaredLogger, gitProviderRepositoryImpl) commonBaseServiceImpl := commonService.NewCommonBaseServiceImpl(sugaredLogger, environmentVariables, moduleReadServiceImpl) commonServiceImpl := commonService.NewCommonServiceImpl(sugaredLogger, chartRepositoryImpl, envConfigOverrideRepositoryImpl, dockerArtifactStoreRepositoryImpl, attributesRepositoryImpl, environmentRepositoryImpl, appRepositoryImpl, gitOpsConfigReadServiceImpl, gitProviderReadServiceImpl, envConfigOverrideReadServiceImpl, commonBaseServiceImpl, teamReadServiceImpl) @@ -497,8 +499,8 @@ func InitializeApp() (*App, error) { mergeUtil := util.MergeUtil{ Logger: sugaredLogger, } - scopedVariableRepositoryImpl := repository12.NewScopedVariableRepository(db, sugaredLogger, transactionUtilImpl) - devtronResourceSearchableKeyRepositoryImpl := repository13.NewDevtronResourceSearchableKeyRepositoryImpl(sugaredLogger, db) + scopedVariableRepositoryImpl := repository13.NewScopedVariableRepository(db, sugaredLogger, transactionUtilImpl) + devtronResourceSearchableKeyRepositoryImpl := repository14.NewDevtronResourceSearchableKeyRepositoryImpl(sugaredLogger, db) devtronResourceSearchableKeyServiceImpl, err := read10.NewDevtronResourceSearchableKeyServiceImpl(sugaredLogger, devtronResourceSearchableKeyRepositoryImpl) if err != nil { return nil, err @@ -515,9 +517,9 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - variableEntityMappingRepositoryImpl := repository12.NewVariableEntityMappingRepository(sugaredLogger, db, transactionUtilImpl) + variableEntityMappingRepositoryImpl := repository13.NewVariableEntityMappingRepository(sugaredLogger, db, transactionUtilImpl) variableEntityMappingServiceImpl := variables.NewVariableEntityMappingServiceImpl(variableEntityMappingRepositoryImpl, sugaredLogger) - variableSnapshotHistoryRepositoryImpl := repository12.NewVariableSnapshotHistoryRepository(sugaredLogger, db) + variableSnapshotHistoryRepositoryImpl := repository13.NewVariableSnapshotHistoryRepository(sugaredLogger, db) variableSnapshotHistoryServiceImpl := variables.NewVariableSnapshotHistoryServiceImpl(variableSnapshotHistoryRepositoryImpl, sugaredLogger) variableTemplateParserImpl, err := parsers.NewVariableTemplateParserImpl(sugaredLogger) if err != nil { @@ -538,7 +540,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - infraConfigRepositoryImpl := repository14.NewInfraProfileRepositoryImpl(db, transactionUtilImpl) + infraConfigRepositoryImpl := repository15.NewInfraProfileRepositoryImpl(db, transactionUtilImpl) pipelineOverrideRepositoryImpl := chartConfig.NewPipelineOverrideRepository(db) utilMergeUtil := &util.MergeUtil{ Logger: sugaredLogger, @@ -571,7 +573,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - scanToolMetadataRepositoryImpl := repository15.NewScanToolMetadataRepositoryImpl(db, sugaredLogger) + scanToolMetadataRepositoryImpl := repository16.NewScanToolMetadataRepositoryImpl(db, sugaredLogger) scanToolMetadataServiceImpl := scanTool.NewScanToolMetadataServiceImpl(sugaredLogger, scanToolMetadataRepositoryImpl) moduleServiceImpl := module.NewModuleServiceImpl(sugaredLogger, serverEnvConfigServerEnvConfig, moduleRepositoryImpl, moduleActionAuditLogRepositoryImpl, helmAppServiceImpl, serverDataStoreServerDataStore, serverCacheServiceImpl, moduleCacheServiceImpl, moduleCronServiceImpl, moduleServiceHelperImpl, moduleResourceStatusRepositoryImpl, scanToolMetadataServiceImpl) eventRESTClientImpl := client2.NewEventRESTClientImpl(sugaredLogger, httpClient, eventClientConfig, pubSubClientServiceImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, attributesRepositoryImpl, moduleServiceImpl) @@ -604,11 +606,11 @@ func InitializeApp() (*App, error) { ciTemplateOverrideRepositoryImpl := pipelineConfig.NewCiTemplateOverrideRepositoryImpl(db, sugaredLogger) ciPipelineConfigReadServiceImpl := read14.NewCiPipelineConfigReadServiceImpl(sugaredLogger, ciPipelineRepositoryImpl, ciTemplateOverrideRepositoryImpl) dockerRegistryIpsConfigServiceImpl := dockerRegistry.NewDockerRegistryIpsConfigServiceImpl(sugaredLogger, dockerRegistryIpsConfigRepositoryImpl, k8sServiceImpl, dockerArtifactStoreRepositoryImpl, clusterReadServiceImpl, ciPipelineConfigReadServiceImpl) - appLevelMetricsRepositoryImpl := repository16.NewAppLevelMetricsRepositoryImpl(db, sugaredLogger) - envLevelAppMetricsRepositoryImpl := repository16.NewEnvLevelAppMetricsRepositoryImpl(db, sugaredLogger) + appLevelMetricsRepositoryImpl := repository17.NewAppLevelMetricsRepositoryImpl(db, sugaredLogger) + envLevelAppMetricsRepositoryImpl := repository17.NewEnvLevelAppMetricsRepositoryImpl(db, sugaredLogger) deployedAppMetricsServiceImpl := deployedAppMetrics.NewDeployedAppMetricsServiceImpl(sugaredLogger, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, chartRefServiceImpl) appListingServiceImpl := app2.NewAppListingServiceImpl(sugaredLogger, appListingRepositoryImpl, appDetailsReadServiceImpl, appRepositoryImpl, appListingViewBuilderImpl, pipelineRepositoryImpl, linkoutsRepositoryImpl, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, environmentRepositoryImpl, chartRepositoryImpl, ciPipelineRepositoryImpl, dockerRegistryIpsConfigServiceImpl, userRepositoryImpl, deployedAppMetricsServiceImpl, ciArtifactRepositoryImpl, envConfigOverrideReadServiceImpl, ciPipelineConfigReadServiceImpl) - workflowStageRepositoryImpl := repository17.NewWorkflowStageRepositoryImpl(sugaredLogger, db) + workflowStageRepositoryImpl := repository18.NewWorkflowStageRepositoryImpl(sugaredLogger, db) workFlowStageStatusServiceImpl := workflowStatus.NewWorkflowStageFlowStatusServiceImpl(sugaredLogger, workflowStageRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, transactionUtilImpl) cdWorkflowRunnerServiceImpl := cd.NewCdWorkflowRunnerServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl, workFlowStageStatusServiceImpl, transactionUtilImpl) appServiceImpl := app2.NewAppService(pipelineOverrideRepositoryImpl, utilMergeUtil, sugaredLogger, pipelineRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, appRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, chartTemplateServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, appStatusServiceImpl, installedAppReadServiceImpl, installedAppVersionHistoryRepositoryImpl, scopedVariableCMCSManagerImpl, acdConfig, gitOpsConfigReadServiceImpl, gitOperationServiceImpl, deploymentTemplateServiceImpl, appListingServiceImpl, deploymentConfigServiceImpl, envConfigOverrideReadServiceImpl, cdWorkflowRunnerServiceImpl) @@ -627,8 +629,8 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - pipelineStageRepositoryImpl := repository18.NewPipelineStageRepository(sugaredLogger, db) - globalPluginRepositoryImpl := repository19.NewGlobalPluginRepository(sugaredLogger, db) + pipelineStageRepositoryImpl := repository19.NewPipelineStageRepository(sugaredLogger, db) + globalPluginRepositoryImpl := repository20.NewGlobalPluginRepository(sugaredLogger, db) globalPluginServiceImpl := plugin.NewGlobalPluginService(sugaredLogger, globalPluginRepositoryImpl, pipelineStageRepositoryImpl, userServiceImpl) pipelineStageServiceImpl := pipeline.NewPipelineStageService(sugaredLogger, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, pipelineRepositoryImpl, scopedVariableManagerImpl, globalPluginServiceImpl) ciTemplateRepositoryImpl := pipelineConfig.NewCiTemplateRepositoryImpl(db, sugaredLogger) @@ -638,12 +640,11 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - materialRepositoryImpl := repository20.NewMaterialRepositoryImpl(db) + materialRepositoryImpl := repository21.NewMaterialRepositoryImpl(db) gitMaterialReadServiceImpl := read15.NewGitMaterialReadServiceImpl(sugaredLogger, materialRepositoryImpl) appCrudOperationServiceImpl := app2.NewAppCrudOperationServiceImpl(appLabelRepositoryImpl, sugaredLogger, appRepositoryImpl, userRepositoryImpl, installedAppRepositoryImpl, genericNoteServiceImpl, installedAppDBServiceImpl, crudOperationServiceConfig, dbMigrationServiceImpl, gitMaterialReadServiceImpl) imageTagRepositoryImpl := repository2.NewImageTagRepository(db, sugaredLogger) customTagServiceImpl := pipeline.NewCustomTagService(sugaredLogger, imageTagRepositoryImpl) - pluginInputVariableParserImpl := pipeline.NewPluginInputVariableParserImpl(sugaredLogger, dockerRegistryConfigImpl, customTagServiceImpl) clientConfig, err := gitSensor.GetConfig() if err != nil { return nil, err @@ -652,54 +653,49 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - prePostCdScriptHistoryRepositoryImpl := repository21.NewPrePostCdScriptHistoryRepositoryImpl(sugaredLogger, db) - configMapHistoryRepositoryImpl := repository21.NewConfigMapHistoryRepositoryImpl(sugaredLogger, db, transactionUtilImpl) + prePostCdScriptHistoryRepositoryImpl := repository22.NewPrePostCdScriptHistoryRepositoryImpl(sugaredLogger, db) + configMapHistoryRepositoryImpl := repository22.NewConfigMapHistoryRepositoryImpl(sugaredLogger, db, transactionUtilImpl) configMapHistoryServiceImpl := configMapAndSecret.NewConfigMapHistoryServiceImpl(sugaredLogger, configMapHistoryRepositoryImpl, pipelineRepositoryImpl, configMapRepositoryImpl, userServiceImpl, scopedVariableCMCSManagerImpl) prePostCdScriptHistoryServiceImpl := history.NewPrePostCdScriptHistoryServiceImpl(sugaredLogger, prePostCdScriptHistoryRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl) - gitMaterialHistoryRepositoryImpl := repository21.NewGitMaterialHistoryRepositoyImpl(db) + gitMaterialHistoryRepositoryImpl := repository22.NewGitMaterialHistoryRepositoyImpl(db) gitMaterialHistoryServiceImpl := history.NewGitMaterialHistoryServiceImpl(gitMaterialHistoryRepositoryImpl, sugaredLogger) - ciPipelineHistoryRepositoryImpl := repository21.NewCiPipelineHistoryRepositoryImpl(db, sugaredLogger) + ciPipelineHistoryRepositoryImpl := repository22.NewCiPipelineHistoryRepositoryImpl(db, sugaredLogger) ciPipelineHistoryServiceImpl := history.NewCiPipelineHistoryServiceImpl(ciPipelineHistoryRepositoryImpl, sugaredLogger, ciPipelineRepositoryImpl) ciBuildConfigRepositoryImpl := pipelineConfig.NewCiBuildConfigRepositoryImpl(db, sugaredLogger) ciBuildConfigServiceImpl := pipeline.NewCiBuildConfigServiceImpl(sugaredLogger, ciBuildConfigRepositoryImpl) ciTemplateServiceImpl := pipeline.NewCiTemplateServiceImpl(sugaredLogger, ciBuildConfigServiceImpl, ciTemplateRepositoryImpl, ciTemplateOverrideRepositoryImpl) pipelineConfigRepositoryImpl := chartConfig.NewPipelineConfigRepository(db) configMapServiceImpl := pipeline.NewConfigMapServiceImpl(chartRepositoryImpl, sugaredLogger, chartRepoRepositoryImpl, mergeUtil, pipelineConfigRepositoryImpl, configMapRepositoryImpl, commonServiceImpl, appRepositoryImpl, configMapHistoryServiceImpl, environmentRepositoryImpl, scopedVariableCMCSManagerImpl) - deploymentTemplateHistoryRepositoryImpl := repository21.NewDeploymentTemplateHistoryRepositoryImpl(sugaredLogger, db) + deploymentTemplateHistoryRepositoryImpl := repository22.NewDeploymentTemplateHistoryRepositoryImpl(sugaredLogger, db) deploymentTemplateHistoryServiceImpl := deploymentTemplate.NewDeploymentTemplateHistoryServiceImpl(sugaredLogger, deploymentTemplateHistoryRepositoryImpl, pipelineRepositoryImpl, chartRepositoryImpl, userServiceImpl, cdWorkflowRepositoryImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) chartReadServiceImpl := read16.NewChartReadServiceImpl(sugaredLogger, chartRepositoryImpl, deploymentConfigServiceImpl, deployedAppMetricsServiceImpl, gitOpsConfigReadServiceImpl, chartRefReadServiceImpl) chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, mergeUtil, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl, envConfigOverrideReadServiceImpl, chartReadServiceImpl) ciCdPipelineOrchestratorImpl := pipeline.NewCiCdPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, clientImpl, ciCdConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, pipelineStageServiceImpl, gitMaterialHistoryServiceImpl, ciPipelineHistoryServiceImpl, ciTemplateReadServiceImpl, ciTemplateServiceImpl, dockerArtifactStoreRepositoryImpl, ciArtifactRepositoryImpl, configMapServiceImpl, customTagServiceImpl, genericNoteServiceImpl, chartServiceImpl, transactionUtilImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl, deploymentConfigReadServiceImpl, chartReadServiceImpl) - ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, workFlowStageStatusServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateReadServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl, infraProviderImpl, ciCdPipelineOrchestratorImpl, attributesServiceImpl, ciWorkflowRepositoryImpl, transactionUtilImpl) - ciLogServiceImpl, err := pipeline.NewCiLogServiceImpl(sugaredLogger, ciServiceImpl, k8sServiceImpl) + pluginInputVariableParserImpl := pipeline.NewPluginInputVariableParserImpl(sugaredLogger, dockerRegistryConfigImpl, customTagServiceImpl) + ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workFlowStageStatusServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciWorkflowRepositoryImpl, transactionUtilImpl) + serviceImpl := trigger.NewServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciPipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateReadServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, ciCdPipelineOrchestratorImpl, attributesServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl, ciServiceImpl, ciWorkflowRepositoryImpl, clientImpl) + gitWebhookServiceImpl := gitWebhook.NewGitWebhookServiceImpl(sugaredLogger, gitWebhookRepositoryImpl, serviceImpl) + gitWebhookRestHandlerImpl := restHandler.NewGitWebhookRestHandlerImpl(sugaredLogger, gitWebhookServiceImpl) + ecrConfig, err := pipeline.GetEcrConfig() if err != nil { return nil, err } + ciTemplateHistoryRepositoryImpl := repository22.NewCiTemplateHistoryRepositoryImpl(db, sugaredLogger) + ciTemplateHistoryServiceImpl := history.NewCiTemplateHistoryServiceImpl(ciTemplateHistoryRepositoryImpl, sugaredLogger) resourceGroupRepositoryImpl := resourceGroup.NewResourceGroupRepositoryImpl(db) resourceGroupMappingRepositoryImpl := resourceGroup.NewResourceGroupMappingRepositoryImpl(db) resourceGroupServiceImpl := resourceGroup2.NewResourceGroupServiceImpl(sugaredLogger, resourceGroupRepositoryImpl, resourceGroupMappingRepositoryImpl, enforcerUtilImpl, devtronResourceSearchableKeyServiceImpl, appStatusRepositoryImpl) - imageTaggingRepositoryImpl := repository22.NewImageTaggingRepositoryImpl(db, transactionUtilImpl) + buildPipelineSwitchServiceImpl := pipeline.NewBuildPipelineSwitchServiceImpl(sugaredLogger, ciPipelineConfigReadServiceImpl, ciPipelineRepositoryImpl, ciCdPipelineOrchestratorImpl, pipelineRepositoryImpl, ciWorkflowRepositoryImpl, appWorkflowRepositoryImpl, ciPipelineHistoryServiceImpl, ciTemplateOverrideRepositoryImpl, ciPipelineMaterialRepositoryImpl) + ciPipelineConfigServiceImpl := pipeline.NewCiPipelineConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, gitMaterialReadServiceImpl, appRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigReadServiceImpl, ciPipelineRepositoryImpl, ecrConfig, appWorkflowRepositoryImpl, ciCdConfig, attributesServiceImpl, pipelineStageServiceImpl, ciPipelineMaterialRepositoryImpl, ciTemplateServiceImpl, ciTemplateReadServiceImpl, ciTemplateOverrideRepositoryImpl, ciTemplateHistoryServiceImpl, enforcerUtilImpl, ciWorkflowRepositoryImpl, resourceGroupServiceImpl, customTagServiceImpl, cdWorkflowRepositoryImpl, buildPipelineSwitchServiceImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl) + ciMaterialConfigServiceImpl := pipeline.NewCiMaterialConfigServiceImpl(sugaredLogger, materialRepositoryImpl, ciTemplateReadServiceImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, gitMaterialHistoryServiceImpl, pipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, transactionUtilImpl, gitMaterialReadServiceImpl) + imageTaggingRepositoryImpl := repository23.NewImageTaggingRepositoryImpl(db, transactionUtilImpl) imageTaggingReadServiceImpl, err := read17.NewImageTaggingReadServiceImpl(imageTaggingRepositoryImpl, sugaredLogger) if err != nil { return nil, err } imageTaggingServiceImpl := imageTagging.NewImageTaggingServiceImpl(imageTaggingRepositoryImpl, imageTaggingReadServiceImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, sugaredLogger) - blobStorageConfigServiceImpl := pipeline.NewBlobStorageConfigServiceImpl(sugaredLogger, k8sServiceImpl, ciCdConfig) - ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, workflowServiceImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, environmentServiceImpl, workFlowStageStatusServiceImpl, k8sServiceImpl) - gitWebhookRepositoryImpl := repository23.NewGitWebhookRepositoryImpl(db) - gitWebhookServiceImpl := gitWebhook.NewGitWebhookServiceImpl(sugaredLogger, ciHandlerImpl, gitWebhookRepositoryImpl) - gitWebhookRestHandlerImpl := restHandler.NewGitWebhookRestHandlerImpl(sugaredLogger, gitWebhookServiceImpl) - ecrConfig, err := pipeline.GetEcrConfig() - if err != nil { - return nil, err - } - ciTemplateHistoryRepositoryImpl := repository21.NewCiTemplateHistoryRepositoryImpl(db, sugaredLogger) - ciTemplateHistoryServiceImpl := history.NewCiTemplateHistoryServiceImpl(ciTemplateHistoryRepositoryImpl, sugaredLogger) - buildPipelineSwitchServiceImpl := pipeline.NewBuildPipelineSwitchServiceImpl(sugaredLogger, ciPipelineConfigReadServiceImpl, ciPipelineRepositoryImpl, ciCdPipelineOrchestratorImpl, pipelineRepositoryImpl, ciWorkflowRepositoryImpl, appWorkflowRepositoryImpl, ciPipelineHistoryServiceImpl, ciTemplateOverrideRepositoryImpl, ciPipelineMaterialRepositoryImpl) - ciPipelineConfigServiceImpl := pipeline.NewCiPipelineConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, gitMaterialReadServiceImpl, appRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigReadServiceImpl, ciPipelineRepositoryImpl, ecrConfig, appWorkflowRepositoryImpl, ciCdConfig, attributesServiceImpl, pipelineStageServiceImpl, ciPipelineMaterialRepositoryImpl, ciTemplateServiceImpl, ciTemplateReadServiceImpl, ciTemplateOverrideRepositoryImpl, ciTemplateHistoryServiceImpl, enforcerUtilImpl, ciWorkflowRepositoryImpl, resourceGroupServiceImpl, customTagServiceImpl, cdWorkflowRepositoryImpl, buildPipelineSwitchServiceImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl) - ciMaterialConfigServiceImpl := pipeline.NewCiMaterialConfigServiceImpl(sugaredLogger, materialRepositoryImpl, ciTemplateReadServiceImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, gitMaterialHistoryServiceImpl, pipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, transactionUtilImpl, gitMaterialReadServiceImpl) deploymentGroupRepositoryImpl := repository2.NewDeploymentGroupRepositoryImpl(sugaredLogger, db) - pipelineStrategyHistoryRepositoryImpl := repository21.NewPipelineStrategyHistoryRepositoryImpl(sugaredLogger, db) + pipelineStrategyHistoryRepositoryImpl := repository22.NewPipelineStrategyHistoryRepositoryImpl(sugaredLogger, db) pipelineStrategyHistoryServiceImpl := history.NewPipelineStrategyHistoryServiceImpl(sugaredLogger, pipelineStrategyHistoryRepositoryImpl, userServiceImpl) propertiesConfigServiceImpl := pipeline.NewPropertiesConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, envConfigOverrideReadServiceImpl, deploymentConfigServiceImpl, chartServiceImpl) installedAppDBExtendedServiceImpl := FullMode.NewInstalledAppDBExtendedServiceImpl(installedAppDBServiceImpl, appStatusServiceImpl, gitOpsConfigReadServiceImpl) @@ -730,6 +726,12 @@ func InitializeApp() (*App, error) { pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, gitMaterialReadServiceImpl, chartRepositoryImpl, ciPipelineConfigServiceImpl, ciMaterialConfigServiceImpl, appArtifactManagerImpl, devtronAppCMCSServiceImpl, devtronAppStrategyServiceImpl, appDeploymentTypeChangeManagerImpl, cdPipelineConfigServiceImpl, devtronAppConfigServiceImpl) deploymentTemplateValidationServiceImpl := deploymentTemplate.NewDeploymentTemplateValidationServiceImpl(sugaredLogger, chartRefServiceImpl, scopedVariableManagerImpl) devtronAppGitOpConfigServiceImpl := gitOpsConfig.NewDevtronAppGitOpConfigServiceImpl(sugaredLogger, chartRepositoryImpl, chartServiceImpl, gitOpsConfigReadServiceImpl, gitOpsValidationServiceImpl, argoClientWrapperServiceImpl, deploymentConfigServiceImpl, chartReadServiceImpl) + ciLogServiceImpl, err := pipeline.NewCiLogServiceImpl(sugaredLogger, k8sServiceImpl) + if err != nil { + return nil, err + } + blobStorageConfigServiceImpl := pipeline.NewBlobStorageConfigServiceImpl(sugaredLogger, k8sServiceImpl, ciCdConfig) + ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, workflowServiceImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, environmentServiceImpl, workFlowStageStatusServiceImpl, k8sServiceImpl) cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, workflowServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, customTagServiceImpl, deploymentConfigServiceImpl, workFlowStageStatusServiceImpl, cdWorkflowRunnerServiceImpl) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl, chartServiceImpl, deploymentConfigServiceImpl, pipelineBuilderImpl) appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, attributesServiceImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, pipelineStageServiceImpl, ciTemplateReadServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigServiceImpl, gitOpsConfigReadServiceImpl, chartReadServiceImpl) @@ -748,7 +750,7 @@ func InitializeApp() (*App, error) { cveStoreRepositoryImpl := repository24.NewCveStoreRepositoryImpl(db, sugaredLogger) policyServiceImpl := imageScanning.NewPolicyServiceImpl(environmentServiceImpl, sugaredLogger, appRepositoryImpl, pipelineOverrideRepositoryImpl, cvePolicyRepositoryImpl, clusterServiceImplExtended, pipelineRepositoryImpl, imageScanResultRepositoryImpl, imageScanDeployInfoRepositoryImpl, imageScanObjectMetaRepositoryImpl, httpClient, ciArtifactRepositoryImpl, ciCdConfig, imageScanHistoryReadServiceImpl, cveStoreRepositoryImpl, ciTemplateRepositoryImpl, clusterReadServiceImpl, transactionUtilImpl) imageScanResultReadServiceImpl := read18.NewImageScanResultReadServiceImpl(sugaredLogger, imageScanResultRepositoryImpl) - pipelineConfigRestHandlerImpl := configure.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, deploymentTemplateValidationServiceImpl, chartServiceImpl, devtronAppGitOpConfigServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, generateManifestDeploymentTemplateServiceImpl, appWorkflowServiceImpl, gitMaterialReadServiceImpl, policyServiceImpl, imageScanResultReadServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingReadServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, ciCdPipelineOrchestratorImpl, gitProviderReadServiceImpl, teamReadServiceImpl, environmentRepositoryImpl, chartReadServiceImpl) + pipelineConfigRestHandlerImpl := configure.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, deploymentTemplateValidationServiceImpl, chartServiceImpl, devtronAppGitOpConfigServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, generateManifestDeploymentTemplateServiceImpl, appWorkflowServiceImpl, gitMaterialReadServiceImpl, policyServiceImpl, imageScanResultReadServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingReadServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, ciCdPipelineOrchestratorImpl, gitProviderReadServiceImpl, teamReadServiceImpl, environmentRepositoryImpl, chartReadServiceImpl, serviceImpl) gitOpsManifestPushServiceImpl := publish.NewGitOpsManifestPushServiceImpl(sugaredLogger, pipelineStatusTimelineServiceImpl, pipelineOverrideRepositoryImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, chartServiceImpl, gitOperationServiceImpl, argoClientWrapperServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, chartTemplateServiceImpl) manifestCreationServiceImpl := manifest.NewManifestCreationServiceImpl(sugaredLogger, dockerRegistryIpsConfigServiceImpl, chartRefServiceImpl, scopedVariableCMCSManagerImpl, k8sCommonServiceImpl, deployedAppMetricsServiceImpl, imageDigestPolicyServiceImpl, utilMergeUtil, appCrudOperationServiceImpl, deploymentTemplateServiceImpl, argoClientWrapperServiceImpl, configMapHistoryRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineOverrideRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, pipelineConfigRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, deploymentConfigServiceImpl, envConfigOverrideReadServiceImpl) configMapHistoryReadServiceImpl := read19.NewConfigMapHistoryReadService(sugaredLogger, configMapHistoryRepositoryImpl, scopedVariableCMCSManagerImpl) @@ -757,7 +759,7 @@ func InitializeApp() (*App, error) { userDeploymentRequestServiceImpl := service3.NewUserDeploymentRequestServiceImpl(sugaredLogger, userDeploymentRequestRepositoryImpl) imageScanDeployInfoReadServiceImpl := read18.NewImageScanDeployInfoReadService(sugaredLogger, imageScanDeployInfoRepositoryImpl) imageScanDeployInfoServiceImpl := imageScanning.NewImageScanDeployInfoService(sugaredLogger, imageScanDeployInfoRepositoryImpl) - manifestPushConfigRepositoryImpl := repository18.NewManifestPushConfigRepository(sugaredLogger, db) + manifestPushConfigRepositoryImpl := repository19.NewManifestPushConfigRepository(sugaredLogger, db) scanToolExecutionHistoryMappingRepositoryImpl := repository24.NewScanToolExecutionHistoryMappingRepositoryImpl(db, sugaredLogger) cdWorkflowReadServiceImpl := read20.NewCdWorkflowReadServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) imageScanServiceImpl := imageScanning.NewImageScanServiceImpl(sugaredLogger, imageScanHistoryRepositoryImpl, imageScanResultRepositoryImpl, imageScanObjectMetaRepositoryImpl, cveStoreRepositoryImpl, imageScanDeployInfoRepositoryImpl, userServiceImpl, appRepositoryImpl, environmentServiceImpl, ciArtifactRepositoryImpl, policyServiceImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, scanToolMetadataRepositoryImpl, scanToolExecutionHistoryMappingRepositoryImpl, cvePolicyRepositoryImpl, cdWorkflowReadServiceImpl) @@ -766,7 +768,7 @@ func InitializeApp() (*App, error) { return nil, err } commonArtifactServiceImpl := artifacts.NewCommonArtifactServiceImpl(sugaredLogger, ciArtifactRepositoryImpl) - workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, cdWorkflowRunnerServiceImpl, ciServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl, runnable, imageScanHistoryRepositoryImpl, imageScanServiceImpl) + workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, cdWorkflowRunnerServiceImpl, ciServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl, runnable, imageScanHistoryRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, environmentRepositoryImpl, k8sCommonServiceImpl, workflowServiceImpl, serviceImpl) externalCiRestHandlerImpl := restHandler.NewExternalCiRestHandlerImpl(sugaredLogger, validate, userServiceImpl, enforcerImpl, workflowDagExecutorImpl) pubSubClientRestHandlerImpl := restHandler.NewPubSubClientRestHandlerImpl(pubSubClientServiceImpl, sugaredLogger, ciCdConfig) webhookRouterImpl := router.NewWebhookRouterImpl(gitWebhookRestHandlerImpl, pipelineConfigRestHandlerImpl, externalCiRestHandlerImpl, pubSubClientRestHandlerImpl) @@ -937,7 +939,7 @@ func InitializeApp() (*App, error) { telemetryRouterImpl := router.NewTelemetryRouterImpl(sugaredLogger, telemetryRestHandlerImpl) bulkUpdateRepositoryImpl := bulkUpdate.NewBulkUpdateRepository(db, sugaredLogger) deployedAppServiceImpl := deployedApp.NewDeployedAppServiceImpl(sugaredLogger, k8sCommonServiceImpl, triggerServiceImpl, environmentRepositoryImpl, pipelineRepositoryImpl, cdWorkflowRepositoryImpl) - bulkUpdateServiceImpl := service7.NewBulkUpdateServiceImpl(bulkUpdateRepositoryImpl, sugaredLogger, environmentRepositoryImpl, pipelineRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineBuilderImpl, enforcerUtilImpl, ciHandlerImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, appWorkflowServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, deployedAppServiceImpl, cdPipelineEventPublishServiceImpl) + bulkUpdateServiceImpl := service7.NewBulkUpdateServiceImpl(bulkUpdateRepositoryImpl, sugaredLogger, environmentRepositoryImpl, pipelineRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineBuilderImpl, enforcerUtilImpl, ciHandlerImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, appWorkflowServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, deployedAppServiceImpl, cdPipelineEventPublishServiceImpl, serviceImpl) bulkUpdateRestHandlerImpl := restHandler.NewBulkUpdateRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, bulkUpdateServiceImpl, chartServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, environmentServiceImpl, gitRegistryConfigImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl) bulkUpdateRouterImpl := router.NewBulkUpdateRouterImpl(bulkUpdateRestHandlerImpl) webhookSecretValidatorImpl := gitWebhook.NewWebhookSecretValidatorImpl(sugaredLogger) @@ -948,18 +950,18 @@ func InitializeApp() (*App, error) { webhookListenerRouterImpl := router.NewWebhookListenerRouterImpl(webhookEventHandlerImpl) appFilteringRestHandlerImpl := appList.NewAppFilteringRestHandlerImpl(sugaredLogger, teamServiceImpl, enforcerImpl, userServiceImpl, clusterServiceImplExtended, environmentServiceImpl, teamReadServiceImpl) appFilteringRouterImpl := appList2.NewAppFilteringRouterImpl(appFilteringRestHandlerImpl) - serviceImpl := resourceTree.NewServiceImpl(sugaredLogger, appListingServiceImpl, appStatusServiceImpl, argoApplicationServiceExtendedImpl, cdApplicationStatusUpdateHandlerImpl, helmAppReadServiceImpl, helmAppServiceImpl, k8sApplicationServiceImpl, k8sCommonServiceImpl, environmentReadServiceImpl) - appListingRestHandlerImpl := appList.NewAppListingRestHandlerImpl(appListingServiceImpl, enforcerImpl, pipelineBuilderImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, userServiceImpl, k8sCommonServiceImpl, installedAppDBExtendedServiceImpl, installedAppResourceServiceImpl, pipelineRepositoryImpl, k8sApplicationServiceImpl, deploymentConfigServiceImpl, serviceImpl) + resourceTreeServiceImpl := resourceTree.NewServiceImpl(sugaredLogger, appListingServiceImpl, appStatusServiceImpl, argoApplicationServiceExtendedImpl, cdApplicationStatusUpdateHandlerImpl, helmAppReadServiceImpl, helmAppServiceImpl, k8sApplicationServiceImpl, k8sCommonServiceImpl, environmentReadServiceImpl) + appListingRestHandlerImpl := appList.NewAppListingRestHandlerImpl(appListingServiceImpl, enforcerImpl, pipelineBuilderImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, userServiceImpl, k8sCommonServiceImpl, installedAppDBExtendedServiceImpl, installedAppResourceServiceImpl, pipelineRepositoryImpl, k8sApplicationServiceImpl, deploymentConfigServiceImpl, resourceTreeServiceImpl) appListingRouterImpl := appList2.NewAppListingRouterImpl(appListingRestHandlerImpl) appInfoRestHandlerImpl := appInfo.NewAppInfoRestHandlerImpl(sugaredLogger, appCrudOperationServiceImpl, userServiceImpl, validate, enforcerUtilImpl, enforcerImpl, helmAppServiceImpl, enforcerUtilHelmImpl, genericNoteServiceImpl, commonEnforcementUtilImpl) appInfoRouterImpl := appInfo2.NewAppInfoRouterImpl(sugaredLogger, appInfoRestHandlerImpl) pipelineDeploymentConfigServiceImpl := pipeline.NewPipelineDeploymentConfigServiceImpl(sugaredLogger, chartRepositoryImpl, pipelineRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, scopedVariableCMCSManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, configMapHistoryReadServiceImpl, envConfigOverrideReadServiceImpl) - pipelineTriggerRestHandlerImpl := trigger.NewPipelineRestHandler(appServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, pipelineDeploymentConfigServiceImpl, deployedAppServiceImpl, triggerServiceImpl, workflowEventPublishServiceImpl) + pipelineTriggerRestHandlerImpl := trigger2.NewPipelineRestHandler(appServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, pipelineDeploymentConfigServiceImpl, deployedAppServiceImpl, triggerServiceImpl, workflowEventPublishServiceImpl) sseSSE := sse.NewSSE() - pipelineTriggerRouterImpl := trigger2.NewPipelineTriggerRouter(pipelineTriggerRestHandlerImpl, sseSSE) + pipelineTriggerRouterImpl := trigger3.NewPipelineTriggerRouter(pipelineTriggerRestHandlerImpl, sseSSE) webhookDataRestHandlerImpl := webhook.NewWebhookDataRestHandlerImpl(sugaredLogger, userServiceImpl, ciPipelineMaterialRepositoryImpl, enforcerUtilImpl, enforcerImpl, clientImpl, webhookEventDataConfigImpl) pipelineConfigRouterImpl := configure2.NewPipelineRouterImpl(pipelineConfigRestHandlerImpl, webhookDataRestHandlerImpl) - prePostCiScriptHistoryRepositoryImpl := repository21.NewPrePostCiScriptHistoryRepositoryImpl(sugaredLogger, db) + prePostCiScriptHistoryRepositoryImpl := repository22.NewPrePostCiScriptHistoryRepositoryImpl(sugaredLogger, db) prePostCiScriptHistoryServiceImpl := history.NewPrePostCiScriptHistoryServiceImpl(sugaredLogger, prePostCiScriptHistoryRepositoryImpl) pipelineHistoryRestHandlerImpl := history2.NewPipelineHistoryRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerImpl, pipelineStrategyHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, prePostCdScriptHistoryServiceImpl, enforcerUtilImpl, deployedConfigurationHistoryServiceImpl) pipelineHistoryRouterImpl := history3.NewPipelineHistoryRouterImpl(pipelineHistoryRestHandlerImpl) @@ -1030,7 +1032,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - ciStatusUpdateCronImpl := cron2.NewCiStatusUpdateCronImpl(sugaredLogger, appServiceImpl, ciWorkflowStatusUpdateConfig, ciPipelineRepositoryImpl, ciHandlerImpl, cronLoggerImpl) + ciStatusUpdateCronImpl := cron2.NewCiStatusUpdateCronImpl(sugaredLogger, appServiceImpl, ciWorkflowStatusUpdateConfig, ciPipelineRepositoryImpl, cronLoggerImpl, workflowDagExecutorImpl) resourceGroupRestHandlerImpl := restHandler.NewResourceGroupRestHandlerImpl(sugaredLogger, enforcerImpl, userServiceImpl, resourceGroupServiceImpl, validate) resourceGroupingRouterImpl := router.NewResourceGroupingRouterImpl(pipelineConfigRestHandlerImpl, appWorkflowRestHandlerImpl, resourceGroupRestHandlerImpl) rbacRoleServiceImpl := user.NewRbacRoleServiceImpl(sugaredLogger, rbacRoleDataRepositoryImpl) @@ -1042,7 +1044,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - ciTriggerCronImpl := cron2.NewCiTriggerCronImpl(sugaredLogger, ciTriggerCronConfig, pipelineStageRepositoryImpl, ciHandlerImpl, ciArtifactRepositoryImpl, globalPluginRepositoryImpl, cronLoggerImpl) + ciTriggerCronImpl := cron2.NewCiTriggerCronImpl(sugaredLogger, ciTriggerCronConfig, pipelineStageRepositoryImpl, ciArtifactRepositoryImpl, globalPluginRepositoryImpl, cronLoggerImpl, serviceImpl) proxyConfig, err := proxy.GetProxyConfig() if err != nil { return nil, err @@ -1078,7 +1080,7 @@ func InitializeApp() (*App, error) { cdWorkflowServiceImpl := cd.NewCdWorkflowServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) cdWorkflowRunnerReadServiceImpl := read20.NewCdWorkflowRunnerReadServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) webhookServiceImpl := pipeline.NewWebhookServiceImpl(ciArtifactRepositoryImpl, sugaredLogger, ciPipelineRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowCommonServiceImpl, workFlowStageStatusServiceImpl, ciServiceImpl) - workflowEventProcessorImpl, err := in.NewWorkflowEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowServiceImpl, cdWorkflowReadServiceImpl, cdWorkflowRunnerServiceImpl, cdWorkflowRunnerReadServiceImpl, workflowDagExecutorImpl, ciHandlerImpl, cdHandlerImpl, eventSimpleFactoryImpl, eventRESTClientImpl, triggerServiceImpl, deployedAppServiceImpl, webhookServiceImpl, validate, environmentVariables, cdWorkflowCommonServiceImpl, cdPipelineConfigServiceImpl, userDeploymentRequestServiceImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, cdWorkflowRepositoryImpl, deploymentConfigServiceImpl) + workflowEventProcessorImpl, err := in.NewWorkflowEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowServiceImpl, cdWorkflowReadServiceImpl, cdWorkflowRunnerServiceImpl, cdWorkflowRunnerReadServiceImpl, workflowDagExecutorImpl, ciHandlerImpl, cdHandlerImpl, eventSimpleFactoryImpl, eventRESTClientImpl, triggerServiceImpl, deployedAppServiceImpl, webhookServiceImpl, validate, environmentVariables, cdWorkflowCommonServiceImpl, cdPipelineConfigServiceImpl, userDeploymentRequestServiceImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, cdWorkflowRepositoryImpl, deploymentConfigServiceImpl, serviceImpl) if err != nil { return nil, err } From eb5eeeb6d4c9b8ef4992c2ca4291292e4a192bd5 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Thu, 3 Apr 2025 00:45:42 +0530 Subject: [PATCH 06/37] feat: added ucid validation for ci/cd workflow status update event --- App.go | 6 +- Wire.go | 5 +- .../configure/BuildPipelineRestHandler.go | 4 +- client/argocdServer/config/Config.go | 2 +- client/telemetry/TelemetryEventClient.go | 209 ++++++------------ .../telemetry/TelemetryEventClientExtended.go | 77 +------ client/telemetry/bean.go | 139 ++++++++++++ .../telemetry/{ => posthog}/PosthogClient.go | 33 +-- client/telemetry/posthog/bean.go | 34 +++ cmd/external-app/externalApp.go | 5 +- cmd/external-app/wire.go | 5 +- cmd/external-app/wire_gen.go | 7 +- go.mod | 8 +- go.sum | 12 +- .../deployment/InstalledAppArgoCdService.go | 4 +- pkg/chartRepo/ChartRepositoryService.go | 2 +- pkg/cluster/ClusterService.go | 113 +++++----- pkg/cluster/bean/bean.go | 7 +- pkg/cluster/clusterUtil.go | 4 +- pkg/delete/DeleteService.go | 2 +- pkg/eventProcessor/bean/workflowEventBean.go | 6 + .../in/WorkflowEventProcessorService.go | 47 +++- pkg/gitops/GitOpsConfigService.go | 4 +- pkg/pipeline/CdHandler.go | 7 +- pkg/pipeline/CiHandler.go | 13 +- pkg/pipeline/WorkflowService.go | 13 +- pkg/pipeline/bean/WorkflowTemplate.go | 9 +- pkg/pipeline/executors/WorkflowUtils.go | 9 +- pkg/ucid/bean.go | 24 ++ pkg/ucid/ucid.go | 92 ++++++++ pkg/ucid/wire_ucid.go | 8 + util/helper.go | 10 +- .../devtron-labs/common-lib/informer/bean.go | 40 ++++ vendor/github.com/golang-jwt/jwt/v4/parser.go | 36 ++- vendor/modules.txt | 12 +- wire_gen.go | 21 +- 36 files changed, 665 insertions(+), 364 deletions(-) create mode 100644 client/telemetry/bean.go rename client/telemetry/{ => posthog}/PosthogClient.go (71%) create mode 100644 client/telemetry/posthog/bean.go create mode 100644 pkg/ucid/bean.go create mode 100644 pkg/ucid/ucid.go create mode 100644 pkg/ucid/wire_ucid.go create mode 100644 vendor/github.com/devtron-labs/common-lib/informer/bean.go diff --git a/App.go b/App.go index 08b240a8b7..37c38ed9d2 100644 --- a/App.go +++ b/App.go @@ -23,6 +23,7 @@ import ( "fmt" "github.com/devtron-labs/common-lib/middlewares" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" + "github.com/devtron-labs/devtron/client/telemetry/posthog" "github.com/devtron-labs/devtron/pkg/eventProcessor" "github.com/devtron-labs/devtron/pkg/eventProcessor/in" "log" @@ -31,7 +32,6 @@ import ( "time" "github.com/devtron-labs/devtron/api/util" - "github.com/devtron-labs/devtron/client/telemetry" "github.com/devtron-labs/devtron/otel" "github.com/devtron-labs/devtron/pkg/auth/user" @@ -55,7 +55,7 @@ type App struct { EnforcerV2 *casbinv2.SyncedEnforcer server *http.Server db *pg.DB - posthogClient *telemetry.PosthogClient + posthogClient *posthog.PosthogClient // eventProcessor.CentralEventProcessor is used to register event processors centralEventProcessor *eventProcessor.CentralEventProcessor // do not remove this. // used for local dev only @@ -73,7 +73,7 @@ func NewApp(router *router.MuxRouter, enforcer *casbin.SyncedEnforcer, db *pg.DB, sessionManager2 *authMiddleware.SessionManager, - posthogClient *telemetry.PosthogClient, + posthogClient *posthog.PosthogClient, loggingMiddleware util.LoggingMiddleware, centralEventProcessor *eventProcessor.CentralEventProcessor, pubSubClient *pubsub.PubSubClientServiceImpl, diff --git a/Wire.go b/Wire.go index 3401d2d623..f7446f1573 100644 --- a/Wire.go +++ b/Wire.go @@ -97,6 +97,7 @@ import ( "github.com/devtron-labs/devtron/client/lens" "github.com/devtron-labs/devtron/client/proxy" "github.com/devtron-labs/devtron/client/telemetry" + "github.com/devtron-labs/devtron/client/telemetry/posthog" "github.com/devtron-labs/devtron/internal/sql/repository" app2 "github.com/devtron-labs/devtron/internal/sql/repository/app" appStatusRepo "github.com/devtron-labs/devtron/internal/sql/repository/appStatus" @@ -164,6 +165,7 @@ import ( resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" "github.com/devtron-labs/devtron/pkg/sql" + "github.com/devtron-labs/devtron/pkg/ucid" util3 "github.com/devtron-labs/devtron/pkg/util" "github.com/devtron-labs/devtron/pkg/variables" "github.com/devtron-labs/devtron/pkg/variables/parsers" @@ -692,7 +694,8 @@ func InitializeApp() (*App, error) { wire.Bind(new(router.TelemetryRouter), new(*router.TelemetryRouterImpl)), restHandler.NewTelemetryRestHandlerImpl, wire.Bind(new(restHandler.TelemetryRestHandler), new(*restHandler.TelemetryRestHandlerImpl)), - telemetry.NewPosthogClient, + posthog.NewPosthogClient, + ucid.WireSet, cloudProviderIdentifier.NewProviderIdentifierServiceImpl, wire.Bind(new(cloudProviderIdentifier.ProviderIdentifierService), new(*cloudProviderIdentifier.ProviderIdentifierServiceImpl)), diff --git a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go index 32448e0abf..cb730a93ab 100644 --- a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go @@ -25,6 +25,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/constants" "github.com/devtron-labs/devtron/pkg/build/artifacts/imageTagging" bean2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + bean3 "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" constants2 "github.com/devtron-labs/devtron/pkg/pipeline/constants" "github.com/devtron-labs/devtron/util/stringsUtil" "golang.org/x/exp/maps" @@ -36,7 +37,6 @@ import ( "github.com/devtron-labs/devtron/util/response/pagination" "github.com/gorilla/schema" - "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/devtron-labs/devtron/api/restHandler/common" "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/sql/repository" @@ -1500,7 +1500,7 @@ func (handler *PipelineConfigRestHandlerImpl) DeleteMaterial(w http.ResponseWrit func (handler *PipelineConfigRestHandlerImpl) HandleWorkflowWebhook(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) - var wfUpdateReq v1alpha1.WorkflowStatus + var wfUpdateReq bean3.CiCdStatus err := decoder.Decode(&wfUpdateReq) if err != nil { handler.Logger.Errorw("request err, HandleWorkflowWebhook", "err", err, "payload", wfUpdateReq) diff --git a/client/argocdServer/config/Config.go b/client/argocdServer/config/Config.go index 20b622c38e..2027a2257f 100644 --- a/client/argocdServer/config/Config.go +++ b/client/argocdServer/config/Config.go @@ -71,7 +71,7 @@ func (impl *ArgoCDConfigGetterImpl) GetGRPCConfig() (*bean.ArgoGRPCConfig, error } func (impl *ArgoCDConfigGetterImpl) GetK8sConfig() (*bean.ArgoK8sConfig, error) { - clusterBean, err := impl.clusterReadService.FindOne(bean2.DEFAULT_CLUSTER) + clusterBean, err := impl.clusterReadService.FindOne(bean2.DefaultCluster) if err != nil { impl.logger.Errorw("error in fetching cluster bean from db", "err", err) return nil, err diff --git a/client/telemetry/TelemetryEventClient.go b/client/telemetry/TelemetryEventClient.go index f32382a06b..5d3ba4ab25 100644 --- a/client/telemetry/TelemetryEventClient.go +++ b/client/telemetry/TelemetryEventClient.go @@ -24,12 +24,15 @@ import ( cloudProviderIdentifier "github.com/devtron-labs/common-lib/cloud-provider-identifier" "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/api/helm-app/gRPC" + posthogClient "github.com/devtron-labs/devtron/client/telemetry/posthog" installedAppReader "github.com/devtron-labs/devtron/pkg/appStore/installedApp/read" bean2 "github.com/devtron-labs/devtron/pkg/attributes/bean" "github.com/devtron-labs/devtron/pkg/auth/user/bean" bean3 "github.com/devtron-labs/devtron/pkg/cluster/bean" module2 "github.com/devtron-labs/devtron/pkg/module/bean" + ucidService "github.com/devtron-labs/devtron/pkg/ucid" cron3 "github.com/devtron-labs/devtron/util/cron" + "go.opentelemetry.io/otel" "net/http" "time" @@ -43,22 +46,13 @@ import ( util3 "github.com/devtron-labs/devtron/pkg/util" "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" - "github.com/patrickmn/go-cache" "github.com/posthog/posthog-go" "github.com/robfig/cron/v3" "github.com/tidwall/gjson" "go.uber.org/zap" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "k8s.io/api/core/v1" - v12 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/version" ) -const LOGIN_COUNT_CONST = "login-count" -const SKIPPED_ONBOARDING_CONST = "SkippedOnboarding" -const ADMIN_EMAIL_ID_CONST = "admin" - type TelemetryEventClientImpl struct { cron *cron.Cron logger *zap.SugaredLogger @@ -69,7 +63,8 @@ type TelemetryEventClientImpl struct { userService user2.UserService attributeRepo repository.AttributesRepository ssoLoginService sso.SSOLoginService - PosthogClient *PosthogClient + posthogClient *posthogClient.PosthogClient + ucid ucidService.Service moduleRepository moduleRepo.ModuleRepository serverDataStore *serverDataStore.ServerDataStore userAuditService user2.UserAuditService @@ -93,9 +88,9 @@ type TelemetryEventClient interface { func NewTelemetryEventClientImpl(logger *zap.SugaredLogger, client *http.Client, clusterService cluster.ClusterService, K8sUtil *k8s.K8sServiceImpl, aCDAuthConfig *util3.ACDAuthConfig, userService user2.UserService, attributeRepo repository.AttributesRepository, ssoLoginService sso.SSOLoginService, - PosthogClient *PosthogClient, moduleRepository moduleRepo.ModuleRepository, - serverDataStore *serverDataStore.ServerDataStore, userAuditService user2.UserAuditService, - helmAppClient gRPC.HelmAppClient, + posthog *posthogClient.PosthogClient, ucid ucidService.Service, + moduleRepository moduleRepo.ModuleRepository, serverDataStore *serverDataStore.ServerDataStore, + userAuditService user2.UserAuditService, helmAppClient gRPC.HelmAppClient, cloudProviderIdentifierService cloudProviderIdentifier.ProviderIdentifierService, cronLogger *cron3.CronLoggerImpl, installedAppReadService installedAppReader.InstalledAppReadServiceEA, envVariables *util.EnvironmentVariables) (*TelemetryEventClientImpl, error) { @@ -112,7 +107,8 @@ func NewTelemetryEventClientImpl(logger *zap.SugaredLogger, client *http.Client, userService: userService, attributeRepo: attributeRepo, ssoLoginService: ssoLoginService, - PosthogClient: PosthogClient, + posthogClient: posthog, + ucid: ucid, moduleRepository: moduleRepository, serverDataStore: serverDataStore, userAuditService: userAuditService, @@ -124,13 +120,13 @@ func NewTelemetryEventClientImpl(logger *zap.SugaredLogger, client *http.Client, } watcher.HeartbeatEventForTelemetry() - _, err := cron.AddFunc(SummaryCronExpr, watcher.SummaryEventForTelemetryEA) + _, err := cron.AddFunc(posthogClient.SummaryCronExpr, watcher.SummaryEventForTelemetryEA) if err != nil { logger.Errorw("error in starting summery event", "err", err) return nil, err } - _, err = cron.AddFunc(HeartbeatCronExpr, watcher.HeartbeatEventForTelemetry) + _, err = cron.AddFunc(posthogClient.HeartbeatCronExpr, watcher.HeartbeatEventForTelemetry) if err != nil { logger.Errorw("error in starting heartbeat event", "err", err) return nil, err @@ -155,59 +151,6 @@ func (impl *TelemetryEventClientImpl) StopCron() { impl.cron.Stop() } -type TelemetryEventEA struct { - UCID string `json:"ucid"` //unique client id - Timestamp time.Time `json:"timestamp"` - EventMessage string `json:"eventMessage,omitempty"` - EventType TelemetryEventType `json:"eventType"` - ServerVersion string `json:"serverVersion,omitempty"` - UserCount int `json:"userCount,omitempty"` - ClusterCount int `json:"clusterCount,omitempty"` - HostURL bool `json:"hostURL,omitempty"` - SSOLogin bool `json:"ssoLogin,omitempty"` - DevtronVersion string `json:"devtronVersion,omitempty"` - DevtronMode string `json:"devtronMode,omitempty"` - InstalledIntegrations []string `json:"installedIntegrations,omitempty"` - InstallFailedIntegrations []string `json:"installFailedIntegrations,omitempty"` - InstallTimedOutIntegrations []string `json:"installTimedOutIntegrations,omitempty"` - LastLoginTime time.Time `json:"LastLoginTime,omitempty"` - InstallingIntegrations []string `json:"installingIntegrations,omitempty"` - DevtronReleaseVersion string `json:"devtronReleaseVersion,omitempty"` - HelmAppAccessCounter string `json:"HelmAppAccessCounter,omitempty"` - HelmAppUpdateCounter string `json:"HelmAppUpdateCounter,omitempty"` - ChartStoreVisitCount string `json:"ChartStoreVisitCount,omitempty"` - SkippedOnboarding bool `json:"SkippedOnboarding"` - HelmChartSuccessfulDeploymentCount int `json:"helmChartSuccessfulDeploymentCount,omitempty"` - ExternalHelmAppClusterCount map[int32]int `json:"ExternalHelmAppClusterCount,omitempty"` - ClusterProvider string `json:"clusterProvider,omitempty"` -} - -const DevtronUniqueClientIdConfigMap = "devtron-ucid" -const DevtronUniqueClientIdConfigMapKey = "UCID" -const InstallEventKey = "installEvent" -const UIEventKey = "uiEventKey" - -type TelemetryEventType string - -const ( - Heartbeat TelemetryEventType = "Heartbeat" - InstallationStart TelemetryEventType = "InstallationStart" - InstallationInProgress TelemetryEventType = "InstallationInProgress" - InstallationInterrupt TelemetryEventType = "InstallationInterrupt" - InstallationSuccess TelemetryEventType = "InstallationSuccess" - InstallationFailure TelemetryEventType = "InstallationFailure" - UpgradeStart TelemetryEventType = "UpgradeStart" - UpgradeInProgress TelemetryEventType = "UpgradeInProgress" - UpgradeInterrupt TelemetryEventType = "UpgradeInterrupt" - UpgradeSuccess TelemetryEventType = "UpgradeSuccess" - UpgradeFailure TelemetryEventType = "UpgradeFailure" - Summary TelemetryEventType = "Summary" - InstallationApplicationError TelemetryEventType = "InstallationApplicationError" - DashboardAccessed TelemetryEventType = "DashboardAccessed" - DashboardLoggedIn TelemetryEventType = "DashboardLoggedIn" - SIG_TERM TelemetryEventType = "SIG_TERM" -) - func (impl *TelemetryEventClientImpl) SummaryDetailsForTelemetry() (cluster []bean3.ClusterBean, user []bean.UserInfo, k8sServerVersion *version.Info, hostURL bool, ssoSetup bool, HelmAppAccessCount string, ChartStoreVisitCount string, SkippedOnboarding bool, HelmAppUpdateCounter string, helmChartSuccessfulDeploymentCount int, ExternalHelmAppClusterCount map[int32]int) { @@ -297,9 +240,9 @@ func (impl *TelemetryEventClientImpl) SummaryDetailsForTelemetry() (cluster []be } //getting userData from emailId - userData, err := impl.userAttributesRepository.GetUserDataByEmailId(ADMIN_EMAIL_ID_CONST) + userData, err := impl.userAttributesRepository.GetUserDataByEmailId(AdminEmailIdConst) - SkippedOnboardingValue := gjson.Get(userData, SKIPPED_ONBOARDING_CONST).Str + SkippedOnboardingValue := gjson.Get(userData, SkippedOnboardingConst).Str if SkippedOnboardingValue == "true" { SkippedOnboarding = true @@ -326,13 +269,13 @@ func (impl *TelemetryEventClientImpl) SummaryEventForTelemetryEA() { func (impl *TelemetryEventClientImpl) SendSummaryEvent(eventType string) error { impl.logger.Infow("sending summary event", "eventType", eventType) - ucid, err := impl.getUCID() + ucid, err := impl.getUCIDAndCheckIsOptedOut(context.Background()) if err != nil { impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) return err } - if IsOptOut { + if posthogClient.IsOptOut { impl.logger.Warnw("client is opt-out for telemetry, there will be no events capture", "ucid", ucid) return err } @@ -404,13 +347,13 @@ func (impl *TelemetryEventClientImpl) EnqueuePostHog(ucid string, eventType Tele } func (impl *TelemetryEventClientImpl) SendGenericTelemetryEvent(eventType string, prop map[string]interface{}) error { - ucid, err := impl.getUCID() + ucid, err := impl.getUCIDAndCheckIsOptedOut(context.Background()) if err != nil { impl.logger.Errorw("exception caught inside telemetry generic event", "err", err) return nil } - if IsOptOut { + if posthogClient.IsOptOut { impl.logger.Warnw("client is opt-out for telemetry, there will be no events capture", "ucid", ucid) return nil } @@ -419,15 +362,15 @@ func (impl *TelemetryEventClientImpl) SendGenericTelemetryEvent(eventType string } func (impl *TelemetryEventClientImpl) EnqueueGenericPostHogEvent(ucid string, eventType string, prop map[string]interface{}) error { - if impl.PosthogClient.Client == nil { + if impl.posthogClient.Client == nil { impl.logger.Warn("no posthog client found, creating new") - client, err := impl.retryPosthogClient(PosthogApiKey, PosthogEndpoint) + client, err := impl.retryPosthogClient(posthogClient.PosthogApiKey, posthogClient.PosthogEndpoint) if err == nil { - impl.PosthogClient.Client = client + impl.posthogClient.Client = client } } - if impl.PosthogClient.Client != nil && !impl.globalEnvVariables.IsAirGapEnvironment { - err := impl.PosthogClient.Client.Enqueue(posthog.Capture{ + if impl.posthogClient.Client != nil && !impl.globalEnvVariables.IsAirGapEnvironment { + err := impl.posthogClient.Client.Enqueue(posthog.Capture{ DistinctId: ucid, Event: eventType, Properties: prop, @@ -441,12 +384,12 @@ func (impl *TelemetryEventClientImpl) EnqueueGenericPostHogEvent(ucid string, ev } func (impl *TelemetryEventClientImpl) HeartbeatEventForTelemetry() { - ucid, err := impl.getUCID() + ucid, err := impl.getUCIDAndCheckIsOptedOut(context.Background()) if err != nil { impl.logger.Errorw("exception caught inside telemetry heartbeat event", "err", err) return } - if IsOptOut { + if posthogClient.IsOptOut { impl.logger.Warnw("client is opt-out for telemetry, there will be no events capture", "ucid", ucid) return } @@ -488,21 +431,21 @@ func (impl *TelemetryEventClientImpl) HeartbeatEventForTelemetry() { } func (impl *TelemetryEventClientImpl) GetTelemetryMetaInfo() (*TelemetryMetaInfo, error) { - ucid, err := impl.getUCID() + ucid, err := impl.getUCIDAndCheckIsOptedOut(context.Background()) if err != nil { impl.logger.Errorw("exception while getting unique client id", "error", err) return nil, err } data := &TelemetryMetaInfo{ - Url: PosthogEndpoint, + Url: posthogClient.PosthogEndpoint, UCID: ucid, - ApiKey: PosthogEncodedApiKey, + ApiKey: posthogClient.PosthogEncodedApiKey, } return data, err } func (impl *TelemetryEventClientImpl) SendTelemetryInstallEventEA() (*TelemetryEventType, error) { - ucid, err := impl.getUCID() + ucid, err := impl.getUCIDAndCheckIsOptedOut(context.Background()) if err != nil { impl.logger.Errorw("exception while getting unique client id", "error", err) return nil, err @@ -546,15 +489,19 @@ func (impl *TelemetryEventClientImpl) SendTelemetryInstallEventEA() (*TelemetryE impl.logger.Errorw("Installation EventForTelemetry EA Mode, payload unmarshal error", "error", err) return nil, nil } - cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, DevtronUniqueClientIdConfigMap, client) + cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, ucidService.DevtronUniqueClientIdConfigMap, client) + if err != nil { + impl.logger.Errorw("Installation EventForTelemetry EA Mode, failed to get DevtronUniqueClientIdConfigMap", "error", err) + return nil, err + } datamap := cm.Data - installEventValue, installEventKeyExists := datamap[InstallEventKey] + installEventValue, installEventKeyExists := datamap[ucidService.InstallEventKey] if installEventKeyExists == false || installEventValue == "1" { err = impl.EnqueuePostHog(ucid, InstallationSuccess, prop) if err == nil { - datamap[InstallEventKey] = "2" + datamap[ucidService.InstallEventKey] = "2" cm.Data = datamap _, err = impl.K8sUtil.UpdateConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, cm, client) if err != nil { @@ -568,7 +515,7 @@ func (impl *TelemetryEventClientImpl) SendTelemetryInstallEventEA() (*TelemetryE } func (impl *TelemetryEventClientImpl) SendTelemetryDashboardAccessEvent() error { - ucid, err := impl.getUCID() + ucid, err := impl.getUCIDAndCheckIsOptedOut(context.Background()) if err != nil { impl.logger.Errorw("exception while getting unique client id", "error", err) return err @@ -612,19 +559,19 @@ func (impl *TelemetryEventClientImpl) SendTelemetryDashboardAccessEvent() error impl.logger.Errorw("DashboardAccessed EventForTelemetry, payload unmarshal error", "error", err) return err } - cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, DevtronUniqueClientIdConfigMap, client) + cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, ucidService.DevtronUniqueClientIdConfigMap, client) if err != nil { impl.logger.Errorw("DashboardAccessed EventForTelemetry,failed to get DevtronUniqueClientIdConfigMap", "error", err) return err } datamap := cm.Data - accessEventValue, installEventKeyExists := datamap[UIEventKey] + accessEventValue, installEventKeyExists := datamap[ucidService.UIEventKey] if installEventKeyExists == false || accessEventValue <= "1" { err = impl.EnqueuePostHog(ucid, DashboardAccessed, prop) if err == nil { - datamap[UIEventKey] = "2" + datamap[ucidService.UIEventKey] = "2" cm.Data = datamap _, err = impl.K8sUtil.UpdateConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, cm, client) if err != nil { @@ -638,7 +585,7 @@ func (impl *TelemetryEventClientImpl) SendTelemetryDashboardAccessEvent() error } func (impl *TelemetryEventClientImpl) SendTelemetryDashboardLoggedInEvent() error { - ucid, err := impl.getUCID() + ucid, err := impl.getUCIDAndCheckIsOptedOut(context.Background()) if err != nil { impl.logger.Errorw("exception while getting unique client id", "error", err) return err @@ -682,19 +629,19 @@ func (impl *TelemetryEventClientImpl) SendTelemetryDashboardLoggedInEvent() erro impl.logger.Errorw("DashboardLoggedIn EventForTelemetry, payload unmarshal error", "error", err) return err } - cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, DevtronUniqueClientIdConfigMap, client) + cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, ucidService.DevtronUniqueClientIdConfigMap, client) if err != nil { impl.logger.Errorw("DashboardLoggedIn EventForTelemetry,failed to get DevtronUniqueClientIdConfigMap", "error", err) return err } datamap := cm.Data - accessEventValue, installEventKeyExists := datamap[UIEventKey] + accessEventValue, installEventKeyExists := datamap[ucidService.UIEventKey] if installEventKeyExists == false || accessEventValue != "3" { err = impl.EnqueuePostHog(ucid, DashboardLoggedIn, prop) if err == nil { - datamap[UIEventKey] = "3" + datamap[ucidService.UIEventKey] = "3" cm.Data = datamap _, err = impl.K8sUtil.UpdateConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, cm, client) if err != nil { @@ -707,57 +654,29 @@ func (impl *TelemetryEventClientImpl) SendTelemetryDashboardLoggedInEvent() erro return nil } -type TelemetryMetaInfo struct { - Url string `json:"url,omitempty"` - UCID string `json:"ucid,omitempty"` - ApiKey string `json:"apiKey,omitempty"` -} - -func (impl *TelemetryEventClientImpl) getUCID() (string, error) { - ucid, found := impl.PosthogClient.cache.Get(DevtronUniqueClientIdConfigMapKey) - if found { - return ucid.(string), nil - } else { - client, err := impl.K8sUtil.GetClientForInCluster() - if err != nil { - impl.logger.Errorw("exception while getting unique client id", "error", err) - return "", err - } - - cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, DevtronUniqueClientIdConfigMap, client) - if errStatus, ok := status.FromError(err); !ok || errStatus.Code() == codes.NotFound || errStatus.Code() == codes.Unknown { - // if not found, create new cm - cm = &v1.ConfigMap{ObjectMeta: v12.ObjectMeta{Name: DevtronUniqueClientIdConfigMap}} - data := map[string]string{} - data[DevtronUniqueClientIdConfigMapKey] = util.Generate(16) // generate unique random number - data[InstallEventKey] = "1" // used in operator to detect event is install or upgrade - data[UIEventKey] = "1" - cm.Data = data - _, err = impl.K8sUtil.CreateConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, cm, client) - if err != nil { - impl.logger.Errorw("exception while getting unique client id", "error", err) - return "", err - } - } - dataMap := cm.Data - ucid = dataMap[DevtronUniqueClientIdConfigMapKey] - impl.PosthogClient.cache.Set(DevtronUniqueClientIdConfigMapKey, ucid, cache.DefaultExpiration) - if cm == nil { - impl.logger.Errorw("configmap not found while getting unique client id", "cm", cm) - return ucid.(string), err - } - flag, err := impl.checkForOptOut(ucid.(string)) +func (impl *TelemetryEventClientImpl) getUCIDAndCheckIsOptedOut(ctx context.Context) (string, error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TelemetryEventClientImpl.getUCIDAndCheckIsOptedOut") + defer span.End() + ucid, found, err := impl.ucid.GetUCID() + if err != nil { + impl.logger.Errorw("exception while getting unique client id", "error", err) + return "", err + } + if !found { + flag, err := impl.checkForOptOut(newCtx, ucid) if err != nil { impl.logger.Errorw("error sending event to posthog, failed check for opt-out", "err", err) return "", err } - IsOptOut = flag + posthogClient.IsOptOut = flag } - return ucid.(string), nil + return ucid, nil } -func (impl *TelemetryEventClientImpl) checkForOptOut(UCID string) (bool, error) { - decodedUrl, err := base64.StdEncoding.DecodeString(TelemetryOptOutApiBaseUrl) +func (impl *TelemetryEventClientImpl) checkForOptOut(ctx context.Context, UCID string) (bool, error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TelemetryEventClientImpl.checkForOptOut") + defer span.End() + decodedUrl, err := base64.StdEncoding.DecodeString(posthogClient.TelemetryOptOutApiBaseUrl) if err != nil { impl.logger.Errorw("check opt-out list failed, decode error", "err", err) return false, err @@ -765,13 +684,13 @@ func (impl *TelemetryEventClientImpl) checkForOptOut(UCID string) (bool, error) encodedUrl := string(decodedUrl) url := fmt.Sprintf("%s/%s", encodedUrl, UCID) - response, err := util.HttpRequest(url) + response, err := util.HttpRequest(newCtx, url) if err != nil { - impl.logger.Errorw("check opt-out list failed, rest api error", "err", err) + impl.logger.Errorw("check opt-out list failed, rest api error", "ucid", UCID, "err", err) return false, err } flag := response["result"].(bool) - return flag, err + return flag, nil } func (impl *TelemetryEventClientImpl) retryPosthogClient(PosthogApiKey string, PosthogEndpoint string) (posthog.Client, error) { @@ -783,7 +702,7 @@ func (impl *TelemetryEventClientImpl) retryPosthogClient(PosthogApiKey string, P return client, err } -// returns installedIntegrations, installFailedIntegrations, installTimedOutIntegrations, installingIntegrations +// buildIntegrationsList - returns installedIntegrations, installFailedIntegrations, installTimedOutIntegrations, installingIntegrations func (impl *TelemetryEventClientImpl) buildIntegrationsList() ([]string, []string, []string, []string, error) { impl.logger.Info("building integrations list for telemetry") diff --git a/client/telemetry/TelemetryEventClientExtended.go b/client/telemetry/TelemetryEventClientExtended.go index 830eb6ae52..1670a6b4db 100644 --- a/client/telemetry/TelemetryEventClientExtended.go +++ b/client/telemetry/TelemetryEventClientExtended.go @@ -17,10 +17,12 @@ package telemetry import ( + "context" "encoding/json" cloudProviderIdentifier "github.com/devtron-labs/common-lib/cloud-provider-identifier" util2 "github.com/devtron-labs/common-lib/utils/k8s" client "github.com/devtron-labs/devtron/api/helm-app/gRPC" + posthogClient "github.com/devtron-labs/devtron/client/telemetry/posthog" installedAppReader "github.com/devtron-labs/devtron/pkg/appStore/installedApp/read" "github.com/devtron-labs/devtron/pkg/auth/sso" user2 "github.com/devtron-labs/devtron/pkg/auth/user" @@ -33,6 +35,7 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" moduleRepo "github.com/devtron-labs/devtron/pkg/module/repo" serverDataStore "github.com/devtron-labs/devtron/pkg/server/store" + ucidService "github.com/devtron-labs/devtron/pkg/ucid" util3 "github.com/devtron-labs/devtron/pkg/util" cron3 "github.com/devtron-labs/devtron/util/cron" "net/http" @@ -50,8 +53,6 @@ import ( "go.uber.org/zap" ) -const AppsCount int = 50 - type TelemetryEventClientImplExtended struct { environmentService environment.EnvironmentService appListingRepository repository.AppListingRepository @@ -73,7 +74,7 @@ type TelemetryEventClientImplExtended struct { func NewTelemetryEventClientImplExtended(logger *zap.SugaredLogger, client *http.Client, clusterService cluster.ClusterService, K8sUtil *util2.K8sServiceImpl, aCDAuthConfig *util3.ACDAuthConfig, environmentService environment.EnvironmentService, userService user2.UserService, - appListingRepository repository.AppListingRepository, PosthogClient *PosthogClient, + appListingRepository repository.AppListingRepository, posthog *posthogClient.PosthogClient, ucid ucidService.Service, ciPipelineConfigReadService ciConfig.CiPipelineConfigReadService, pipelineRepository pipelineConfig.PipelineRepository, gitProviderRepository repository3.GitProviderRepository, attributeRepo repository.AttributesRepository, ssoLoginService sso.SSOLoginService, appRepository app.AppRepository, @@ -114,7 +115,8 @@ func NewTelemetryEventClientImplExtended(logger *zap.SugaredLogger, client *http userService: userService, attributeRepo: attributeRepo, ssoLoginService: ssoLoginService, - PosthogClient: PosthogClient, + posthogClient: posthog, + ucid: ucid, moduleRepository: moduleRepository, serverDataStore: serverDataStore, userAuditService: userAuditService, @@ -127,13 +129,13 @@ func NewTelemetryEventClientImplExtended(logger *zap.SugaredLogger, client *http } watcher.HeartbeatEventForTelemetry() - _, err := cron.AddFunc(SummaryCronExpr, watcher.SummaryEventForTelemetry) + _, err := cron.AddFunc(posthogClient.SummaryCronExpr, watcher.SummaryEventForTelemetry) if err != nil { logger.Errorw("error in starting summery event", "err", err) return nil, err } - _, err = cron.AddFunc(HeartbeatCronExpr, watcher.HeartbeatEventForTelemetry) + _, err = cron.AddFunc(posthogClient.HeartbeatCronExpr, watcher.HeartbeatEventForTelemetry) if err != nil { logger.Errorw("error in starting heartbeat event", "err", err) return nil, err @@ -141,65 +143,6 @@ func NewTelemetryEventClientImplExtended(logger *zap.SugaredLogger, client *http return watcher, err } -type TelemetryEventDto struct { - UCID string `json:"ucid"` //unique client id - Timestamp time.Time `json:"timestamp"` - EventMessage string `json:"eventMessage,omitempty"` - EventType TelemetryEventType `json:"eventType"` - ProdAppCount int `json:"prodAppCount,omitempty"` - NonProdAppCount int `json:"nonProdAppCount,omitempty"` - UserCount int `json:"userCount,omitempty"` - EnvironmentCount int `json:"environmentCount,omitempty"` - ClusterCount int `json:"clusterCount,omitempty"` - CiCreatedPerDay int `json:"ciCreatedPerDay"` - CdCreatedPerDay int `json:"cdCreatedPerDay"` - CiDeletedPerDay int `json:"ciDeletedPerDay"` - CdDeletedPerDay int `json:"cdDeletedPerDay"` - CiTriggeredPerDay int `json:"ciTriggeredPerDay"` - CdTriggeredPerDay int `json:"cdTriggeredPerDay"` - HelmChartCount int `json:"helmChartCount,omitempty"` - SecurityScanCountPerDay int `json:"securityScanCountPerDay,omitempty"` - GitAccountsCount int `json:"gitAccountsCount,omitempty"` - GitOpsCount int `json:"gitOpsCount,omitempty"` - RegistryCount int `json:"registryCount,omitempty"` - HostURL bool `json:"hostURL,omitempty"` - SSOLogin bool `json:"ssoLogin,omitempty"` - AppCount int `json:"appCount,omitempty"` - AppsWithGitRepoConfigured int `json:"appsWithGitRepoConfigured,omitempty"` - AppsWithDockerConfigured int `json:"appsWithDockerConfigured,omitempty"` - AppsWithDeploymentTemplateConfigured int `json:"appsWithDeploymentTemplateConfigured,omitempty"` - AppsWithCiPipelineConfigured int `json:"appsWithCiPipelineConfigured,omitempty"` - AppsWithCdPipelineConfigured int `json:"appsWithCdPipelineConfigured,omitempty"` - Build bool `json:"build,omitempty"` - Deployment bool `json:"deployment,omitempty"` - ServerVersion string `json:"serverVersion,omitempty"` - DevtronGitVersion string `json:"devtronGitVersion,omitempty"` - DevtronVersion string `json:"devtronVersion,omitempty"` - DevtronMode string `json:"devtronMode,omitempty"` - InstalledIntegrations []string `json:"installedIntegrations,omitempty"` - InstallFailedIntegrations []string `json:"installFailedIntegrations,omitempty"` - InstallTimedOutIntegrations []string `json:"installTimedOutIntegrations,omitempty"` - InstallingIntegrations []string `json:"installingIntegrations,omitempty"` - DevtronReleaseVersion string `json:"devtronReleaseVersion,omitempty"` - LastLoginTime time.Time `json:"LastLoginTime,omitempty"` - SelfDockerfileCount int `json:"selfDockerfileCount"` - ManagedDockerfileCount int `json:"managedDockerfileCount"` - BuildPackCount int `json:"buildPackCount"` - SelfDockerfileSuccessCount int `json:"selfDockerfileSuccessCount"` - SelfDockerfileFailureCount int `json:"selfDockerfileFailureCount"` - ManagedDockerfileSuccessCount int `json:"managedDockerfileSuccessCount"` - ManagedDockerfileFailureCount int `json:"managedDockerfileFailureCount"` - BuildPackSuccessCount int `json:"buildPackSuccessCount"` - BuildPackFailureCount int `json:"buildPackFailureCount"` - HelmAppAccessCounter string `json:"HelmAppAccessCounter,omitempty"` - ChartStoreVisitCount string `json:"ChartStoreVisitCount,omitempty"` - SkippedOnboarding bool `json:"SkippedOnboarding"` - HelmAppUpdateCounter string `json:"HelmAppUpdateCounter,omitempty"` - HelmChartSuccessfulDeploymentCount int `json:"helmChartSuccessfulDeploymentCount,omitempty"` - ExternalHelmAppClusterCount map[int32]int `json:"ExternalHelmAppClusterCount"` - ClusterProvider string `json:"clusterProvider,omitempty"` -} - func (impl *TelemetryEventClientImplExtended) SummaryEventForTelemetry() { err := impl.SendSummaryEvent(string(Summary)) if err != nil { @@ -209,13 +152,13 @@ func (impl *TelemetryEventClientImplExtended) SummaryEventForTelemetry() { func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string) error { impl.logger.Infow("sending summary event", "eventType", eventType) - ucid, err := impl.getUCID() + ucid, err := impl.getUCIDAndCheckIsOptedOut(context.Background()) if err != nil { impl.logger.Errorw("exception caught inside telemetry summary event while retrieving ucid", "err", err) return err } - if IsOptOut { + if posthogClient.IsOptOut { impl.logger.Warnw("client is opt-out for telemetry, there will be no events capture", "ucid", ucid) return err } diff --git a/client/telemetry/bean.go b/client/telemetry/bean.go new file mode 100644 index 0000000000..f3e95dd0e7 --- /dev/null +++ b/client/telemetry/bean.go @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2020-2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package telemetry + +import "time" + +type TelemetryMetaInfo struct { + Url string `json:"url,omitempty"` + UCID string `json:"ucid,omitempty"` + ApiKey string `json:"apiKey,omitempty"` +} + +const ( + SkippedOnboardingConst = "SkippedOnboarding" + AdminEmailIdConst = "admin" +) + +type TelemetryEventType string + +const ( + Heartbeat TelemetryEventType = "Heartbeat" + InstallationStart TelemetryEventType = "InstallationStart" + InstallationInProgress TelemetryEventType = "InstallationInProgress" + InstallationInterrupt TelemetryEventType = "InstallationInterrupt" + InstallationSuccess TelemetryEventType = "InstallationSuccess" + InstallationFailure TelemetryEventType = "InstallationFailure" + UpgradeStart TelemetryEventType = "UpgradeStart" + UpgradeInProgress TelemetryEventType = "UpgradeInProgress" + UpgradeInterrupt TelemetryEventType = "UpgradeInterrupt" + UpgradeSuccess TelemetryEventType = "UpgradeSuccess" + UpgradeFailure TelemetryEventType = "UpgradeFailure" + Summary TelemetryEventType = "Summary" + InstallationApplicationError TelemetryEventType = "InstallationApplicationError" + DashboardAccessed TelemetryEventType = "DashboardAccessed" + DashboardLoggedIn TelemetryEventType = "DashboardLoggedIn" + SIG_TERM TelemetryEventType = "SIG_TERM" +) + +type TelemetryEventEA struct { + UCID string `json:"ucid"` //unique client id + Timestamp time.Time `json:"timestamp"` + EventMessage string `json:"eventMessage,omitempty"` + EventType TelemetryEventType `json:"eventType"` + ServerVersion string `json:"serverVersion,omitempty"` + UserCount int `json:"userCount,omitempty"` + ClusterCount int `json:"clusterCount,omitempty"` + HostURL bool `json:"hostURL,omitempty"` + SSOLogin bool `json:"ssoLogin,omitempty"` + DevtronVersion string `json:"devtronVersion,omitempty"` + DevtronMode string `json:"devtronMode,omitempty"` + InstalledIntegrations []string `json:"installedIntegrations,omitempty"` + InstallFailedIntegrations []string `json:"installFailedIntegrations,omitempty"` + InstallTimedOutIntegrations []string `json:"installTimedOutIntegrations,omitempty"` + LastLoginTime time.Time `json:"LastLoginTime,omitempty"` + InstallingIntegrations []string `json:"installingIntegrations,omitempty"` + DevtronReleaseVersion string `json:"devtronReleaseVersion,omitempty"` + HelmAppAccessCounter string `json:"HelmAppAccessCounter,omitempty"` + HelmAppUpdateCounter string `json:"HelmAppUpdateCounter,omitempty"` + ChartStoreVisitCount string `json:"ChartStoreVisitCount,omitempty"` + SkippedOnboarding bool `json:"SkippedOnboarding"` + HelmChartSuccessfulDeploymentCount int `json:"helmChartSuccessfulDeploymentCount,omitempty"` + ExternalHelmAppClusterCount map[int32]int `json:"ExternalHelmAppClusterCount,omitempty"` + ClusterProvider string `json:"clusterProvider,omitempty"` +} + +const AppsCount int = 50 + +type TelemetryEventDto struct { + UCID string `json:"ucid"` //unique client id + Timestamp time.Time `json:"timestamp"` + EventMessage string `json:"eventMessage,omitempty"` + EventType TelemetryEventType `json:"eventType"` + ProdAppCount int `json:"prodAppCount,omitempty"` + NonProdAppCount int `json:"nonProdAppCount,omitempty"` + UserCount int `json:"userCount,omitempty"` + EnvironmentCount int `json:"environmentCount,omitempty"` + ClusterCount int `json:"clusterCount,omitempty"` + CiCreatedPerDay int `json:"ciCreatedPerDay"` + CdCreatedPerDay int `json:"cdCreatedPerDay"` + CiDeletedPerDay int `json:"ciDeletedPerDay"` + CdDeletedPerDay int `json:"cdDeletedPerDay"` + CiTriggeredPerDay int `json:"ciTriggeredPerDay"` + CdTriggeredPerDay int `json:"cdTriggeredPerDay"` + HelmChartCount int `json:"helmChartCount,omitempty"` + SecurityScanCountPerDay int `json:"securityScanCountPerDay,omitempty"` + GitAccountsCount int `json:"gitAccountsCount,omitempty"` + GitOpsCount int `json:"gitOpsCount,omitempty"` + RegistryCount int `json:"registryCount,omitempty"` + HostURL bool `json:"hostURL,omitempty"` + SSOLogin bool `json:"ssoLogin,omitempty"` + AppCount int `json:"appCount,omitempty"` + AppsWithGitRepoConfigured int `json:"appsWithGitRepoConfigured,omitempty"` + AppsWithDockerConfigured int `json:"appsWithDockerConfigured,omitempty"` + AppsWithDeploymentTemplateConfigured int `json:"appsWithDeploymentTemplateConfigured,omitempty"` + AppsWithCiPipelineConfigured int `json:"appsWithCiPipelineConfigured,omitempty"` + AppsWithCdPipelineConfigured int `json:"appsWithCdPipelineConfigured,omitempty"` + Build bool `json:"build,omitempty"` + Deployment bool `json:"deployment,omitempty"` + ServerVersion string `json:"serverVersion,omitempty"` + DevtronGitVersion string `json:"devtronGitVersion,omitempty"` + DevtronVersion string `json:"devtronVersion,omitempty"` + DevtronMode string `json:"devtronMode,omitempty"` + InstalledIntegrations []string `json:"installedIntegrations,omitempty"` + InstallFailedIntegrations []string `json:"installFailedIntegrations,omitempty"` + InstallTimedOutIntegrations []string `json:"installTimedOutIntegrations,omitempty"` + InstallingIntegrations []string `json:"installingIntegrations,omitempty"` + DevtronReleaseVersion string `json:"devtronReleaseVersion,omitempty"` + LastLoginTime time.Time `json:"LastLoginTime,omitempty"` + SelfDockerfileCount int `json:"selfDockerfileCount"` + ManagedDockerfileCount int `json:"managedDockerfileCount"` + BuildPackCount int `json:"buildPackCount"` + SelfDockerfileSuccessCount int `json:"selfDockerfileSuccessCount"` + SelfDockerfileFailureCount int `json:"selfDockerfileFailureCount"` + ManagedDockerfileSuccessCount int `json:"managedDockerfileSuccessCount"` + ManagedDockerfileFailureCount int `json:"managedDockerfileFailureCount"` + BuildPackSuccessCount int `json:"buildPackSuccessCount"` + BuildPackFailureCount int `json:"buildPackFailureCount"` + HelmAppAccessCounter string `json:"HelmAppAccessCounter,omitempty"` + ChartStoreVisitCount string `json:"ChartStoreVisitCount,omitempty"` + SkippedOnboarding bool `json:"SkippedOnboarding"` + HelmAppUpdateCounter string `json:"HelmAppUpdateCounter,omitempty"` + HelmChartSuccessfulDeploymentCount int `json:"helmChartSuccessfulDeploymentCount,omitempty"` + ExternalHelmAppClusterCount map[int32]int `json:"ExternalHelmAppClusterCount"` + ClusterProvider string `json:"clusterProvider,omitempty"` +} diff --git a/client/telemetry/PosthogClient.go b/client/telemetry/posthog/PosthogClient.go similarity index 71% rename from client/telemetry/PosthogClient.go rename to client/telemetry/posthog/PosthogClient.go index 565470dc5c..7e6213864f 100644 --- a/client/telemetry/PosthogClient.go +++ b/client/telemetry/posthog/PosthogClient.go @@ -14,11 +14,13 @@ * limitations under the License. */ -package telemetry +package posthog import ( + "context" "encoding/base64" "encoding/json" + "go.opentelemetry.io/otel" "time" "github.com/devtron-labs/devtron/util" @@ -32,26 +34,13 @@ type PosthogClient struct { cache *cache.Cache } -var ( - PosthogApiKey string = "" - PosthogEndpoint string = "https://app.posthog.com" - SummaryCronExpr string = "0 0 * * *" // Run once a day, midnight - HeartbeatCronExpr string = "0 0/6 * * *" - CacheExpiry int = 1440 - PosthogEncodedApiKey string = "" - IsOptOut bool = false -) - -const ( - TelemetryApiKeyEndpoint string = "aHR0cHM6Ly90ZWxlbWV0cnkuZGV2dHJvbi5haS9kZXZ0cm9uL3RlbGVtZXRyeS9wb3N0aG9nSW5mbw==" - TelemetryOptOutApiBaseUrl string = "aHR0cHM6Ly90ZWxlbWV0cnkuZGV2dHJvbi5haS9kZXZ0cm9uL3RlbGVtZXRyeS9vcHQtb3V0" - ResponseApiKey string = "PosthogApiKey" - ResponseUrlKey string = "PosthogEndpoint" -) +func (p *PosthogClient) GetCache() *cache.Cache { + return p.cache +} func NewPosthogClient(logger *zap.SugaredLogger) (*PosthogClient, error) { if PosthogApiKey == "" { - encodedApiKey, apiKey, posthogUrl, err := getPosthogApiKey(TelemetryApiKeyEndpoint, logger) + encodedApiKey, apiKey, posthogUrl, err := getPosthogApiKey(context.Background(), TelemetryApiKeyEndpoint, logger) if err != nil { logger.Errorw("exception caught while getting api key", "err", err) } else { @@ -75,14 +64,16 @@ func NewPosthogClient(logger *zap.SugaredLogger) (*PosthogClient, error) { return pgClient, nil } -func getPosthogApiKey(encodedPosthogApiKeyUrl string, logger *zap.SugaredLogger) (string, string, string, error) { +func getPosthogApiKey(ctx context.Context, encodedPosthogApiKeyUrl string, logger *zap.SugaredLogger) (string, string, string, error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "telemetry.getPosthogApiKey") + defer span.End() decodedPosthogApiKeyUrl, err := base64.StdEncoding.DecodeString(encodedPosthogApiKeyUrl) if err != nil { logger.Errorw("error fetching posthog api key, decode error", "err", err) return "", "", "", err } apiKeyUrl := string(decodedPosthogApiKeyUrl) - response, err := util.HttpRequest(apiKeyUrl) + response, err := util.HttpRequest(newCtx, apiKeyUrl) if err != nil { logger.Errorw("error fetching posthog api key, http call", "err", err) return "", "", "", err @@ -94,7 +85,7 @@ func getPosthogApiKey(encodedPosthogApiKeyUrl string, logger *zap.SugaredLogger) return "", "", "", err } var datamap map[string]string - if err := json.Unmarshal(posthogInfoByte, &datamap); err != nil { + if err = json.Unmarshal(posthogInfoByte, &datamap); err != nil { logger.Errorw("error while unmarshal data", "err", err) return "", "", "", err } diff --git a/client/telemetry/posthog/bean.go b/client/telemetry/posthog/bean.go new file mode 100644 index 0000000000..8910b7d8fe --- /dev/null +++ b/client/telemetry/posthog/bean.go @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package posthog + +var ( + PosthogApiKey string = "" + PosthogEndpoint string = "https://app.posthog.com" + SummaryCronExpr string = "0 0 * * *" // Run once a day, midnight + HeartbeatCronExpr string = "0 0/6 * * *" + CacheExpiry int = 1440 + PosthogEncodedApiKey string = "" + IsOptOut bool = false +) + +const ( + TelemetryApiKeyEndpoint string = "aHR0cHM6Ly90ZWxlbWV0cnkuZGV2dHJvbi5haS9kZXZ0cm9uL3RlbGVtZXRyeS9wb3N0aG9nSW5mbw==" + TelemetryOptOutApiBaseUrl string = "aHR0cHM6Ly90ZWxlbWV0cnkuZGV2dHJvbi5haS9kZXZ0cm9uL3RlbGVtZXRyeS9vcHQtb3V0" + ResponseApiKey string = "PosthogApiKey" + ResponseUrlKey string = "PosthogEndpoint" +) diff --git a/cmd/external-app/externalApp.go b/cmd/external-app/externalApp.go index b7f90fb2dd..8d5ee62cb7 100644 --- a/cmd/external-app/externalApp.go +++ b/cmd/external-app/externalApp.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "github.com/devtron-labs/devtron/client/telemetry/posthog" "net/http" "os" "time" @@ -40,14 +41,14 @@ type App struct { Logger *zap.SugaredLogger server *http.Server telemetry telemetry.TelemetryEventClient - posthogClient *telemetry.PosthogClient + posthogClient *posthog.PosthogClient } func NewApp(db *pg.DB, sessionManager *authMiddleware.SessionManager, MuxRouter *MuxRouter, telemetry telemetry.TelemetryEventClient, - posthogClient *telemetry.PosthogClient, + posthogClient *posthog.PosthogClient, Logger *zap.SugaredLogger) *App { return &App{ db: db, diff --git a/cmd/external-app/wire.go b/cmd/external-app/wire.go index 4d05bdbbda..4dc491cd7f 100644 --- a/cmd/external-app/wire.go +++ b/cmd/external-app/wire.go @@ -60,6 +60,7 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer/session" "github.com/devtron-labs/devtron/client/dashboard" "github.com/devtron-labs/devtron/client/telemetry" + "github.com/devtron-labs/devtron/client/telemetry/posthog" "github.com/devtron-labs/devtron/internal/sql/repository" app2 "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/appStatus" @@ -86,6 +87,7 @@ import ( "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool" security2 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" "github.com/devtron-labs/devtron/pkg/sql" + "github.com/devtron-labs/devtron/pkg/ucid" util2 "github.com/devtron-labs/devtron/pkg/util" util3 "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/commonEnforcementFunctionsUtil" @@ -131,7 +133,8 @@ func InitializeApp() (*App, error) { util.NewSugardLogger, util.IntValidator, util2.GetACDAuthConfig, - telemetry.NewPosthogClient, + posthog.NewPosthogClient, + ucid.WireSet, delete2.NewDeleteServiceImpl, gitMaterial.GitMaterialWireSet, scanTool.ScanToolWireSet, diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index b0a26de083..52ff9e5076 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -52,6 +52,7 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer/repoCredsK8sClient" "github.com/devtron-labs/devtron/client/dashboard" "github.com/devtron-labs/devtron/client/telemetry" + "github.com/devtron-labs/devtron/client/telemetry/posthog" repository5 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/appStatus" @@ -129,6 +130,7 @@ import ( "github.com/devtron-labs/devtron/pkg/team/read" repository2 "github.com/devtron-labs/devtron/pkg/team/repository" "github.com/devtron-labs/devtron/pkg/terminal" + "github.com/devtron-labs/devtron/pkg/ucid" "github.com/devtron-labs/devtron/pkg/userResource" util3 "github.com/devtron-labs/devtron/pkg/util" "github.com/devtron-labs/devtron/pkg/webhook/helm" @@ -384,12 +386,13 @@ func InitializeApp() (*App, error) { chartProviderRouterImpl := chartProvider2.NewChartProviderRouterImpl(chartProviderRestHandlerImpl) dockerRegRestHandlerImpl := restHandler.NewDockerRegRestHandlerImpl(dockerRegistryConfigImpl, sugaredLogger, chartProviderServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, deleteServiceImpl) dockerRegRouterImpl := router.NewDockerRegRouterImpl(dockerRegRestHandlerImpl) - posthogClient, err := telemetry.NewPosthogClient(sugaredLogger) + posthogClient, err := posthog.NewPosthogClient(sugaredLogger) if err != nil { return nil, err } + serviceImpl := ucid.NewServiceImpl(sugaredLogger, posthogClient, k8sServiceImpl, acdAuthConfig) providerIdentifierServiceImpl := providerIdentifier.NewProviderIdentifierServiceImpl(sugaredLogger) - telemetryEventClientImpl, err := telemetry.NewTelemetryEventClientImpl(sugaredLogger, httpClient, clusterServiceImpl, k8sServiceImpl, acdAuthConfig, userServiceImpl, attributesRepositoryImpl, ssoLoginServiceImpl, posthogClient, moduleRepositoryImpl, serverDataStoreServerDataStore, userAuditServiceImpl, helmAppClientImpl, providerIdentifierServiceImpl, cronLoggerImpl, installedAppReadServiceEAImpl, environmentVariables) + telemetryEventClientImpl, err := telemetry.NewTelemetryEventClientImpl(sugaredLogger, httpClient, clusterServiceImpl, k8sServiceImpl, acdAuthConfig, userServiceImpl, attributesRepositoryImpl, ssoLoginServiceImpl, posthogClient, serviceImpl, moduleRepositoryImpl, serverDataStoreServerDataStore, userAuditServiceImpl, helmAppClientImpl, providerIdentifierServiceImpl, cronLoggerImpl, installedAppReadServiceEAImpl, environmentVariables) if err != nil { return nil, err } diff --git a/go.mod b/go.mod index baafc62c9e..c9ccd0dd10 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/go-pg/pg v6.15.1+incompatible github.com/go-resty/resty/v2 v2.7.0 github.com/gogo/protobuf v1.3.2 - github.com/golang-jwt/jwt/v4 v4.5.1 + github.com/golang-jwt/jwt/v4 v4.5.2 github.com/golang/mock v1.6.0 github.com/golang/protobuf v1.5.4 github.com/google/cel-go v0.17.8 @@ -306,9 +306,9 @@ require ( ) replace ( - github.com/argoproj/argo-workflows/v3 v3.5.13 => github.com/devtron-labs/argo-workflows/v3 v3.5.13 - github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250402105331-120c20ff29d9 - github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250402105331-120c20ff29d9 + github.com/argoproj/argo-workflows/v3 => github.com/devtron-labs/argo-workflows/v3 v3.5.13 + github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250402155428-3fa1ffc068e2 + github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250402155428-3fa1ffc068e2 github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127 github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5 k8s.io/api => k8s.io/api v0.29.7 diff --git a/go.sum b/go.sum index c12064620f..9b29696f0c 100644 --- a/go.sum +++ b/go.sum @@ -829,10 +829,10 @@ github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzq github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/devtron-labs/argo-workflows/v3 v3.5.13 h1:3pINq0gXOSeTw2z/vYe+j80lRpSN5Rp/8mfQORh8SmU= github.com/devtron-labs/argo-workflows/v3 v3.5.13/go.mod h1:/vqxcovDPT4zqr4DjR5v7CF8ggpY1l3TSa2CIG3jmjA= -github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250402105331-120c20ff29d9 h1:Imo0tZ6diaOwFzwJZWJkadEgrFO41gcOIGUmhYT4SUc= -github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250402105331-120c20ff29d9/go.mod h1:5lv4Wfj5ERhhvDGXe2IeES6qxjvUVCcohaRwKnWBMNo= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250402105331-120c20ff29d9 h1:sMbW7oI9AL40MH9jcz4S4szyGs3LwmgcIU0UrReRZbw= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250402105331-120c20ff29d9/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= +github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250402155428-3fa1ffc068e2 h1:Ik3LiAVccf6gqrbEwryD/uPYbSV/i7z4UTvYcLgM8jk= +github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250402155428-3fa1ffc068e2/go.mod h1:FfaLDXN1ZXxyRpnskBqVIYkpkWDCzBmDgIO9xqLnxdQ= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250402155428-3fa1ffc068e2 h1:k2bWvWk6if6I7k+URmrh1Dp2fleNDNu/pl+ZnImeUGE= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250402155428-3fa1ffc068e2/go.mod h1:zkNShlkcHxsmnL0gKNbs0uyRL8lZonGKr5Km63uTLI0= github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU= github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y= github.com/devtron-labs/protos v0.0.3-0.20250323220609-ecf8a0f7305e h1:U6UdYbW8a7xn5IzFPd8cywjVVPfutGJCudjePAfL/Hs= @@ -993,8 +993,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= -github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= diff --git a/pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppArgoCdService.go b/pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppArgoCdService.go index 9b637360f0..05c6b07ef2 100644 --- a/pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppArgoCdService.go +++ b/pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppArgoCdService.go @@ -25,7 +25,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" "github.com/devtron-labs/devtron/internal/util" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" - cluster2 "github.com/devtron-labs/devtron/pkg/cluster" + bean2 "github.com/devtron-labs/devtron/pkg/cluster/bean" "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" util2 "github.com/devtron-labs/devtron/util" @@ -167,7 +167,7 @@ func (impl *FullModeDeploymentServiceImpl) DeleteACD(acdAppName string, ctx cont func (impl *FullModeDeploymentServiceImpl) createInArgo(ctx context.Context, chartGitAttribute *commonBean.ChartGitAttribute, envModel bean.EnvironmentBean, argocdAppName string) error { appNamespace := envModel.Namespace if appNamespace == "" { - appNamespace = cluster2.DEFAULT_NAMESPACE + appNamespace = bean2.DefaultNamespace } appReq := &argocdServer.AppTemplate{ ApplicationName: argocdAppName, diff --git a/pkg/chartRepo/ChartRepositoryService.go b/pkg/chartRepo/ChartRepositoryService.go index 1df2c48e43..8cdef0a9fd 100644 --- a/pkg/chartRepo/ChartRepositoryService.go +++ b/pkg/chartRepo/ChartRepositoryService.go @@ -501,7 +501,7 @@ func (impl *ChartRepositoryServiceImpl) ValidateAndUpdateChartRepo(request *Char } func (impl *ChartRepositoryServiceImpl) TriggerChartSyncManual(chartProviderConfig *ChartProviderConfig) error { - defaultClusterBean, err := impl.clusterReadService.FindOne(bean2.DEFAULT_CLUSTER) + defaultClusterBean, err := impl.clusterReadService.FindOne(bean2.DefaultCluster) if err != nil { impl.logger.Errorw("defaultClusterBean err, TriggerChartSyncManual", "err", err) return err diff --git a/pkg/cluster/ClusterService.go b/pkg/cluster/ClusterService.go index 9bab50066a..1b00c2f84f 100644 --- a/pkg/cluster/ClusterService.go +++ b/pkg/cluster/ClusterService.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "fmt" + informerBean "github.com/devtron-labs/common-lib/informer" "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/pkg/cluster/adapter" "github.com/devtron-labs/devtron/pkg/cluster/bean" @@ -28,7 +29,6 @@ import ( cronUtil "github.com/devtron-labs/devtron/util/cron" "github.com/robfig/cron/v3" "log" - "net/http" "net/url" "sync" "time" @@ -38,8 +38,8 @@ import ( casbin2 "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" repository3 "github.com/devtron-labs/devtron/pkg/auth/user/repository" "github.com/devtron-labs/devtron/pkg/k8s/informer" - errors1 "github.com/juju/errors" - "k8s.io/apimachinery/pkg/api/errors" + customErr "github.com/juju/errors" + k8sError "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes" @@ -56,18 +56,6 @@ import ( "go.uber.org/zap" ) -const ( - DEFAULT_CLUSTER = "default_cluster" - DEFAULT_NAMESPACE = "default" - CLUSTER_MODIFY_EVENT_SECRET_TYPE = "cluster.request/modify" - CLUSTER_ACTION_ADD = "add" - CLUSTER_ACTION_UPDATE = "update" - SECRET_FIELD_CLUSTER_ID = "cluster_id" - SECRET_FIELD_UPDATED_ON = "updated_on" - SECRET_FIELD_ACTION = "action" - TokenFilePath = "/var/run/secrets/kubernetes.io/serviceaccount/token" -) - type ClusterService interface { Save(parent context.Context, bean *bean.ClusterBean, userId int32) (*bean.ClusterBean, error) UpdateClusterDescription(bean *bean.ClusterBean, userId int32) error @@ -195,7 +183,7 @@ func (impl *ClusterServiceImpl) Save(parent context.Context, bean *bean.ClusterB if err != nil { if len(err.Error()) > 2000 { - err = errors.NewBadRequest("unable to connect to cluster") + err = k8sError.NewBadRequest("unable to connect to cluster") } return nil, err } @@ -248,16 +236,16 @@ func (impl *ClusterServiceImpl) Save(parent context.Context, bean *bean.ClusterB secretName := ParseSecretNameForKubelinkInformer(bean.Id) data := make(map[string][]byte) - data[SECRET_FIELD_CLUSTER_ID] = []byte(fmt.Sprintf("%v", bean.Id)) - data[SECRET_FIELD_ACTION] = []byte(CLUSTER_ACTION_ADD) - data[SECRET_FIELD_UPDATED_ON] = []byte(time.Now().String()) // this field will ensure that informer detects change as other fields can be constant even if cluster config changes - _, err = impl.K8sUtil.CreateSecret(DEFAULT_NAMESPACE, data, secretName, CLUSTER_MODIFY_EVENT_SECRET_TYPE, k8sClient, nil, nil) + data[informerBean.SecretFieldClusterId] = []byte(fmt.Sprintf("%v", bean.Id)) + data[informerBean.SecretFieldAction] = []byte(informerBean.ClusterActionAdd) + data[clusterBean.SecretFieldUpdatedOn] = []byte(time.Now().String()) // this field will ensure that informer detects change as other fields can be constant even if cluster config changes + // TODO Asutosh: Why not UPSERT ?? + _, err = impl.K8sUtil.CreateSecret(clusterBean.DefaultNamespace, data, secretName, informerBean.ClusterModifyEventSecretType, k8sClient, nil, nil) if err != nil { - impl.logger.Errorw("error in updating secret for informers") + impl.logger.Errorw("error in creating secret for informers", "secretName", secretName, "err", err) return bean, nil } - - return bean, err + return bean, nil } // UpdateClusterDescription is new api service logic to only update description, this should be done in cluster update operation only @@ -415,7 +403,7 @@ func (impl *ClusterServiceImpl) Update(ctx context.Context, bean *bean.ClusterBe } if bean.ServerUrl != model.ServerUrl || bean.InsecureSkipTLSVerify != model.InsecureSkipTlsVerify || dbConfigBearerToken != requestConfigBearerToken || dbConfigTlsKey != requestConfigTlsKey || dbConfigCertData != requestConfigCertData || dbConfigCAData != requestConfigCAData { - if bean.ClusterName == clusterBean.DEFAULT_CLUSTER { + if bean.ClusterName == clusterBean.DefaultCluster { impl.logger.Errorw("default_cluster is reserved by the system and cannot be updated, default_cluster", "name", bean.ClusterName) return nil, fmt.Errorf("default_cluster is reserved by the system and cannot be updated") } @@ -480,37 +468,47 @@ func (impl *ClusterServiceImpl) Update(ctx context.Context, bean *bean.ClusterBe impl.SyncNsInformer(bean) } impl.logger.Infow("saving secret for cluster informer") + if bean.HasConfigOrUrlChanged { + data := make(map[string][]byte) + data[informerBean.SecretFieldClusterId] = []byte(fmt.Sprintf("%v", bean.Id)) + data[informerBean.SecretFieldAction] = []byte(informerBean.ClusterActionUpdate) + data[clusterBean.SecretFieldUpdatedOn] = []byte(time.Now().String()) // this field will ensure that informer detects change as other fields can be constant even if cluster config changes + if err = impl.upsertClusterSecret(bean, data); err != nil { + impl.logger.Errorw("error upserting cluster secret", "data", data, "err", err) + // TODO Asutosh: why error is not propagated ?? + return bean, nil + } + } + return bean, nil +} + +func (impl *ClusterServiceImpl) upsertClusterSecret(bean *bean.ClusterBean, data map[string][]byte) error { k8sClient, err := impl.K8sUtil.GetCoreV1ClientInCluster() if err != nil { - return bean, nil + impl.logger.Errorw("error in getting k8s client", "err", err) + return err } - // below secret will act as an event for informer running on secret object in kubelink - if bean.HasConfigOrUrlChanged { - secretName := ParseSecretNameForKubelinkInformer(bean.Id) - secret, err := impl.K8sUtil.GetSecret(DEFAULT_NAMESPACE, secretName, k8sClient) - statusError, _ := err.(*errors.StatusError) - if err != nil && statusError.Status().Code != http.StatusNotFound { - impl.logger.Errorw("secret not found", "err", err) - return bean, nil + // below secret will act as an event for informer running on a secret object in kubelink and kubewatch + secretName := ParseSecretNameForKubelinkInformer(bean.Id) + secret, err := impl.K8sUtil.GetSecret(clusterBean.DefaultNamespace, secretName, k8sClient) + if err != nil && !k8sError.IsNotFound(err) { + impl.logger.Errorw("error in getting cluster secret", "secretName", secretName, "err", err) + return err + } else if k8sError.IsNotFound(err) { + _, err = impl.K8sUtil.CreateSecret(clusterBean.DefaultNamespace, data, secretName, informerBean.ClusterModifyEventSecretType, k8sClient, nil, nil) + if err != nil { + impl.logger.Errorw("error in creating secret for informers", "secretName", secretName, "err", err) + return err } - data := make(map[string][]byte) - data[SECRET_FIELD_CLUSTER_ID] = []byte(fmt.Sprintf("%v", bean.Id)) - data[SECRET_FIELD_ACTION] = []byte(CLUSTER_ACTION_UPDATE) - data[SECRET_FIELD_UPDATED_ON] = []byte(time.Now().String()) // this field will ensure that informer detects change as other fields can be constant even if cluster config changes - if secret == nil { - _, err = impl.K8sUtil.CreateSecret(DEFAULT_NAMESPACE, data, secretName, CLUSTER_MODIFY_EVENT_SECRET_TYPE, k8sClient, nil, nil) - if err != nil { - impl.logger.Errorw("error in creating secret for informers") - } - } else { - secret.Data = data - secret, err = impl.K8sUtil.UpdateSecret(DEFAULT_NAMESPACE, secret, k8sClient) - if err != nil { - impl.logger.Errorw("error in updating secret for informers") - } + } else { + secret.Data = data + secret, err = impl.K8sUtil.UpdateSecret(clusterBean.DefaultNamespace, secret, k8sClient) + if err != nil { + impl.logger.Errorw("error in updating secret for informers", "secretName", secretName, "err", err) + return err } } - return bean, nil + return nil } func (impl *ClusterServiceImpl) SyncNsInformer(bean *bean.ClusterBean) { @@ -609,8 +607,9 @@ func (impl *ClusterServiceImpl) DeleteFromDb(bean *bean.ClusterBean, userId int3 impl.logger.Errorw("error in getting in cluster k8s client", "err", err, "clusterName", bean.ClusterName) return "", nil } + // TODO Asutosh: why we maintain this duplicate code ?? secretName := ParseSecretNameForKubelinkInformer(bean.Id) - err = impl.K8sUtil.DeleteSecret(DEFAULT_NAMESPACE, secretName, k8sClient) + err = impl.K8sUtil.DeleteSecret(clusterBean.DefaultNamespace, secretName, k8sClient) impl.logger.Errorw("error in deleting secret", "error", err) return existingCluster.ClusterName, nil } @@ -621,7 +620,7 @@ func (impl *ClusterServiceImpl) CheckIfConfigIsValid(cluster *bean.ClusterBean) if err != nil { if _, ok := err.(*url.Error); ok { return fmt.Errorf("Incorrect server url : %v", err) - } else if statusError, ok := err.(*errors.StatusError); ok { + } else if statusError, ok := err.(*k8sError.StatusError); ok { if statusError != nil { errReason := statusError.ErrStatus.Reason var errMsg string @@ -850,16 +849,16 @@ func (impl *ClusterServiceImpl) ValidateKubeconfig(kubeConfig string) (map[strin err := json.Unmarshal([]byte(kubeConfig), &kubeConfigDataMap) if err != nil { impl.logger.Errorw("error in unmarshalling kubeConfig") - return nil, errors1.New("invalid kubeConfig found , " + err.Error()) + return nil, customErr.New("invalid kubeConfig found , " + err.Error()) } if kubeConfigDataMap["apiVersion"] == nil { impl.logger.Errorw("api version missing from kubeConfig") - return nil, errors1.New("api version missing from kubeConfig") + return nil, customErr.New("api version missing from kubeConfig") } if kubeConfigDataMap["kind"] == nil { impl.logger.Errorw("kind missing from kubeConfig") - return nil, errors1.New("kind missing from kubeConfig") + return nil, customErr.New("kind missing from kubeConfig") } gvk.Version = kubeConfigDataMap["apiVersion"].(string) @@ -909,7 +908,7 @@ func (impl *ClusterServiceImpl) ValidateKubeconfig(kubeConfig string) (map[strin clusterBeanObject.ErrorInConnecting = "cluster name missing from kubeconfig" } - if clusterBeanObject.ClusterName == clusterBean.DEFAULT_CLUSTER { + if clusterBeanObject.ClusterName == clusterBean.DefaultCluster { clusterBeanObject.ErrorInConnecting = "default_cluster is reserved by the system and cannot be updated" } @@ -1000,7 +999,7 @@ func (impl *ClusterServiceImpl) ValidateKubeconfig(kubeConfig string) (map[strin if len(ValidateObjects) == 0 { impl.logger.Errorw("No valid cluster object provided in kubeconfig for context", "context", kubeConfig) - return nil, errors1.New("No valid cluster object provided in kubeconfig for context") + return nil, customErr.New("No valid cluster object provided in kubeconfig for context") } else { return ValidateObjects, nil } @@ -1014,7 +1013,7 @@ func (impl *ClusterServiceImpl) GetAndUpdateConnectionStatusForOneCluster(k8sCli if err != nil { if _, ok := err.(*url.Error); ok { err = fmt.Errorf("Incorrect server url : %v", err) - } else if statusError, ok := err.(*errors.StatusError); ok { + } else if statusError, ok := err.(*k8sError.StatusError); ok { if statusError != nil { errReason := statusError.ErrStatus.Reason var errMsg string diff --git a/pkg/cluster/bean/bean.go b/pkg/cluster/bean/bean.go index 6548613adb..a2438a2b44 100644 --- a/pkg/cluster/bean/bean.go +++ b/pkg/cluster/bean/bean.go @@ -7,7 +7,7 @@ import ( const ( DefaultClusterId = 1 - DEFAULT_CLUSTER = "default_cluster" + DefaultCluster = "default_cluster" ) type PrometheusAuth struct { @@ -81,3 +81,8 @@ type DefaultClusterComponent struct { EnvName string `json:"envName"` Status string `json:"status"` } + +const ( + DefaultNamespace = "default" + SecretFieldUpdatedOn = "updated_on" +) diff --git a/pkg/cluster/clusterUtil.go b/pkg/cluster/clusterUtil.go index 102519ff19..48ed681400 100644 --- a/pkg/cluster/clusterUtil.go +++ b/pkg/cluster/clusterUtil.go @@ -3,9 +3,9 @@ package cluster import "fmt" const ( - SECRET_NAME = "cluster-event" + SecretName = "cluster-event" ) func ParseSecretNameForKubelinkInformer(clusterId int) string { - return fmt.Sprintf("%s-%d", SECRET_NAME, clusterId) + return fmt.Sprintf("%s-%d", SecretName, clusterId) } diff --git a/pkg/delete/DeleteService.go b/pkg/delete/DeleteService.go index e9a53b3922..278d4182d0 100644 --- a/pkg/delete/DeleteService.go +++ b/pkg/delete/DeleteService.go @@ -108,7 +108,7 @@ func (impl DeleteServiceImpl) DeleteClusterSecret(deleteRequest *bean2.ClusterBe return nil } secretName := cluster.ParseSecretNameForKubelinkInformer(deleteRequest.Id) - err = impl.K8sUtil.DeleteSecret(cluster.DEFAULT_NAMESPACE, secretName, k8sClient) + err = impl.K8sUtil.DeleteSecret(bean2.DefaultNamespace, secretName, k8sClient) return err } diff --git a/pkg/eventProcessor/bean/workflowEventBean.go b/pkg/eventProcessor/bean/workflowEventBean.go index 4b33b8cba4..ddf532fd72 100644 --- a/pkg/eventProcessor/bean/workflowEventBean.go +++ b/pkg/eventProcessor/bean/workflowEventBean.go @@ -19,6 +19,7 @@ package bean import ( "context" "encoding/json" + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/devtron-labs/common-lib/utils/registry" "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" @@ -114,3 +115,8 @@ type DevtronAppReleaseContextType struct { CancelContext context.CancelCauseFunc RunnerId int } + +type CiCdStatus struct { + DevtronAdministratorInstance string `json:"devtronAdministratorInstance"` + *v1alpha1.WorkflowStatus +} diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index d91bd56caf..45ed82b9c2 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -18,6 +18,7 @@ package in import ( "context" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -46,6 +47,7 @@ import ( eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/out/bean" "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/executors" + "github.com/devtron-labs/devtron/pkg/ucid" "github.com/devtron-labs/devtron/pkg/workflow/cd" "github.com/devtron-labs/devtron/pkg/workflow/cd/adapter" cdWorkflowBean "github.com/devtron-labs/devtron/pkg/workflow/cd/bean" @@ -86,6 +88,7 @@ type WorkflowEventProcessorImpl struct { cdWorkflowCommonService cd.CdWorkflowCommonService cdPipelineConfigService pipeline.CdPipelineConfigService userDeploymentRequestService service.UserDeploymentRequestService + ucid ucid.Service devtronAppReleaseContextMap map[int]bean.DevtronAppReleaseContextType devtronAppReleaseContextMapLock *sync.Mutex @@ -115,6 +118,7 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, cdWorkflowCommonService cd.CdWorkflowCommonService, cdPipelineConfigService pipeline.CdPipelineConfigService, userDeploymentRequestService service.UserDeploymentRequestService, + ucid ucid.Service, pipelineRepository pipelineConfig.PipelineRepository, ciArtifactRepository repository.CiArtifactRepository, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, @@ -139,6 +143,7 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, cdWorkflowCommonService: cdWorkflowCommonService, cdPipelineConfigService: cdPipelineConfigService, userDeploymentRequestService: userDeploymentRequestService, + ucid: ucid, devtronAppReleaseContextMap: make(map[int]bean.DevtronAppReleaseContextType), devtronAppReleaseContextMapLock: &sync.Mutex{}, pipelineRepository: pipelineRepository, @@ -387,13 +392,28 @@ func (impl *WorkflowEventProcessorImpl) SubscribeHibernateBulkAction() error { func (impl *WorkflowEventProcessorImpl) SubscribeCIWorkflowStatusUpdate() error { callback := func(msg *model.PubSubMsg) { - wfStatus := v1alpha1.WorkflowStatus{} + wfStatus := bean.CiCdStatus{} err := json.Unmarshal([]byte(msg.Data), &wfStatus) if err != nil { impl.logger.Errorw("error while unmarshalling wf status update", "err", err, "msg", msg.Data) return } - + if len(wfStatus.DevtronAdministratorInstance) != 0 { + devtronUCID, _, err := impl.ucid.GetUCID() + if err != nil { + impl.logger.Errorw("error in getting UCID", "err", err) + return + } + decodedUCID, err := base64.StdEncoding.DecodeString(wfStatus.DevtronAdministratorInstance) + if err != nil { + impl.logger.Errorw("error in decoding UCID", "err", err) + return + } + if string(decodedUCID) != devtronUCID { + impl.logger.Warnw("mis match in UCID. skipping...", "decodedUCID", string(decodedUCID), "devtronUCID", devtronUCID) + return + } + } err = impl.ciHandler.CheckAndReTriggerCI(wfStatus) if err != nil { impl.logger.Errorw("error in checking and re triggering ci", "err", err) @@ -410,7 +430,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCIWorkflowStatusUpdate() error // add required logging here var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) { - wfStatus := v1alpha1.WorkflowStatus{} + wfStatus := bean.CiCdStatus{} err := json.Unmarshal([]byte(msg.Data), &wfStatus) if err != nil { return "error while unmarshalling wf status update", []interface{}{"err", err, "msg", msg.Data} @@ -431,13 +451,28 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCIWorkflowStatusUpdate() error func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error { callback := func(msg *model.PubSubMsg) { - wfStatus := v1alpha1.WorkflowStatus{} + wfStatus := bean.CiCdStatus{} err := json.Unmarshal([]byte(msg.Data), &wfStatus) if err != nil { impl.logger.Error("Error while unmarshalling wfStatus json object", "error", err) return } - + if len(wfStatus.DevtronAdministratorInstance) != 0 { + devtronUCID, _, err := impl.ucid.GetUCID() + if err != nil { + impl.logger.Errorw("error in getting UCID", "err", err) + return + } + decodedUCID, err := base64.StdEncoding.DecodeString(wfStatus.DevtronAdministratorInstance) + if err != nil { + impl.logger.Errorw("error in decoding UCID", "err", err) + return + } + if string(decodedUCID) != devtronUCID { + impl.logger.Warnw("mis match in UCID. skipping...", "decodedUCID", string(decodedUCID), "devtronUCID", devtronUCID) + return + } + } wfrId, wfrStatus, stateChanged, err := impl.cdHandler.UpdateWorkflow(wfStatus) impl.logger.Debugw("UpdateWorkflow", "wfrId", wfrId, "wfrStatus", wfrStatus) if err != nil { @@ -492,7 +527,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error // add required logging here var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) { - wfStatus := v1alpha1.WorkflowStatus{} + wfStatus := bean.CiCdStatus{} err := json.Unmarshal([]byte(msg.Data), &wfStatus) if err != nil { return "error while unmarshalling wfStatus json object", []interface{}{"error", err} diff --git a/pkg/gitops/GitOpsConfigService.go b/pkg/gitops/GitOpsConfigService.go index a0776f75b3..f095136674 100644 --- a/pkg/gitops/GitOpsConfigService.go +++ b/pkg/gitops/GitOpsConfigService.go @@ -251,7 +251,7 @@ func (impl *GitOpsConfigServiceImpl) registerGitOpsClientConfig(ctx context.Cont } } else { - clusterBean, err := impl.clusterReadService.FindOne(bean2.DEFAULT_CLUSTER) + clusterBean, err := impl.clusterReadService.FindOne(bean2.DefaultCluster) if err != nil { return nil, err } @@ -538,7 +538,7 @@ func (impl *GitOpsConfigServiceImpl) patchGitOpsClientConfig(model *repository.G } } else { - clusterBean, err := impl.clusterReadService.FindOne(bean2.DEFAULT_CLUSTER) + clusterBean, err := impl.clusterReadService.FindOne(bean2.DefaultCluster) if err != nil { return err } diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index ba2618db49..e11c2ca76e 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -30,6 +30,7 @@ import ( bean3 "github.com/devtron-labs/devtron/pkg/cluster/bean" repository3 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" + bean6 "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" "github.com/devtron-labs/devtron/pkg/pipeline/constants" util2 "github.com/devtron-labs/devtron/pkg/pipeline/util" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" @@ -69,7 +70,7 @@ const ( ) type CdHandler interface { - UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, string, bool, error) + UpdateWorkflow(workflowStatus bean6.CiCdStatus) (int, string, bool, error) GetCdBuildHistory(appId int, environmentId int, pipelineId int, offset int, size int) ([]pipelineBean.CdWorkflowWithArtifact, error) GetRunningWorkflowLogs(environmentId int, pipelineId int, workflowId int) (*bufio.Reader, func() error, error) FetchCdWorkflowDetails(appId int, environmentId int, pipelineId int, buildId int) (types.WorkflowResponse, error) @@ -265,7 +266,7 @@ func (impl *CdHandlerImpl) handleForceAbortCaseForCdStage(workflowRunner *pipeli return nil } -func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, string, bool, error) { +func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus bean6.CiCdStatus) (int, string, bool, error) { wfStatusRs := impl.extractWorkfowStatus(workflowStatus) workflowName, status, podStatus, message, podName := wfStatusRs.WorkflowName, wfStatusRs.Status, wfStatusRs.PodStatus, wfStatusRs.Message, wfStatusRs.PodName impl.Logger.Debugw("cd update for ", "wf ", workflowName, "status", status) @@ -320,7 +321,7 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus return savedWorkflow.Id, savedWorkflow.Status, false, nil } -func (impl *CdHandlerImpl) extractWorkfowStatus(workflowStatus v1alpha1.WorkflowStatus) *types.WorkflowStatus { +func (impl *CdHandlerImpl) extractWorkfowStatus(workflowStatus bean6.CiCdStatus) *types.WorkflowStatus { workflowName := "" status := string(workflowStatus.Phase) podStatus := "Pending" diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 7cd3503bfa..fa7ee5d455 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -32,6 +32,7 @@ import ( bean5 "github.com/devtron-labs/devtron/pkg/cluster/bean" "github.com/devtron-labs/devtron/pkg/cluster/environment" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" + bean6 "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" constants2 "github.com/devtron-labs/devtron/pkg/pipeline/constants" util3 "github.com/devtron-labs/devtron/pkg/pipeline/util" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" @@ -74,7 +75,7 @@ import ( type CiHandler interface { HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) - CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error + CheckAndReTriggerCI(workflowStatus bean6.CiCdStatus) error FetchMaterialsByPipelineId(pipelineId int, showAll bool) ([]pipelineConfig.CiPipelineMaterialResponse, error) FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId int, gitMaterialId int, showAll bool) ([]pipelineConfig.CiPipelineMaterialResponse, error) FetchWorkflowDetails(appId int, pipelineId int, buildId int) (types.WorkflowResponse, error) @@ -88,7 +89,7 @@ type CiHandler interface { GetBuildHistory(pipelineId int, appId int, offset int, size int) ([]types.WorkflowResponse, error) DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) - UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, error) + UpdateWorkflow(workflowStatus bean6.CiCdStatus) (int, error) FetchCiStatusForTriggerView(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) FetchCiStatusForTriggerViewV1(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) @@ -171,7 +172,7 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline return cih } -func (impl *CiHandlerImpl) CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error { +func (impl *CiHandlerImpl) CheckAndReTriggerCI(workflowStatus bean6.CiCdStatus) error { //return if re-trigger feature is disabled if !impl.config.WorkflowRetriesEnabled() { @@ -1110,7 +1111,7 @@ func (impl *CiHandlerImpl) GetHistoricBuildLogs(workflowId int, ciWorkflow *pipe return resp, err } -func ExtractWorkflowStatus(workflowStatus v1alpha1.WorkflowStatus) (string, string, string, string, string, string) { +func ExtractWorkflowStatus(workflowStatus bean6.CiCdStatus) (string, string, string, string, string, string) { workflowName := "" status := string(workflowStatus.Phase) podStatus := "" @@ -1140,7 +1141,7 @@ func ExtractWorkflowStatus(workflowStatus v1alpha1.WorkflowStatus) (string, stri return workflowName, status, podStatus, message, logLocation, podName } -func (impl *CiHandlerImpl) extractPodStatusAndWorkflow(workflowStatus v1alpha1.WorkflowStatus) (string, string, *pipelineConfig.CiWorkflow, error) { +func (impl *CiHandlerImpl) extractPodStatusAndWorkflow(workflowStatus bean6.CiCdStatus) (string, string, *pipelineConfig.CiWorkflow, error) { workflowName, status, _, message, _, _ := ExtractWorkflowStatus(workflowStatus) if workflowName == "" { impl.Logger.Errorw("extract workflow status, invalid wf name", "workflowName", workflowName, "status", status, "message", message) @@ -1176,7 +1177,7 @@ func (impl *CiHandlerImpl) getRefWorkflowAndCiRetryCount(savedWorkflow *pipeline return retryCount, savedWorkflow, err } -func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, error) { +func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus bean6.CiCdStatus) (int, error) { workflowName, status, podStatus, message, _, podName := ExtractWorkflowStatus(workflowStatus) if workflowName == "" { impl.Logger.Errorw("extract workflow status, invalid wf name", "workflowName", workflowName, "status", status, "podStatus", podStatus, "message", message) diff --git a/pkg/pipeline/WorkflowService.go b/pkg/pipeline/WorkflowService.go index 3cc2215867..c51c1aab96 100644 --- a/pkg/pipeline/WorkflowService.go +++ b/pkg/pipeline/WorkflowService.go @@ -18,6 +18,7 @@ package pipeline import ( "context" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -38,6 +39,7 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders" "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders/infraGetters" "github.com/devtron-labs/devtron/pkg/pipeline/types" + "github.com/devtron-labs/devtron/pkg/ucid" "go.uber.org/zap" v12 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -72,6 +74,7 @@ type WorkflowServiceImpl struct { k8sUtil *k8s.K8sServiceImpl k8sCommonService k8s2.K8sCommonService infraProvider infraProviders.InfraProvider + ucid ucid.Service } // TODO: Move to bean @@ -85,7 +88,8 @@ func NewWorkflowServiceImpl(Logger *zap.SugaredLogger, k8sUtil *k8s.K8sServiceImpl, systemWorkflowExecutor executors.SystemWorkflowExecutor, k8sCommonService k8s2.K8sCommonService, - infraProvider infraProviders.InfraProvider) (*WorkflowServiceImpl, error) { + infraProvider infraProviders.InfraProvider, + ucid ucid.Service) (*WorkflowServiceImpl, error) { commonWorkflowService := &WorkflowServiceImpl{ Logger: Logger, ciCdConfig: ciCdConfig, @@ -97,6 +101,7 @@ func NewWorkflowServiceImpl(Logger *zap.SugaredLogger, systemWorkflowExecutor: systemWorkflowExecutor, k8sCommonService: k8sCommonService, infraProvider: infraProvider, + ucid: ucid, } restConfig, err := k8sUtil.GetK8sInClusterRestConfig() if err != nil { @@ -213,6 +218,12 @@ func (impl *WorkflowServiceImpl) createWorkflowTemplate(workflowRequest *types.W clusterConfig, err := impl.getClusterConfig(workflowRequest) workflowTemplate.ClusterConfig = clusterConfig workflowTemplate.WorkflowType = workflowRequest.GetWorkflowTypeForWorkflowRequest() + devtronUCID, _, err := impl.ucid.GetUCID() + if err != nil { + impl.Logger.Errorw("error in getting UCID", "err", err) + return bean3.WorkflowTemplate{}, err + } + workflowTemplate.DevtronInstanceUID = base64.StdEncoding.EncodeToString([]byte(devtronUCID)) return workflowTemplate, nil } diff --git a/pkg/pipeline/bean/WorkflowTemplate.go b/pkg/pipeline/bean/WorkflowTemplate.go index 55e91e565c..d39968fa2e 100644 --- a/pkg/pipeline/bean/WorkflowTemplate.go +++ b/pkg/pipeline/bean/WorkflowTemplate.go @@ -18,6 +18,7 @@ package bean import ( blob_storage "github.com/devtron-labs/common-lib/blob-storage" + informerBean "github.com/devtron-labs/common-lib/informer" "github.com/devtron-labs/devtron/api/bean" v1 "k8s.io/api/core/v1" v12 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -47,6 +48,7 @@ type WorkflowTemplate struct { RefPlugins []*RefPluginObject TerminationGracePeriod int WorkflowType string + DevtronInstanceUID string } const ( @@ -76,8 +78,11 @@ func (workflowTemplate *WorkflowTemplate) SetActiveDeadlineSeconds(timeout int64 } func (workflowTemplate *WorkflowTemplate) CreateObjectMetadata() *v12.ObjectMeta { - - workflowLabels := map[string]string{WorkflowGenerateNamePrefix: workflowTemplate.WorkflowNamePrefix} + workflowLabels := map[string]string{ + WorkflowGenerateNamePrefix: workflowTemplate.WorkflowNamePrefix, + informerBean.WorkflowTypeLabelKey: workflowTemplate.WorkflowType, + informerBean.DevtronAdministratorInstanceLabelKey: workflowTemplate.DevtronInstanceUID, + } switch workflowTemplate.WorkflowType { case CI_WORKFLOW_NAME: workflowLabels["devtron.ai/workflow-purpose"] = "ci" diff --git a/pkg/pipeline/executors/WorkflowUtils.go b/pkg/pipeline/executors/WorkflowUtils.go index 0c9f18e911..c812f761fe 100644 --- a/pkg/pipeline/executors/WorkflowUtils.go +++ b/pkg/pipeline/executors/WorkflowUtils.go @@ -21,6 +21,7 @@ import ( "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned" v1alpha12 "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned/typed/workflow/v1alpha1" + informerBean "github.com/devtron-labs/common-lib/informer" "github.com/devtron-labs/common-lib/utils" bean2 "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository" @@ -294,9 +295,9 @@ func CheckIfReTriggerRequired(status, message, workflowRunnerStatus string) bool func GetWorkflowLabelsForSystemExecutor(workflowTemplate bean.WorkflowTemplate) map[string]string { return map[string]string{ - DEVTRON_WORKFLOW_LABEL_KEY: DEVTRON_WORKFLOW_LABEL_VALUE, - "devtron.ai/purpose": "workflow", - "workflowType": workflowTemplate.WorkflowType, - bean.WorkflowGenerateNamePrefix: workflowTemplate.WorkflowNamePrefix, + DEVTRON_WORKFLOW_LABEL_KEY: DEVTRON_WORKFLOW_LABEL_VALUE, + "devtron.ai/purpose": "workflow", + informerBean.WorkflowTypeLabelKey: workflowTemplate.WorkflowType, + bean.WorkflowGenerateNamePrefix: workflowTemplate.WorkflowNamePrefix, } } diff --git a/pkg/ucid/bean.go b/pkg/ucid/bean.go new file mode 100644 index 0000000000..acd595ad57 --- /dev/null +++ b/pkg/ucid/bean.go @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ucid + +const ( + DevtronUniqueClientIdConfigMap = "devtron-ucid" + DevtronUniqueClientIdConfigMapKey = "UCID" + UIEventKey = "uiEventKey" + InstallEventKey = "installEvent" +) diff --git a/pkg/ucid/ucid.go b/pkg/ucid/ucid.go new file mode 100644 index 0000000000..6dcc5de475 --- /dev/null +++ b/pkg/ucid/ucid.go @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ucid + +import ( + "fmt" + "github.com/devtron-labs/common-lib/utils/k8s" + "github.com/devtron-labs/devtron/client/telemetry/posthog" + util3 "github.com/devtron-labs/devtron/pkg/util" + "github.com/devtron-labs/devtron/util" + "github.com/patrickmn/go-cache" + "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + v1 "k8s.io/api/core/v1" + v12 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type Service interface { + GetUCID() (string, bool, error) +} + +type ServiceImpl struct { + logger *zap.SugaredLogger + posthogClient *posthog.PosthogClient + K8sUtil *k8s.K8sServiceImpl + aCDAuthConfig *util3.ACDAuthConfig +} + +func NewServiceImpl( + logger *zap.SugaredLogger, + posthogClient *posthog.PosthogClient, + K8sUtil *k8s.K8sServiceImpl, + aCDAuthConfig *util3.ACDAuthConfig, +) *ServiceImpl { + return &ServiceImpl{ + logger: logger, + posthogClient: posthogClient, + K8sUtil: K8sUtil, + aCDAuthConfig: aCDAuthConfig, + } +} + +func (impl *ServiceImpl) GetUCID() (string, bool, error) { + ucid, found := impl.posthogClient.GetCache().Get(DevtronUniqueClientIdConfigMapKey) + if found { + return ucid.(string), found, nil + } else { + client, err := impl.K8sUtil.GetClientForInCluster() + if err != nil { + impl.logger.Errorw("exception while getting unique client id", "error", err) + return "", found, err + } + cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, DevtronUniqueClientIdConfigMap, client) + if errStatus, ok := status.FromError(err); !ok || errStatus.Code() == codes.NotFound || errStatus.Code() == codes.Unknown { + // if not found, create new cm + cm = &v1.ConfigMap{ObjectMeta: v12.ObjectMeta{Name: DevtronUniqueClientIdConfigMap}} + data := map[string]string{} + data[DevtronUniqueClientIdConfigMapKey] = util.Generate(16) // generate unique random number + data[InstallEventKey] = "1" // used in operator to detect event is install or upgrade + data[UIEventKey] = "1" + cm.Data = data + _, err = impl.K8sUtil.CreateConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, cm, client) + if err != nil { + impl.logger.Errorw("exception while getting unique client id", "error", err) + return "", found, err + } + } + if cm == nil { + impl.logger.Errorw("configmap not found while getting unique client id") + return ucid.(string), found, fmt.Errorf("configmap %q not found while getting unique client id", DevtronUniqueClientIdConfigMap) + } + dataMap := cm.Data + ucid = dataMap[DevtronUniqueClientIdConfigMapKey] + impl.posthogClient.GetCache().Set(DevtronUniqueClientIdConfigMapKey, ucid, cache.DefaultExpiration) + } + return ucid.(string), found, nil +} diff --git a/pkg/ucid/wire_ucid.go b/pkg/ucid/wire_ucid.go new file mode 100644 index 0000000000..f8f4402af2 --- /dev/null +++ b/pkg/ucid/wire_ucid.go @@ -0,0 +1,8 @@ +package ucid + +import "github.com/google/wire" + +var WireSet = wire.NewSet( + NewServiceImpl, + wire.Bind(new(Service), new(*ServiceImpl)), +) diff --git a/util/helper.go b/util/helper.go index c8d3e00c27..50fbbf91ab 100644 --- a/util/helper.go +++ b/util/helper.go @@ -25,8 +25,8 @@ import ( "fmt" "github.com/devtron-labs/devtron/internal/middleware" "github.com/juju/errors" + "go.opentelemetry.io/otel" "io" - "io/ioutil" "math/rand" "net/http" "os" @@ -139,8 +139,10 @@ func GenerateNewWorkflowName(name string) string { return fmt.Sprintf("%s-clone-%s", name, Generate(4)) } -func HttpRequest(url string) (map[string]interface{}, error) { - req, err := http.NewRequest(http.MethodGet, url, nil) +func HttpRequest(ctx context.Context, url string) (map[string]interface{}, error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "util.HttpRequest") + defer span.End() + req, err := http.NewRequestWithContext(newCtx, http.MethodGet, url, nil) if err != nil { return nil, err } @@ -151,7 +153,7 @@ func HttpRequest(url string) (map[string]interface{}, error) { return nil, err } if res.StatusCode >= 200 && res.StatusCode <= 299 { - resBody, err := ioutil.ReadAll(res.Body) + resBody, err := io.ReadAll(res.Body) if err != nil { return nil, err } diff --git a/vendor/github.com/devtron-labs/common-lib/informer/bean.go b/vendor/github.com/devtron-labs/common-lib/informer/bean.go new file mode 100644 index 0000000000..e3b0304328 --- /dev/null +++ b/vendor/github.com/devtron-labs/common-lib/informer/bean.go @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package informer + +const ( + ClusterModifyEventSecretType = "cluster.request/modify" + ClusterActionAdd = "add" + ClusterActionUpdate = "update" + ClusterActionDelete = "delete" + SecretFieldAction = "action" + SecretFieldClusterId = "cluster_id" +) + +const ( + WorkflowTypeLabelKey = "workflowType" + CiWorkflowName = "ci" + CdWorkflowName = "cd" +) + +const ( + DevtronAdministratorInstanceLabelKey = "devtron.ai/administrator-instance" +) + +const ( + PodDeletedMessage = "pod deleted" +) diff --git a/vendor/github.com/golang-jwt/jwt/v4/parser.go b/vendor/github.com/golang-jwt/jwt/v4/parser.go index 9dd36e5a5a..0fc510a0aa 100644 --- a/vendor/github.com/golang-jwt/jwt/v4/parser.go +++ b/vendor/github.com/golang-jwt/jwt/v4/parser.go @@ -7,6 +7,8 @@ import ( "strings" ) +const tokenDelimiter = "." + type Parser struct { // If populated, only these methods will be considered valid. // @@ -122,9 +124,10 @@ func (p *Parser) ParseWithClaims(tokenString string, claims Claims, keyFunc Keyf // It's only ever useful in cases where you know the signature is valid (because it has // been checked previously in the stack) and you want to extract values from it. func (p *Parser) ParseUnverified(tokenString string, claims Claims) (token *Token, parts []string, err error) { - parts = strings.Split(tokenString, ".") - if len(parts) != 3 { - return nil, parts, NewValidationError("token contains an invalid number of segments", ValidationErrorMalformed) + var ok bool + parts, ok = splitToken(tokenString) + if !ok { + return nil, nil, NewValidationError("token contains an invalid number of segments", ValidationErrorMalformed) } token = &Token{Raw: tokenString} @@ -174,3 +177,30 @@ func (p *Parser) ParseUnverified(tokenString string, claims Claims) (token *Toke return token, parts, nil } + +// splitToken splits a token string into three parts: header, claims, and signature. It will only +// return true if the token contains exactly two delimiters and three parts. In all other cases, it +// will return nil parts and false. +func splitToken(token string) ([]string, bool) { + parts := make([]string, 3) + header, remain, ok := strings.Cut(token, tokenDelimiter) + if !ok { + return nil, false + } + parts[0] = header + claims, remain, ok := strings.Cut(remain, tokenDelimiter) + if !ok { + return nil, false + } + parts[1] = claims + // One more cut to ensure the signature is the last part of the token and there are no more + // delimiters. This avoids an issue where malicious input could contain additional delimiters + // causing unecessary overhead parsing tokens. + signature, _, unexpected := strings.Cut(remain, tokenDelimiter) + if unexpected { + return nil, false + } + parts[2] = signature + + return parts, true +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 2834d578cf..7084a09385 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -472,7 +472,7 @@ github.com/davecgh/go-spew/spew # github.com/deckarep/golang-set v1.8.0 ## explicit; go 1.17 github.com/deckarep/golang-set -# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250402105331-120c20ff29d9 +# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250402155428-3fa1ffc068e2 ## explicit; go 1.21 github.com/devtron-labs/authenticator/apiToken github.com/devtron-labs/authenticator/client @@ -480,7 +480,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250402105331-120c20ff29d9 +# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250402155428-3fa1ffc068e2 ## explicit; go 1.21 github.com/devtron-labs/common-lib/async github.com/devtron-labs/common-lib/blob-storage @@ -492,6 +492,7 @@ github.com/devtron-labs/common-lib/fetchAllEnv github.com/devtron-labs/common-lib/git-manager github.com/devtron-labs/common-lib/git-manager/util github.com/devtron-labs/common-lib/imageScan/bean +github.com/devtron-labs/common-lib/informer github.com/devtron-labs/common-lib/middlewares github.com/devtron-labs/common-lib/pubsub-lib github.com/devtron-labs/common-lib/pubsub-lib/metrics @@ -699,7 +700,7 @@ github.com/gogo/protobuf/jsonpb github.com/gogo/protobuf/proto github.com/gogo/protobuf/sortkeys github.com/gogo/protobuf/types -# github.com/golang-jwt/jwt/v4 v4.5.1 +# github.com/golang-jwt/jwt/v4 v4.5.2 ## explicit; go 1.16 github.com/golang-jwt/jwt/v4 # github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da @@ -2350,8 +2351,9 @@ xorm.io/xorm/log xorm.io/xorm/names xorm.io/xorm/schemas xorm.io/xorm/tags -# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250402105331-120c20ff29d9 -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250402105331-120c20ff29d9 +# github.com/argoproj/argo-workflows/v3 => github.com/devtron-labs/argo-workflows/v3 v3.5.13 +# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250402155428-3fa1ffc068e2 +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250402155428-3fa1ffc068e2 # github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127 # github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5 # k8s.io/api => k8s.io/api v0.29.7 diff --git a/wire_gen.go b/wire_gen.go index b8264c531a..de7e59d8ef 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -91,6 +91,7 @@ import ( "github.com/devtron-labs/devtron/client/lens" "github.com/devtron-labs/devtron/client/proxy" "github.com/devtron-labs/devtron/client/telemetry" + "github.com/devtron-labs/devtron/client/telemetry/posthog" repository2 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/appStatus" @@ -256,6 +257,7 @@ import ( read4 "github.com/devtron-labs/devtron/pkg/team/read" repository8 "github.com/devtron-labs/devtron/pkg/team/repository" "github.com/devtron-labs/devtron/pkg/terminal" + "github.com/devtron-labs/devtron/pkg/ucid" "github.com/devtron-labs/devtron/pkg/userResource" util3 "github.com/devtron-labs/devtron/pkg/util" "github.com/devtron-labs/devtron/pkg/variables" @@ -623,7 +625,12 @@ func InitializeApp() (*App, error) { } ciInfraGetter := ci.NewCiInfraGetter(sugaredLogger, infraConfigServiceImpl, infraConfigAuditServiceImpl) infraProviderImpl := infraProviders.NewInfraProviderImpl(sugaredLogger, infraGetter, ciInfraGetter) - workflowServiceImpl, err := pipeline.NewWorkflowServiceImpl(sugaredLogger, environmentRepositoryImpl, ciCdConfig, configReadServiceImpl, globalCMCSServiceImpl, argoWorkflowExecutorImpl, k8sServiceImpl, systemWorkflowExecutorImpl, k8sCommonServiceImpl, infraProviderImpl) + posthogClient, err := posthog.NewPosthogClient(sugaredLogger) + if err != nil { + return nil, err + } + serviceImpl := ucid.NewServiceImpl(sugaredLogger, posthogClient, k8sServiceImpl, acdAuthConfig) + workflowServiceImpl, err := pipeline.NewWorkflowServiceImpl(sugaredLogger, environmentRepositoryImpl, ciCdConfig, configReadServiceImpl, globalCMCSServiceImpl, argoWorkflowExecutorImpl, k8sServiceImpl, systemWorkflowExecutorImpl, k8sCommonServiceImpl, infraProviderImpl, serviceImpl) if err != nil { return nil, err } @@ -924,12 +931,8 @@ func InitializeApp() (*App, error) { ssoLoginServiceImpl := sso.NewSSOLoginServiceImpl(sugaredLogger, ssoLoginRepositoryImpl, k8sServiceImpl, environmentVariables, userAuthOidcHelperImpl) ssoLoginRestHandlerImpl := sso2.NewSsoLoginRestHandlerImpl(validate, sugaredLogger, enforcerImpl, userServiceImpl, ssoLoginServiceImpl) ssoLoginRouterImpl := sso2.NewSsoLoginRouterImpl(ssoLoginRestHandlerImpl) - posthogClient, err := telemetry.NewPosthogClient(sugaredLogger) - if err != nil { - return nil, err - } providerIdentifierServiceImpl := providerIdentifier.NewProviderIdentifierServiceImpl(sugaredLogger) - telemetryEventClientImplExtended, err := telemetry.NewTelemetryEventClientImplExtended(sugaredLogger, httpClient, clusterServiceImplExtended, k8sServiceImpl, acdAuthConfig, environmentServiceImpl, userServiceImpl, appListingRepositoryImpl, posthogClient, ciPipelineConfigReadServiceImpl, pipelineRepositoryImpl, gitProviderRepositoryImpl, attributesRepositoryImpl, ssoLoginServiceImpl, appRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, gitMaterialReadServiceImpl, ciTemplateRepositoryImpl, chartRepositoryImpl, userAuditServiceImpl, ciBuildConfigServiceImpl, moduleRepositoryImpl, serverDataStoreServerDataStore, helmAppClientImpl, installedAppReadServiceImpl, userAttributesRepositoryImpl, providerIdentifierServiceImpl, cronLoggerImpl, gitOpsConfigReadServiceImpl) + telemetryEventClientImplExtended, err := telemetry.NewTelemetryEventClientImplExtended(sugaredLogger, httpClient, clusterServiceImplExtended, k8sServiceImpl, acdAuthConfig, environmentServiceImpl, userServiceImpl, appListingRepositoryImpl, posthogClient, serviceImpl, ciPipelineConfigReadServiceImpl, pipelineRepositoryImpl, gitProviderRepositoryImpl, attributesRepositoryImpl, ssoLoginServiceImpl, appRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, gitMaterialReadServiceImpl, ciTemplateRepositoryImpl, chartRepositoryImpl, userAuditServiceImpl, ciBuildConfigServiceImpl, moduleRepositoryImpl, serverDataStoreServerDataStore, helmAppClientImpl, installedAppReadServiceImpl, userAttributesRepositoryImpl, providerIdentifierServiceImpl, cronLoggerImpl, gitOpsConfigReadServiceImpl) if err != nil { return nil, err } @@ -948,8 +951,8 @@ func InitializeApp() (*App, error) { webhookListenerRouterImpl := router.NewWebhookListenerRouterImpl(webhookEventHandlerImpl) appFilteringRestHandlerImpl := appList.NewAppFilteringRestHandlerImpl(sugaredLogger, teamServiceImpl, enforcerImpl, userServiceImpl, clusterServiceImplExtended, environmentServiceImpl, teamReadServiceImpl) appFilteringRouterImpl := appList2.NewAppFilteringRouterImpl(appFilteringRestHandlerImpl) - serviceImpl := resourceTree.NewServiceImpl(sugaredLogger, appListingServiceImpl, appStatusServiceImpl, argoApplicationServiceExtendedImpl, cdApplicationStatusUpdateHandlerImpl, helmAppReadServiceImpl, helmAppServiceImpl, k8sApplicationServiceImpl, k8sCommonServiceImpl, environmentReadServiceImpl) - appListingRestHandlerImpl := appList.NewAppListingRestHandlerImpl(appListingServiceImpl, enforcerImpl, pipelineBuilderImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, userServiceImpl, k8sCommonServiceImpl, installedAppDBExtendedServiceImpl, installedAppResourceServiceImpl, pipelineRepositoryImpl, k8sApplicationServiceImpl, deploymentConfigServiceImpl, serviceImpl) + resourceTreeServiceImpl := resourceTree.NewServiceImpl(sugaredLogger, appListingServiceImpl, appStatusServiceImpl, argoApplicationServiceExtendedImpl, cdApplicationStatusUpdateHandlerImpl, helmAppReadServiceImpl, helmAppServiceImpl, k8sApplicationServiceImpl, k8sCommonServiceImpl, environmentReadServiceImpl) + appListingRestHandlerImpl := appList.NewAppListingRestHandlerImpl(appListingServiceImpl, enforcerImpl, pipelineBuilderImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, userServiceImpl, k8sCommonServiceImpl, installedAppDBExtendedServiceImpl, installedAppResourceServiceImpl, pipelineRepositoryImpl, k8sApplicationServiceImpl, deploymentConfigServiceImpl, resourceTreeServiceImpl) appListingRouterImpl := appList2.NewAppListingRouterImpl(appListingRestHandlerImpl) appInfoRestHandlerImpl := appInfo.NewAppInfoRestHandlerImpl(sugaredLogger, appCrudOperationServiceImpl, userServiceImpl, validate, enforcerUtilImpl, enforcerImpl, helmAppServiceImpl, enforcerUtilHelmImpl, genericNoteServiceImpl, commonEnforcementUtilImpl) appInfoRouterImpl := appInfo2.NewAppInfoRouterImpl(sugaredLogger, appInfoRestHandlerImpl) @@ -1078,7 +1081,7 @@ func InitializeApp() (*App, error) { cdWorkflowServiceImpl := cd.NewCdWorkflowServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) cdWorkflowRunnerReadServiceImpl := read20.NewCdWorkflowRunnerReadServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) webhookServiceImpl := pipeline.NewWebhookServiceImpl(ciArtifactRepositoryImpl, sugaredLogger, ciPipelineRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowCommonServiceImpl, workFlowStageStatusServiceImpl, ciServiceImpl) - workflowEventProcessorImpl, err := in.NewWorkflowEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowServiceImpl, cdWorkflowReadServiceImpl, cdWorkflowRunnerServiceImpl, cdWorkflowRunnerReadServiceImpl, workflowDagExecutorImpl, ciHandlerImpl, cdHandlerImpl, eventSimpleFactoryImpl, eventRESTClientImpl, triggerServiceImpl, deployedAppServiceImpl, webhookServiceImpl, validate, environmentVariables, cdWorkflowCommonServiceImpl, cdPipelineConfigServiceImpl, userDeploymentRequestServiceImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, cdWorkflowRepositoryImpl, deploymentConfigServiceImpl) + workflowEventProcessorImpl, err := in.NewWorkflowEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowServiceImpl, cdWorkflowReadServiceImpl, cdWorkflowRunnerServiceImpl, cdWorkflowRunnerReadServiceImpl, workflowDagExecutorImpl, ciHandlerImpl, cdHandlerImpl, eventSimpleFactoryImpl, eventRESTClientImpl, triggerServiceImpl, deployedAppServiceImpl, webhookServiceImpl, validate, environmentVariables, cdWorkflowCommonServiceImpl, cdPipelineConfigServiceImpl, userDeploymentRequestServiceImpl, serviceImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, cdWorkflowRepositoryImpl, deploymentConfigServiceImpl) if err != nil { return nil, err } From 3237725d2eff0abeae810b2755f17eafa6af4991 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Thu, 3 Apr 2025 11:47:09 +0530 Subject: [PATCH 07/37] telemetry svc refactoring --- client/telemetry/TelemetryEventClient.go | 2 +- .../in/WorkflowEventProcessorService.go | 4 +- pkg/pipeline/WorkflowService.go | 2 +- pkg/ucid/ucid.go | 86 ++++++++++--------- 4 files changed, 49 insertions(+), 45 deletions(-) diff --git a/client/telemetry/TelemetryEventClient.go b/client/telemetry/TelemetryEventClient.go index 5d3ba4ab25..a64196242f 100644 --- a/client/telemetry/TelemetryEventClient.go +++ b/client/telemetry/TelemetryEventClient.go @@ -657,7 +657,7 @@ func (impl *TelemetryEventClientImpl) SendTelemetryDashboardLoggedInEvent() erro func (impl *TelemetryEventClientImpl) getUCIDAndCheckIsOptedOut(ctx context.Context) (string, error) { newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TelemetryEventClientImpl.getUCIDAndCheckIsOptedOut") defer span.End() - ucid, found, err := impl.ucid.GetUCID() + ucid, found, err := impl.ucid.GetUCIDWithCache(impl.posthogClient.GetCache()) if err != nil { impl.logger.Errorw("exception while getting unique client id", "error", err) return "", err diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index 45ed82b9c2..49b127f95a 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -399,7 +399,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCIWorkflowStatusUpdate() error return } if len(wfStatus.DevtronAdministratorInstance) != 0 { - devtronUCID, _, err := impl.ucid.GetUCID() + devtronUCID, _, err := impl.ucid.GetUCIDWithOutCache() if err != nil { impl.logger.Errorw("error in getting UCID", "err", err) return @@ -458,7 +458,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error return } if len(wfStatus.DevtronAdministratorInstance) != 0 { - devtronUCID, _, err := impl.ucid.GetUCID() + devtronUCID, _, err := impl.ucid.GetUCIDWithOutCache() if err != nil { impl.logger.Errorw("error in getting UCID", "err", err) return diff --git a/pkg/pipeline/WorkflowService.go b/pkg/pipeline/WorkflowService.go index c51c1aab96..660350a271 100644 --- a/pkg/pipeline/WorkflowService.go +++ b/pkg/pipeline/WorkflowService.go @@ -218,7 +218,7 @@ func (impl *WorkflowServiceImpl) createWorkflowTemplate(workflowRequest *types.W clusterConfig, err := impl.getClusterConfig(workflowRequest) workflowTemplate.ClusterConfig = clusterConfig workflowTemplate.WorkflowType = workflowRequest.GetWorkflowTypeForWorkflowRequest() - devtronUCID, _, err := impl.ucid.GetUCID() + devtronUCID, _, err := impl.ucid.GetUCIDWithOutCache() if err != nil { impl.Logger.Errorw("error in getting UCID", "err", err) return bean3.WorkflowTemplate{}, err diff --git a/pkg/ucid/ucid.go b/pkg/ucid/ucid.go index 6dcc5de475..65c03e8887 100644 --- a/pkg/ucid/ucid.go +++ b/pkg/ucid/ucid.go @@ -19,74 +19,78 @@ package ucid import ( "fmt" "github.com/devtron-labs/common-lib/utils/k8s" - "github.com/devtron-labs/devtron/client/telemetry/posthog" - util3 "github.com/devtron-labs/devtron/pkg/util" - "github.com/devtron-labs/devtron/util" - "github.com/patrickmn/go-cache" + "github.com/devtron-labs/devtron/pkg/util" + globalUtil "github.com/devtron-labs/devtron/util" + goCache "github.com/patrickmn/go-cache" "go.uber.org/zap" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" v1 "k8s.io/api/core/v1" - v12 "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) type Service interface { - GetUCID() (string, bool, error) + GetUCIDWithCache(cache *goCache.Cache) (string, bool, error) + GetUCIDWithOutCache() (string, bool, error) } type ServiceImpl struct { logger *zap.SugaredLogger - posthogClient *posthog.PosthogClient K8sUtil *k8s.K8sServiceImpl - aCDAuthConfig *util3.ACDAuthConfig + aCDAuthConfig *util.ACDAuthConfig } func NewServiceImpl( logger *zap.SugaredLogger, - posthogClient *posthog.PosthogClient, K8sUtil *k8s.K8sServiceImpl, - aCDAuthConfig *util3.ACDAuthConfig, + aCDAuthConfig *util.ACDAuthConfig, ) *ServiceImpl { return &ServiceImpl{ logger: logger, - posthogClient: posthogClient, K8sUtil: K8sUtil, aCDAuthConfig: aCDAuthConfig, } } -func (impl *ServiceImpl) GetUCID() (string, bool, error) { - ucid, found := impl.posthogClient.GetCache().Get(DevtronUniqueClientIdConfigMapKey) - if found { - return ucid.(string), found, nil - } else { - client, err := impl.K8sUtil.GetClientForInCluster() +func (impl *ServiceImpl) GetUCIDWithOutCache() (string, bool, error) { + return impl.GetUCIDWithCache(nil) +} + +func (impl *ServiceImpl) GetUCIDWithCache(cache *goCache.Cache) (string, bool, error) { + if cache != nil { + ucid, found := cache.Get(DevtronUniqueClientIdConfigMapKey) + if found { + return ucid.(string), true, nil + } + } + client, err := impl.K8sUtil.GetClientForInCluster() + if err != nil { + impl.logger.Errorw("exception while getting unique client id", "error", err) + return "", false, err + } + cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, DevtronUniqueClientIdConfigMap, client) + if errStatus, ok := status.FromError(err); !ok || errStatus.Code() == codes.NotFound || errStatus.Code() == codes.Unknown { + // if not found, create new cm + cm = &v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: DevtronUniqueClientIdConfigMap}} + data := map[string]string{} + data[DevtronUniqueClientIdConfigMapKey] = globalUtil.Generate(16) // generate unique random number + data[InstallEventKey] = "1" // used in operator to detect event is install or upgrade + data[UIEventKey] = "1" + cm.Data = data + _, err = impl.K8sUtil.CreateConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, cm, client) if err != nil { impl.logger.Errorw("exception while getting unique client id", "error", err) - return "", found, err + return "", false, err } - cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, DevtronUniqueClientIdConfigMap, client) - if errStatus, ok := status.FromError(err); !ok || errStatus.Code() == codes.NotFound || errStatus.Code() == codes.Unknown { - // if not found, create new cm - cm = &v1.ConfigMap{ObjectMeta: v12.ObjectMeta{Name: DevtronUniqueClientIdConfigMap}} - data := map[string]string{} - data[DevtronUniqueClientIdConfigMapKey] = util.Generate(16) // generate unique random number - data[InstallEventKey] = "1" // used in operator to detect event is install or upgrade - data[UIEventKey] = "1" - cm.Data = data - _, err = impl.K8sUtil.CreateConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, cm, client) - if err != nil { - impl.logger.Errorw("exception while getting unique client id", "error", err) - return "", found, err - } - } - if cm == nil { - impl.logger.Errorw("configmap not found while getting unique client id") - return ucid.(string), found, fmt.Errorf("configmap %q not found while getting unique client id", DevtronUniqueClientIdConfigMap) - } - dataMap := cm.Data - ucid = dataMap[DevtronUniqueClientIdConfigMapKey] - impl.posthogClient.GetCache().Set(DevtronUniqueClientIdConfigMapKey, ucid, cache.DefaultExpiration) } - return ucid.(string), found, nil + if cm == nil { + impl.logger.Errorw("configmap not found while getting unique client id") + return "", false, fmt.Errorf("configmap %q not found while getting unique client id", DevtronUniqueClientIdConfigMap) + } + dataMap := cm.Data + ucid := dataMap[DevtronUniqueClientIdConfigMapKey] + if cache != nil { + cache.Set(DevtronUniqueClientIdConfigMapKey, ucid, goCache.DefaultExpiration) + } + return ucid, false, nil } From 8525719c16c98609c8f9dfbfbaa9e7e44e722ceb Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Thu, 3 Apr 2025 12:10:42 +0530 Subject: [PATCH 08/37] updated wire gen files --- cmd/external-app/wire_gen.go | 2 +- wire_gen.go | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index 52ff9e5076..f1d6763cc8 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -390,7 +390,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - serviceImpl := ucid.NewServiceImpl(sugaredLogger, posthogClient, k8sServiceImpl, acdAuthConfig) + serviceImpl := ucid.NewServiceImpl(sugaredLogger, k8sServiceImpl, acdAuthConfig) providerIdentifierServiceImpl := providerIdentifier.NewProviderIdentifierServiceImpl(sugaredLogger) telemetryEventClientImpl, err := telemetry.NewTelemetryEventClientImpl(sugaredLogger, httpClient, clusterServiceImpl, k8sServiceImpl, acdAuthConfig, userServiceImpl, attributesRepositoryImpl, ssoLoginServiceImpl, posthogClient, serviceImpl, moduleRepositoryImpl, serverDataStoreServerDataStore, userAuditServiceImpl, helmAppClientImpl, providerIdentifierServiceImpl, cronLoggerImpl, installedAppReadServiceEAImpl, environmentVariables) if err != nil { diff --git a/wire_gen.go b/wire_gen.go index de7e59d8ef..c4ff527793 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -625,11 +625,7 @@ func InitializeApp() (*App, error) { } ciInfraGetter := ci.NewCiInfraGetter(sugaredLogger, infraConfigServiceImpl, infraConfigAuditServiceImpl) infraProviderImpl := infraProviders.NewInfraProviderImpl(sugaredLogger, infraGetter, ciInfraGetter) - posthogClient, err := posthog.NewPosthogClient(sugaredLogger) - if err != nil { - return nil, err - } - serviceImpl := ucid.NewServiceImpl(sugaredLogger, posthogClient, k8sServiceImpl, acdAuthConfig) + serviceImpl := ucid.NewServiceImpl(sugaredLogger, k8sServiceImpl, acdAuthConfig) workflowServiceImpl, err := pipeline.NewWorkflowServiceImpl(sugaredLogger, environmentRepositoryImpl, ciCdConfig, configReadServiceImpl, globalCMCSServiceImpl, argoWorkflowExecutorImpl, k8sServiceImpl, systemWorkflowExecutorImpl, k8sCommonServiceImpl, infraProviderImpl, serviceImpl) if err != nil { return nil, err @@ -931,6 +927,10 @@ func InitializeApp() (*App, error) { ssoLoginServiceImpl := sso.NewSSOLoginServiceImpl(sugaredLogger, ssoLoginRepositoryImpl, k8sServiceImpl, environmentVariables, userAuthOidcHelperImpl) ssoLoginRestHandlerImpl := sso2.NewSsoLoginRestHandlerImpl(validate, sugaredLogger, enforcerImpl, userServiceImpl, ssoLoginServiceImpl) ssoLoginRouterImpl := sso2.NewSsoLoginRouterImpl(ssoLoginRestHandlerImpl) + posthogClient, err := posthog.NewPosthogClient(sugaredLogger) + if err != nil { + return nil, err + } providerIdentifierServiceImpl := providerIdentifier.NewProviderIdentifierServiceImpl(sugaredLogger) telemetryEventClientImplExtended, err := telemetry.NewTelemetryEventClientImplExtended(sugaredLogger, httpClient, clusterServiceImplExtended, k8sServiceImpl, acdAuthConfig, environmentServiceImpl, userServiceImpl, appListingRepositoryImpl, posthogClient, serviceImpl, ciPipelineConfigReadServiceImpl, pipelineRepositoryImpl, gitProviderRepositoryImpl, attributesRepositoryImpl, ssoLoginServiceImpl, appRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, gitMaterialReadServiceImpl, ciTemplateRepositoryImpl, chartRepositoryImpl, userAuditServiceImpl, ciBuildConfigServiceImpl, moduleRepositoryImpl, serverDataStoreServerDataStore, helmAppClientImpl, installedAppReadServiceImpl, userAttributesRepositoryImpl, providerIdentifierServiceImpl, cronLoggerImpl, gitOpsConfigReadServiceImpl) if err != nil { From 44abd3fbff849a754e00dd61664f7687062a9fd9 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Thu, 3 Apr 2025 12:23:51 +0530 Subject: [PATCH 09/37] fix: CheckIfReTriggerRequired condition and CdHandlerImpl.UpdateWorkflow method --- .../pipelineConfig/CdWorfkflowRepository.go | 11 +++++++++++ pkg/pipeline/CdHandler.go | 4 ++-- pkg/pipeline/executors/WorkflowUtils.go | 7 +++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index a737a7076a..9aa3eeaa28 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -50,6 +50,7 @@ type CdWorkflowRepository interface { FindWorkflowRunnerByCdWorkflowId(wfIds []int) ([]*CdWorkflowRunner, error) FindPreviousCdWfRunnerByStatus(pipelineId int, currentWFRunnerId int, status []string) ([]*CdWorkflowRunner, error) FindWorkflowRunnerById(wfrId int) (*CdWorkflowRunner, error) + FindPreOrPostCdWorkflowRunnerById(wfrId int) (*CdWorkflowRunner, error) FindBasicWorkflowRunnerById(wfrId int) (*CdWorkflowRunner, error) FindRetriedWorkflowCountByReferenceId(wfrId int) (int, error) FindLatestWfrByAppIdAndEnvironmentId(appId int, environmentId int) (*CdWorkflowRunner, error) @@ -528,6 +529,16 @@ func (impl *CdWorkflowRepositoryImpl) FindWorkflowRunnerById(wfrId int) (*CdWork return wfr, err } +func (impl *CdWorkflowRepositoryImpl) FindPreOrPostCdWorkflowRunnerById(wfrId int) (*CdWorkflowRunner, error) { + wfr := &CdWorkflowRunner{} + err := impl.dbConnection.Model(wfr). + Column("cd_workflow_runner.*", "CdWorkflow", "CdWorkflow.Pipeline", "CdWorkflow.CiArtifact", "CdWorkflow.Pipeline.Environment"). + Where("cd_workflow_runner.id = ?", wfrId). + Where("cd_workflow_runner.workflow_type != ?", apiBean.CD_WORKFLOW_TYPE_DEPLOY). + Select() + return wfr, err +} + func (impl *CdWorkflowRepositoryImpl) FindBasicWorkflowRunnerById(wfrId int) (*CdWorkflowRunner, error) { wfr := &CdWorkflowRunner{} err := impl.dbConnection.Model(wfr). diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index e11c2ca76e..a45551df66 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -279,9 +279,9 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus bean6.CiCdStatus) (int, return 0, "", false, err } - savedWorkflow, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(workflowId) + savedWorkflow, err := impl.cdWorkflowRepository.FindPreOrPostCdWorkflowRunnerById(workflowId) if err != nil { - impl.Logger.Error("cannot get saved wf", "err", err) + impl.Logger.Error("cannot get saved wf", "workflowId", workflowId, "err", err) return 0, "", false, err } diff --git a/pkg/pipeline/executors/WorkflowUtils.go b/pkg/pipeline/executors/WorkflowUtils.go index c812f761fe..f59022d797 100644 --- a/pkg/pipeline/executors/WorkflowUtils.go +++ b/pkg/pipeline/executors/WorkflowUtils.go @@ -288,8 +288,11 @@ func GetClientInstance(config *rest.Config, namespace string) (v1alpha12.Workflo } func CheckIfReTriggerRequired(status, message, workflowRunnerStatus string) bool { - return ((status == string(v1alpha1.NodeError) || status == string(v1alpha1.NodeFailed)) && - message == cdWorkflow.POD_DELETED_MESSAGE) && (workflowRunnerStatus != cdWorkflow.WorkflowCancel && workflowRunnerStatus != cdWorkflow.WorkflowAborted) + return (status == string(v1alpha1.NodeError) || status == string(v1alpha1.NodeFailed)) && + message == cdWorkflow.POD_DELETED_MESSAGE && + workflowRunnerStatus != cdWorkflow.WorkflowCancel && + workflowRunnerStatus != cdWorkflow.WorkflowAborted && + workflowRunnerStatus != cdWorkflow.WorkflowSucceeded } From 30fd819b7ac8fd3a10ffda53e439cca19fba834f Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Thu, 3 Apr 2025 13:43:35 +0530 Subject: [PATCH 10/37] fix: workflow message update --- .../pipelineConfig/bean/cdWorkflow/bean.go | 54 ------------------- pkg/pipeline/CdHandler.go | 30 +++++++---- pkg/pipeline/CiHandler.go | 16 ++++-- 3 files changed, 31 insertions(+), 69 deletions(-) delete mode 100644 internal/sql/repository/pipelineConfig/bean/cdWorkflow/bean.go diff --git a/internal/sql/repository/pipelineConfig/bean/cdWorkflow/bean.go b/internal/sql/repository/pipelineConfig/bean/cdWorkflow/bean.go deleted file mode 100644 index 645ef52eb2..0000000000 --- a/internal/sql/repository/pipelineConfig/bean/cdWorkflow/bean.go +++ /dev/null @@ -1,54 +0,0 @@ -package cdWorkflow - -import ( - "errors" - "github.com/devtron-labs/common-lib/utils/k8s/health" - "github.com/devtron-labs/devtron/client/argocdServer/bean" -) - -var WfrTerminalStatusList = []string{WorkflowAborted, WorkflowFailed, WorkflowSucceeded, bean.HIBERNATING, string(health.HealthStatusHealthy), string(health.HealthStatusDegraded)} - -type WorkflowStatus int - -const ( - WF_UNKNOWN WorkflowStatus = iota - REQUEST_ACCEPTED - ENQUEUED - QUE_ERROR - WF_STARTED - DROPPED_STALE - DEQUE_ERROR - TRIGGER_ERROR -) - -const ( - WorkflowStarting = "Starting" - WorkflowInQueue = "Queued" - WorkflowInitiated = "Initiating" - WorkflowInProgress = "Progressing" - WorkflowAborted = "Aborted" - WorkflowFailed = "Failed" - WorkflowSucceeded = "Succeeded" - WorkflowTimedOut = "TimedOut" - WorkflowUnableToFetchState = "UnableToFetch" - WorkflowTypeDeploy = "DEPLOY" - WorkflowTypePre = "PRE" - WorkflowTypePost = "POST" -) - -func (a WorkflowStatus) String() string { - return [...]string{"WF_UNKNOWN", "REQUEST_ACCEPTED", "ENQUEUED", "QUE_ERROR", "WF_STARTED", "DROPPED_STALE", "DEQUE_ERROR", "TRIGGER_ERROR"}[a] -} - -var ErrorDeploymentSuperseded = errors.New(NEW_DEPLOYMENT_INITIATED) - -const ( - WORKFLOW_EXECUTOR_TYPE_AWF = "AWF" - WORKFLOW_EXECUTOR_TYPE_SYSTEM = "SYSTEM" - NEW_DEPLOYMENT_INITIATED = "A new deployment was initiated before this deployment completed!" - PIPELINE_DELETED = "The pipeline has been deleted!" - FOUND_VULNERABILITY = "Found vulnerability on image" - GITOPS_REPO_NOT_CONFIGURED = "GitOps repository is not configured for the app" -) - -type WorkflowExecutorType string diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index a45551df66..6a12c1ed01 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -22,8 +22,8 @@ import ( "fmt" "github.com/devtron-labs/common-lib/utils" bean4 "github.com/devtron-labs/common-lib/utils/bean" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/adapter/cdWorkflow" - cdWorkflow2 "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" + cdWorkflowAdapter "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/adapter/cdWorkflow" + cdWorkflowBean "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" bean2 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/build/artifacts/imageTagging" "github.com/devtron-labs/devtron/pkg/cluster/adapter" @@ -39,6 +39,7 @@ import ( "net/http" "os" "path/filepath" + "slices" "strconv" "strings" "time" @@ -227,7 +228,7 @@ func (impl *CdHandlerImpl) CancelStage(workflowRunnerId int, forceAbort bool, us return 0, err } } - workflowRunner.Status = cdWorkflow2.WorkflowCancel + workflowRunner.Status = cdWorkflowBean.WorkflowCancel workflowRunner.UpdatedOn = time.Now() workflowRunner.UpdatedBy = userId err = impl.cdWorkflowRunnerService.UpdateCdWorkflowRunnerWithStage(workflowRunner) @@ -239,7 +240,7 @@ func (impl *CdHandlerImpl) CancelStage(workflowRunnerId int, forceAbort bool, us } func (impl *CdHandlerImpl) updateWorkflowRunnerForForceAbort(workflowRunner *pipelineConfig.CdWorkflowRunner) error { - workflowRunner.Status = cdWorkflow2.WorkflowCancel + workflowRunner.Status = cdWorkflowBean.WorkflowCancel workflowRunner.PodStatus = string(bean2.Failed) workflowRunner.Message = constants.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE err := impl.cdWorkflowRunnerService.UpdateCdWorkflowRunnerWithStage(workflowRunner) @@ -288,13 +289,22 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus bean6.CiCdStatus) (int, cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() cdArtifactLocation := fmt.Sprintf(cdArtifactLocationFormat, savedWorkflow.CdWorkflowId, savedWorkflow.Id) if impl.stateChanged(status, podStatus, message, workflowStatus.FinishedAt.Time, savedWorkflow) { - if savedWorkflow.Status != cdWorkflow2.WorkflowCancel { + if savedWorkflow.Status != cdWorkflowBean.WorkflowCancel { savedWorkflow.Status = status } savedWorkflow.CdArtifactLocation = cdArtifactLocation savedWorkflow.PodStatus = podStatus - savedWorkflow.Message = message - savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time + if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.Status) { + savedWorkflow.Message = message + // NOTE: we are doing this to fix where a pre-cd / post-cd workflow message becomes larger than 1000, and in db we had set the charter limit to 1000 + if len(message) > 1000 { + savedWorkflow.Message = message[:1000] + } + savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time + } else { + impl.Logger.Warnw("cd stage already in terminal state. skipped message and finishedOn from being updated", + "wfId", savedWorkflow.Id, "status", savedWorkflow.Status, "message", message, "finishedOn", workflowStatus.FinishedAt.Time) + } savedWorkflow.Name = workflowName // removed log location from here since we are saving it at trigger savedWorkflow.PodName = podName @@ -312,7 +322,7 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus bean6.CiCdStatus) (int, impl.Logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", appId, "envId", envId, "err", err) return 0, "", true, err } - util3.TriggerCDMetrics(cdWorkflow.GetTriggerMetricsFromRunnerObj(savedWorkflow, envDeploymentConfig), impl.config.ExposeCDMetrics) + util3.TriggerCDMetrics(cdWorkflowAdapter.GetTriggerMetricsFromRunnerObj(savedWorkflow, envDeploymentConfig), impl.config.ExposeCDMetrics) if string(v1alpha1.NodeError) == savedWorkflow.Status || string(v1alpha1.NodeFailed) == savedWorkflow.Status { impl.Logger.Warnw("cd stage failed for workflow: ", "wfId", savedWorkflow.Id) } @@ -558,7 +568,7 @@ func (impl *CdHandlerImpl) getWorkflowLogs(pipelineId int, cdWorkflow *pipelineC if logStream == nil || err != nil { if !cdWorkflow.BlobStorageEnabled { return nil, nil, errors.New("logs-not-stored-in-repository") - } else if string(v1alpha1.NodeSucceeded) == cdWorkflow.Status || string(v1alpha1.NodeError) == cdWorkflow.Status || string(v1alpha1.NodeFailed) == cdWorkflow.Status || cdWorkflow.Status == cdWorkflow2.WorkflowCancel { + } else if string(v1alpha1.NodeSucceeded) == cdWorkflow.Status || string(v1alpha1.NodeError) == cdWorkflow.Status || string(v1alpha1.NodeFailed) == cdWorkflow.Status || cdWorkflow.Status == cdWorkflowBean.WorkflowCancel { impl.Logger.Debugw("pod is not live", "podName", cdWorkflow.PodName, "err", err) return impl.getLogsFromRepository(pipelineId, cdWorkflow, clusterConfig, runStageInEnv) } @@ -583,7 +593,7 @@ func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pip PipelineId: cdWorkflow.CdWorkflow.PipelineId, WorkflowId: cdWorkflow.Id, PodName: cdWorkflow.PodName, - LogsFilePath: cdWorkflow.LogLocation, // impl.ciCdConfig.CiDefaultBuildLogsKeyPrefix + "/" + cdWorkflow.Name + "/main.log", //TODO - fixme + LogsFilePath: cdWorkflow.LogLocation, // impl.ciCdConfig.CiDefaultBuildLogsKeyPrefix + "/" + cdWorkflowAda.Name + "/main.log", //TODO - fixme CloudProvider: impl.config.CloudProvider, AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index fa7ee5d455..7549bbe34f 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -41,6 +41,7 @@ import ( "os" "path/filepath" "regexp" + "slices" "strconv" "strings" "time" @@ -1203,16 +1204,21 @@ func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus bean6.CiCdStatus) (int, savedWorkflow.Status = status } savedWorkflow.PodStatus = podStatus - savedWorkflow.Message = message - // NOTE: we are doing this for a quick fix where ci pending message become larger than 250 and in db we had set the charter limit to 250 - if len(message) > 250 { - savedWorkflow.Message = message[:250] + if !slices.Contains(cdWorkflow.WfrTerminalStatusList, savedWorkflow.Status) { + savedWorkflow.Message = message + // NOTE: we are doing this for a quick fix where ci pending message become larger than 250 and in db we had set the charter limit to 250 + if len(message) > 250 { + savedWorkflow.Message = message[:250] + } + savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time + } else { + impl.Logger.Warnw("cd stage already in terminal state. skipped message and finishedOn from being updated", + "wfId", savedWorkflow.Id, "status", savedWorkflow.Status, "message", message, "finishedOn", workflowStatus.FinishedAt.Time) } if savedWorkflow.ExecutorType == cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM && savedWorkflow.Status == cdWorkflow.WorkflowCancel { savedWorkflow.PodStatus = "Failed" savedWorkflow.Message = constants2.TERMINATE_MESSAGE } - savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time savedWorkflow.Name = workflowName //savedWorkflow.LogLocation = "/ci-pipeline/" + strconv.Itoa(savedWorkflow.CiPipelineId) + "/workflow/" + strconv.Itoa(savedWorkflow.Id) + "/logs" //TODO need to fetch from workflow object //savedWorkflow.LogLocation = logLocation // removed because we are saving log location at trigger From 8ca9d531dffa3aeb1d45cbd4e95195b6948d535c Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Thu, 3 Apr 2025 13:50:55 +0530 Subject: [PATCH 11/37] updated import alias --- pkg/pipeline/CdHandler.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 6a12c1ed01..ef088b323f 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -22,7 +22,7 @@ import ( "fmt" "github.com/devtron-labs/common-lib/utils" bean4 "github.com/devtron-labs/common-lib/utils/bean" - cdWorkflowAdapter "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/adapter/cdWorkflow" + pipelineAdapter "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/adapter/cdWorkflow" cdWorkflowBean "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" bean2 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/build/artifacts/imageTagging" @@ -56,7 +56,7 @@ import ( pipelineBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/types" resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup" - util3 "github.com/devtron-labs/devtron/util" + globalUtil "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/rbac" "github.com/go-pg/pg" "go.opentelemetry.io/otel" @@ -322,7 +322,7 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus bean6.CiCdStatus) (int, impl.Logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", appId, "envId", envId, "err", err) return 0, "", true, err } - util3.TriggerCDMetrics(cdWorkflowAdapter.GetTriggerMetricsFromRunnerObj(savedWorkflow, envDeploymentConfig), impl.config.ExposeCDMetrics) + globalUtil.TriggerCDMetrics(pipelineAdapter.GetTriggerMetricsFromRunnerObj(savedWorkflow, envDeploymentConfig), impl.config.ExposeCDMetrics) if string(v1alpha1.NodeError) == savedWorkflow.Status || string(v1alpha1.NodeFailed) == savedWorkflow.Status { impl.Logger.Warnw("cd stage failed for workflow: ", "wfId", savedWorkflow.Id) } @@ -593,7 +593,7 @@ func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pip PipelineId: cdWorkflow.CdWorkflow.PipelineId, WorkflowId: cdWorkflow.Id, PodName: cdWorkflow.PodName, - LogsFilePath: cdWorkflow.LogLocation, // impl.ciCdConfig.CiDefaultBuildLogsKeyPrefix + "/" + cdWorkflowAda.Name + "/main.log", //TODO - fixme + LogsFilePath: cdWorkflow.LogLocation, // impl.ciCdConfig.CiDefaultBuildLogsKeyPrefix + "/" + cdWorkflow.Name + "/main.log", //TODO - fixme CloudProvider: impl.config.CloudProvider, AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, From 4e73878e46a5ee1b4f5e8e40c01fe15d547f21e0 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Thu, 3 Apr 2025 14:24:32 +0530 Subject: [PATCH 12/37] fix: ci/ cd status update inconsistencies --- .../configure/BuildPipelineRestHandler.go | 2 +- .../in/WorkflowEventProcessorService.go | 25 ++++++++++-------- pkg/pipeline/CdHandler.go | 18 ++++++------- pkg/pipeline/CiHandler.go | 26 +++++++++---------- 4 files changed, 37 insertions(+), 34 deletions(-) diff --git a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go index cb730a93ab..54c549df45 100644 --- a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go @@ -1508,7 +1508,7 @@ func (handler *PipelineConfigRestHandlerImpl) HandleWorkflowWebhook(w http.Respo return } handler.Logger.Infow("request payload, HandleWorkflowWebhook", "payload", wfUpdateReq) - resp, err := handler.ciHandler.UpdateWorkflow(wfUpdateReq) + resp, _, err := handler.ciHandler.UpdateWorkflow(wfUpdateReq) if err != nil { handler.Logger.Errorw("service err, HandleWorkflowWebhook", "err", err, "payload", wfUpdateReq) common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index 49b127f95a..3496f92aaf 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -414,18 +414,20 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCIWorkflowStatusUpdate() error return } } - err = impl.ciHandler.CheckAndReTriggerCI(wfStatus) + // update the ci workflow status + _, stateChanged, err := impl.ciHandler.UpdateWorkflow(wfStatus) if err != nil { - impl.logger.Errorw("error in checking and re triggering ci", "err", err) - //don't return as we have to update the workflow status - } - - _, err = impl.ciHandler.UpdateWorkflow(wfStatus) - if err != nil { - impl.logger.Errorw("error on update workflow status", "err", err, "msg", msg.Data) + impl.logger.Errorw("error on update workflow status", "msg", msg.Data, "err", err) return } - + if stateChanged { + // check if we need to re-trigger the ci + err = impl.ciHandler.CheckAndReTriggerCI(wfStatus) + if err != nil { + impl.logger.Errorw("error in checking and re triggering ci", "wfStatus", wfStatus, "err", err) + return + } + } } // add required logging here @@ -474,9 +476,9 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error } } wfrId, wfrStatus, stateChanged, err := impl.cdHandler.UpdateWorkflow(wfStatus) - impl.logger.Debugw("UpdateWorkflow", "wfrId", wfrId, "wfrStatus", wfrStatus) + impl.logger.Debugw("cd UpdateWorkflow for wfStatus", "wfrId", wfrId, "wfrStatus", wfrStatus, "wfStatus", wfStatus) if err != nil { - impl.logger.Error("err", err) + impl.logger.Errorw("error in cd workflow status update", "wfrId", wfrId, "wfrStatus", wfrStatus, "wfStatus", wfStatus, "err", err) return } @@ -492,6 +494,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error err := impl.cdHandler.DeactivateImageReservationPathsOnFailure(wfr.ImagePathReservationIds) if err != nil { impl.logger.Errorw("error in removing image path reservation ", "imagePathReservationIds", wfr.ImagePathReservationIds, "err", err) + // not returning here as we need to send the notification event and re-trigger the cd stage (if required) } } } diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index ef088b323f..533a39515b 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -268,15 +268,15 @@ func (impl *CdHandlerImpl) handleForceAbortCaseForCdStage(workflowRunner *pipeli } func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus bean6.CiCdStatus) (int, string, bool, error) { - wfStatusRs := impl.extractWorkfowStatus(workflowStatus) + wfStatusRs := impl.extractWorkflowStatus(workflowStatus) workflowName, status, podStatus, message, podName := wfStatusRs.WorkflowName, wfStatusRs.Status, wfStatusRs.PodStatus, wfStatusRs.Message, wfStatusRs.PodName - impl.Logger.Debugw("cd update for ", "wf ", workflowName, "status", status) + impl.Logger.Debugw("cd workflow status update event for", "wf ", workflowName, "status", status) if workflowName == "" { return 0, "", false, errors.New("invalid wf name") } workflowId, err := strconv.Atoi(workflowName[:strings.Index(workflowName, "-")]) if err != nil { - impl.Logger.Error("invalid wf status update req", "err", err) + impl.Logger.Errorw("invalid wf status update req", "workflowName", workflowName, "err", err) return 0, "", false, err } @@ -309,29 +309,29 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus bean6.CiCdStatus) (int, // removed log location from here since we are saving it at trigger savedWorkflow.PodName = podName savedWorkflow.UpdateAuditLog(1) - impl.Logger.Debugw("updating workflow ", "workflow", savedWorkflow) + impl.Logger.Debugw("updating cd workflow runner", "workflow", savedWorkflow) err = impl.cdWorkflowRunnerService.UpdateCdWorkflowRunnerWithStage(savedWorkflow) if err != nil { - impl.Logger.Error("update wf failed for id " + strconv.Itoa(savedWorkflow.Id)) - return 0, "", true, err + impl.Logger.Errorw("update wf failed for id", "wfId", savedWorkflow.Id, "err", err) + return savedWorkflow.Id, "", true, err } appId := savedWorkflow.CdWorkflow.Pipeline.AppId envId := savedWorkflow.CdWorkflow.Pipeline.EnvironmentId envDeploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(appId, envId) if err != nil { impl.Logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", appId, "envId", envId, "err", err) - return 0, "", true, err + return savedWorkflow.Id, savedWorkflow.Status, true, err } globalUtil.TriggerCDMetrics(pipelineAdapter.GetTriggerMetricsFromRunnerObj(savedWorkflow, envDeploymentConfig), impl.config.ExposeCDMetrics) if string(v1alpha1.NodeError) == savedWorkflow.Status || string(v1alpha1.NodeFailed) == savedWorkflow.Status { - impl.Logger.Warnw("cd stage failed for workflow: ", "wfId", savedWorkflow.Id) + impl.Logger.Warnw("cd stage failed for workflow", "wfId", savedWorkflow.Id) } return savedWorkflow.Id, savedWorkflow.Status, true, nil } return savedWorkflow.Id, savedWorkflow.Status, false, nil } -func (impl *CdHandlerImpl) extractWorkfowStatus(workflowStatus bean6.CiCdStatus) *types.WorkflowStatus { +func (impl *CdHandlerImpl) extractWorkflowStatus(workflowStatus bean6.CiCdStatus) *types.WorkflowStatus { workflowName := "" status := string(workflowStatus.Phase) podStatus := "Pending" diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 7549bbe34f..81aaeb187d 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -90,7 +90,7 @@ type CiHandler interface { GetBuildHistory(pipelineId int, appId int, offset int, size int) ([]types.WorkflowResponse, error) DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) - UpdateWorkflow(workflowStatus bean6.CiCdStatus) (int, error) + UpdateWorkflow(workflowStatus bean6.CiCdStatus) (int, bool, error) FetchCiStatusForTriggerView(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) FetchCiStatusForTriggerViewV1(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) @@ -1165,35 +1165,34 @@ func (impl *CiHandlerImpl) extractPodStatusAndWorkflow(workflowStatus bean6.CiCd } func (impl *CiHandlerImpl) getRefWorkflowAndCiRetryCount(savedWorkflow *pipelineConfig.CiWorkflow) (int, *pipelineConfig.CiWorkflow, error) { - var err error - if savedWorkflow.ReferenceCiWorkflowId != 0 { + var err error savedWorkflow, err = impl.ciWorkflowRepository.FindById(savedWorkflow.ReferenceCiWorkflowId) - } - if err != nil { - impl.Logger.Errorw("cannot get saved wf", "err", err) - return 0, savedWorkflow, err + if err != nil { + impl.Logger.Errorw("cannot get saved wf", "err", err) + return 0, savedWorkflow, err + } } retryCount, err := impl.ciWorkflowRepository.FindRetriedWorkflowCountByReferenceId(savedWorkflow.Id) return retryCount, savedWorkflow, err } -func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus bean6.CiCdStatus) (int, error) { +func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus bean6.CiCdStatus) (int, bool, error) { workflowName, status, podStatus, message, _, podName := ExtractWorkflowStatus(workflowStatus) if workflowName == "" { impl.Logger.Errorw("extract workflow status, invalid wf name", "workflowName", workflowName, "status", status, "podStatus", podStatus, "message", message) - return 0, errors.New("invalid wf name") + return 0, false, errors.New("invalid wf name") } workflowId, err := strconv.Atoi(workflowName[:strings.Index(workflowName, "-")]) if err != nil { impl.Logger.Errorw("invalid wf status update req", "err", err) - return 0, err + return 0, false, err } savedWorkflow, err := impl.ciWorkflowRepository.FindById(workflowId) if err != nil { impl.Logger.Errorw("cannot get saved wf", "err", err) - return 0, err + return 0, false, err } ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() @@ -1228,11 +1227,12 @@ func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus bean6.CiCdStatus) (int, err = impl.ciService.UpdateCiWorkflowWithStage(savedWorkflow) if err != nil { impl.Logger.Error("update wf failed for id " + strconv.Itoa(savedWorkflow.Id)) - return 0, err + return savedWorkflow.Id, true, err } impl.sendCIFailEvent(savedWorkflow, status, message) + return savedWorkflow.Id, true, nil } - return savedWorkflow.Id, nil + return savedWorkflow.Id, false, nil } func (impl *CiHandlerImpl) sendCIFailEvent(savedWorkflow *pipelineConfig.CiWorkflow, status, message string) { From b49821ebc7cfaa8bb6443c90a6c9ffabb7a877bf Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Thu, 3 Apr 2025 23:03:08 +0530 Subject: [PATCH 13/37] fix: checkForOptOut method error handling --- client/telemetry/TelemetryEventClient.go | 2 +- go.mod | 4 ++-- go.sum | 8 ++++---- .../devtron-labs/authenticator/client/oidcClient.go | 11 +++++++---- vendor/modules.txt | 8 ++++---- 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/client/telemetry/TelemetryEventClient.go b/client/telemetry/TelemetryEventClient.go index a64196242f..7a4a697094 100644 --- a/client/telemetry/TelemetryEventClient.go +++ b/client/telemetry/TelemetryEventClient.go @@ -686,8 +686,8 @@ func (impl *TelemetryEventClientImpl) checkForOptOut(ctx context.Context, UCID s response, err := util.HttpRequest(newCtx, url) if err != nil { + // this should be non-blocking call and should not fail the request for ucid getting impl.logger.Errorw("check opt-out list failed, rest api error", "ucid", UCID, "err", err) - return false, err } flag := response["result"].(bool) return flag, nil diff --git a/go.mod b/go.mod index c9ccd0dd10..aa0d1cdcd6 100644 --- a/go.mod +++ b/go.mod @@ -307,8 +307,8 @@ require ( replace ( github.com/argoproj/argo-workflows/v3 => github.com/devtron-labs/argo-workflows/v3 v3.5.13 - github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250402155428-3fa1ffc068e2 - github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250402155428-3fa1ffc068e2 + github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250403171407-721f466f49ce + github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250403171407-721f466f49ce github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127 github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5 k8s.io/api => k8s.io/api v0.29.7 diff --git a/go.sum b/go.sum index 9b29696f0c..2a9b9bb3e6 100644 --- a/go.sum +++ b/go.sum @@ -829,10 +829,10 @@ github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzq github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/devtron-labs/argo-workflows/v3 v3.5.13 h1:3pINq0gXOSeTw2z/vYe+j80lRpSN5Rp/8mfQORh8SmU= github.com/devtron-labs/argo-workflows/v3 v3.5.13/go.mod h1:/vqxcovDPT4zqr4DjR5v7CF8ggpY1l3TSa2CIG3jmjA= -github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250402155428-3fa1ffc068e2 h1:Ik3LiAVccf6gqrbEwryD/uPYbSV/i7z4UTvYcLgM8jk= -github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250402155428-3fa1ffc068e2/go.mod h1:FfaLDXN1ZXxyRpnskBqVIYkpkWDCzBmDgIO9xqLnxdQ= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250402155428-3fa1ffc068e2 h1:k2bWvWk6if6I7k+URmrh1Dp2fleNDNu/pl+ZnImeUGE= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250402155428-3fa1ffc068e2/go.mod h1:zkNShlkcHxsmnL0gKNbs0uyRL8lZonGKr5Km63uTLI0= +github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250403171407-721f466f49ce h1:A4+SH98TFhHo/cUWi9CE3rbw0TjU6uH7cfOFOsmQrqk= +github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250403171407-721f466f49ce/go.mod h1:FfaLDXN1ZXxyRpnskBqVIYkpkWDCzBmDgIO9xqLnxdQ= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250403171407-721f466f49ce h1:hp5p/mPLsoCyqM51b88Uu8AgCTn0K2bwKEsB5Fy2C24= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250403171407-721f466f49ce/go.mod h1:zkNShlkcHxsmnL0gKNbs0uyRL8lZonGKr5Km63uTLI0= github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU= github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y= github.com/devtron-labs/protos v0.0.3-0.20250323220609-ecf8a0f7305e h1:U6UdYbW8a7xn5IzFPd8cywjVVPfutGJCudjePAfL/Hs= diff --git a/vendor/github.com/devtron-labs/authenticator/client/oidcClient.go b/vendor/github.com/devtron-labs/authenticator/client/oidcClient.go index b7dcba2f99..551dec305e 100644 --- a/vendor/github.com/devtron-labs/authenticator/client/oidcClient.go +++ b/vendor/github.com/devtron-labs/authenticator/client/oidcClient.go @@ -44,9 +44,11 @@ func GetSettings(conf *DexConfig) (*oidc.Settings, error) { settings := &oidc.Settings{ URL: conf.Url, OIDCConfig: oidc.OIDCConfig{CLIClientID: conf.DexClientID, - ClientSecret: conf.DexClientSecret, - Issuer: proxyUrl, - ServerSecret: conf.ServerSecret}, + ClientSecret: conf.DexClientSecret, + Issuer: proxyUrl, + ServerSecret: conf.ServerSecret, + RequestedScopes: conf.DexScopes, + }, UserSessionDuration: time.Duration(conf.UserSessionDurationSeconds) * time.Second, AdminPasswordMtime: conf.AdminPasswordMtime, } @@ -88,7 +90,8 @@ type DexConfig struct { UserSessionDurationSeconds int `env:"USER_SESSION_DURATION_SECONDS" envDefault:"86400"` AdminPasswordMtime time.Time `json:"ADMIN_PASSWORD_MTIME"` DexConfigRaw string - DevtronSecretName string `env:"DEVTRON_SECRET_NAME" envDefault:"devtron-secret"` + DevtronSecretName string `env:"DEVTRON_SECRET_NAME" envDefault:"devtron-secret"` + DexScopes []string `env:"DEX_SCOPES" envDefault:"" envSeparator:","` } func (c *DexConfig) GetDexProxyUrl() (string, error) { diff --git a/vendor/modules.txt b/vendor/modules.txt index 7084a09385..ae3dc39f9b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -472,7 +472,7 @@ github.com/davecgh/go-spew/spew # github.com/deckarep/golang-set v1.8.0 ## explicit; go 1.17 github.com/deckarep/golang-set -# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250402155428-3fa1ffc068e2 +# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250403171407-721f466f49ce ## explicit; go 1.21 github.com/devtron-labs/authenticator/apiToken github.com/devtron-labs/authenticator/client @@ -480,7 +480,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250402155428-3fa1ffc068e2 +# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250403171407-721f466f49ce ## explicit; go 1.21 github.com/devtron-labs/common-lib/async github.com/devtron-labs/common-lib/blob-storage @@ -2352,8 +2352,8 @@ xorm.io/xorm/names xorm.io/xorm/schemas xorm.io/xorm/tags # github.com/argoproj/argo-workflows/v3 => github.com/devtron-labs/argo-workflows/v3 v3.5.13 -# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250402155428-3fa1ffc068e2 -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250402155428-3fa1ffc068e2 +# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250403171407-721f466f49ce +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250403171407-721f466f49ce # github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127 # github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5 # k8s.io/api => k8s.io/api v0.29.7 From 186394a4536d7acea18322b322d747fecea52eef Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Thu, 3 Apr 2025 23:27:24 +0530 Subject: [PATCH 14/37] fix: panic handling --- .../configure/BuildPipelineRestHandler.go | 4 ++-- env_gen.json | 2 +- env_gen.md | 1 + pkg/eventProcessor/bean/workflowEventBean.go | 6 ++++++ .../in/WorkflowEventProcessorService.go | 8 ++++---- pkg/pipeline/CdHandler.go | 8 ++++---- pkg/pipeline/CiHandler.go | 18 +++++++++--------- 7 files changed, 27 insertions(+), 20 deletions(-) diff --git a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go index 54c549df45..997da569b8 100644 --- a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go @@ -25,7 +25,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/constants" "github.com/devtron-labs/devtron/pkg/build/artifacts/imageTagging" bean2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" - bean3 "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" + eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" constants2 "github.com/devtron-labs/devtron/pkg/pipeline/constants" "github.com/devtron-labs/devtron/util/stringsUtil" "golang.org/x/exp/maps" @@ -1500,7 +1500,7 @@ func (handler *PipelineConfigRestHandlerImpl) DeleteMaterial(w http.ResponseWrit func (handler *PipelineConfigRestHandlerImpl) HandleWorkflowWebhook(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) - var wfUpdateReq bean3.CiCdStatus + var wfUpdateReq eventProcessorBean.CiCdStatus err := decoder.Decode(&wfUpdateReq) if err != nil { handler.Logger.Errorw("request err, HandleWorkflowWebhook", "err", err, "payload", wfUpdateReq) diff --git a/env_gen.json b/env_gen.json index b4b889fc79..ad32c0da18 100644 --- a/env_gen.json +++ b/env_gen.json @@ -1 +1 @@ -[{"Category":"CD","Fields":[{"Env":"ARGO_APP_MANUAL_SYNC_TIME","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_HELM_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_PIPELINE_STATUS_TIMEOUT_DURATION","EnvType":"string","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS","EnvType":"int","EnvValue":"12","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_CHART_ARGO_CD_INSTALL_REQUEST_TIMEOUT","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT","EnvType":"int","EnvValue":"6","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXPOSE_CD_METRICS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FEATURE_MIGRATE_ARGOCD_APPLICATION_ENABLE","EnvType":"bool","EnvValue":"false","EnvDescription":"enable migration of external argocd application to devtron pipeline","Example":"","Deprecated":"false"},{"Env":"HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME","EnvType":"string","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IS_INTERNAL_USE","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MIGRATE_DEPLOYMENT_CONFIG_DATA","EnvType":"bool","EnvValue":"false","EnvDescription":"migrate deployment config data from charts table to deployment_config table","Example":"","Deprecated":"false"},{"Env":"PIPELINE_DEGRADED_TIME","EnvType":"string","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_DEVTRON_APP","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_EXTERNAL_HELM_APP","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_HELM_APP","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_LINKED_HELM_APP","EnvType":"int","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUN_HELM_INSTALL_IN_ASYNC_MODE_HELM_APPS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SHOULD_CHECK_NAMESPACE_ON_CLONE","EnvType":"bool","EnvValue":"false","EnvDescription":"should we check if namespace exists or not while cloning app","Example":"","Deprecated":"false"},{"Env":"USE_DEPLOYMENT_CONFIG_DATA","EnvType":"bool","EnvValue":"false","EnvDescription":"use deployment config data from deployment_config table","Example":"","Deprecated":"true"}]},{"Category":"CI_RUNNER","Fields":[{"Env":"AZURE_ACCOUNT_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_ACCOUNT_NAME","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_CACHE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_LOG","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_CONNECTION_INSECURE","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_URL","EnvType":"string","EnvValue":"http://devtron-minio.devtroncd:9000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BASE_LOG_LOCATION_PATH","EnvType":"string","EnvValue":"/home/devtron/","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_GCP_CREDENTIALS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_PROVIDER","EnvType":"","EnvValue":"S3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ACCESS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_BUCKET_VERSIONED","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT_INSECURE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_SECRET_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_CACHE_PATH","EnvType":"string","EnvValue":"/var/lib/devtron/buildx","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_K8S_DRIVER_OPTIONS","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_PROVENANCE_MODE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILD_LOG_TTL_VALUE_IN_SECS","EnvType":"int","EnvValue":"3600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CACHE_LIMIT","EnvType":"int64","EnvValue":"5000000000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_DEFAULT_ADDRESS_POOL_BASE_CIDR","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_DEFAULT_ADDRESS_POOL_SIZE","EnvType":"int","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_LIMIT_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_LIMIT_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"dedicated","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"ci","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_REQ_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_REQ_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_WORKFLOW_EXECUTOR_TYPE","EnvType":"","EnvValue":"AWF","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_WORKFLOW_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"cd-runner","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_DEFAULT_ADDRESS_POOL_BASE_CIDR","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_DEFAULT_ADDRESS_POOL_SIZE","EnvType":"int","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_IGNORE_DOCKER_CACHE","EnvType":"bool","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_LOGS_KEY_PREFIX","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_RUNNER_DOCKER_MTU_VALUE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_SUCCESS_AUTO_TRIGGER_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_VOLUME_MOUNTS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_WORKFLOW_EXECUTOR_TYPE","EnvType":"","EnvValue":"AWF","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_ARTIFACT_KEY_LOCATION","EnvType":"string","EnvValue":"arsenal-v1/ci-artifacts","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_BUCKET","EnvType":"string","EnvValue":"devtron-pro-ci-logs","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_KEY_PREFIX","EnvType":"string","EnvValue":"arsenal-v1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET","EnvType":"string","EnvValue":"ci-caching","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET_REGION","EnvType":"string","EnvValue":"us-east-2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_ARTIFACT_KEY_LOCATION","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_LOGS_BUCKET_REGION","EnvType":"string","EnvValue":"us-east-2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_NAMESPACE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_TIMEOUT","EnvType":"int64","EnvValue":"3600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CI_IMAGE","EnvType":"string","EnvValue":"686244538589.dkr.ecr.us-east-2.amazonaws.com/cirunner:47","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtron-ci","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_TARGET_PLATFORM","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DOCKER_BUILD_CACHE_PATH","EnvType":"string","EnvValue":"/var/lib/docker","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_BUILD_CONTEXT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_WORKFLOW_EXECUTION_STAGE","EnvType":"bool","EnvValue":"true","EnvDescription":"if enabled then we will display build stages separately for CI/Job/Pre-Post CD","Example":"true","Deprecated":"false"},{"Env":"EXTERNAL_BLOB_STORAGE_CM_NAME","EnvType":"string","EnvValue":"blob-storage-cm","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_BLOB_STORAGE_SECRET_NAME","EnvType":"string","EnvValue":"blob-storage-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"dedicated","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"ci","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_API_SECRET","EnvType":"string","EnvValue":"devtroncd-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_PAYLOAD","EnvType":"string","EnvValue":"{\"ciProjectDetails\":[{\"gitRepository\":\"https://github.com/vikram1601/getting-started-nodejs.git\",\"checkoutPath\":\"./abc\",\"commitHash\":\"239077135f8cdeeccb7857e2851348f558cb53d3\",\"commitTime\":\"2022-10-30T20:00:00\",\"branch\":\"master\",\"message\":\"Update README.md\",\"author\":\"User Name \"}],\"dockerImage\":\"445808685819.dkr.ecr.us-east-2.amazonaws.com/orch:23907713-2\"}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_WEB_HOOK_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IGNORE_CM_CS_IN_CI_JOB","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_RETRY_COUNT","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_RETRY_INTERVAL","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCANNER_ENDPOINT","EnvType":"string","EnvValue":"http://image-scanner-new-demo-devtroncd-service.devtroncd:80","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_MAX_RETRIES","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_RETRY_DELAY","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IN_APP_LOGGING_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_CD_WORKFLOW_RUNNER_RETRIES","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_CI_WORKFLOW_RETRIES","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MODE","EnvType":"string","EnvValue":"DEV","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"localhost:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ORCH_HOST","EnvType":"string","EnvValue":"http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ORCH_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PRE_CI_CACHE_PATH","EnvType":"string","EnvValue":"/devtroncd-cache","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SHOW_DOCKER_BUILD_ARGS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SKIP_CI_JOB_BUILD_CACHE_PUSH_PULL","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SKIP_CREATING_ECR_REPO","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TERMINATION_GRACE_PERIOD_SECS","EnvType":"int","EnvValue":"180","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_ARTIFACT_LISTING_QUERY_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_BLOB_STORAGE_CONFIG_IN_CI_WORKFLOW","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_BUILDX","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_DOCKER_API_TO_GET_DIGEST","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_EXTERNAL_NODE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_IMAGE_TAG_FROM_GIT_PROVIDER_FOR_TAG_BASED_BUILD","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"WF_CONTROLLER_INSTANCE_ID","EnvType":"string","EnvValue":"devtron-runner","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"WORKFLOW_CACHE_CONFIG","EnvType":"string","EnvValue":"{}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"WORKFLOW_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"ci-runner","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"DEVTRON","Fields":[{"Env":"-","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ADDITIONAL_NODE_GROUP_LABELS","EnvType":"","EnvValue":"","EnvDescription":"Add comma separated list of additional node group labels to default labels","Example":"karpenter.sh/nodepool,cloud.google.com/gke-nodepool","Deprecated":"false"},{"Env":"APP_SYNC_IMAGE","EnvType":"string","EnvValue":"quay.io/devtron/chart-sync:1227622d-132-3775","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_JOB_RESOURCES_OBJ","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"chart-sync","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_SHUTDOWN_WAIT_DURATION","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_AUTO_SYNC_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_GIT_COMMIT_RETRY_COUNT_ON_CONFLICT","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_GIT_COMMIT_RETRY_DELAY_ON_CONFLICT","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_REPO_REGISTER_RETRY_COUNT","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_REPO_REGISTER_RETRY_DELAY","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ASYNC_BUILDX_CACHE_EXPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BATCH_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_CACHE_MODE_MIN","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_PORT","EnvType":"string","EnvValue":"8000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CExpirationTime","EnvType":"int","EnvValue":"600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_TRIGGER_CRON_TIME","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_WORKFLOW_STATUS_UPDATE_CRON","EnvType":"string","EnvValue":"*/5 * * * *","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLI_CMD_TIMEOUT_GLOBAL_SECONDS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLUSTER_STATUS_CRON_TIME","EnvType":"int","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_TIMEOUT","EnvType":"float64","EnvValue":"3600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_BOM_URL","EnvType":"string","EnvValue":"https://raw.githubusercontent.com/devtron-labs/devtron/%s/charts/devtron/devtron-bom.yaml","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_DEX_SECRET_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_CHART_NAME","EnvType":"string","EnvValue":"devtron-operator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_NAME","EnvType":"string","EnvValue":"devtron","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_REPO_NAME","EnvType":"string","EnvValue":"devtron","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_REPO_URL","EnvType":"string","EnvValue":"https://helm.devtron.ai","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_INSTALLATION_TYPE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_MODULES_IDENTIFIER_IN_HELM_VALUES","EnvType":"string","EnvValue":"installer.modules","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_SECRET_NAME","EnvType":"string","EnvValue":"devtron-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_VERSION_IDENTIFIER_IN_HELM_VALUES","EnvType":"string","EnvValue":"installer.release","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_CID","EnvType":"string","EnvValue":"example-app","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_CLIENT_ID","EnvType":"string","EnvValue":"argo-cd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_CSTOREKEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_JWTKEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_RURL","EnvType":"string","EnvValue":"http://127.0.0.1:8080/callback","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_SECRET","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ECR_REPO_NAME_PREFIX","EnvType":"string","EnvValue":"test/","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_ASYNC_ARGO_CD_INSTALL_DEVTRON_CHART","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_ASYNC_INSTALL_DEVTRON_CHART","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EPHEMERAL_SERVER_VERSION_REGEX","EnvType":"string","EnvValue":"v[1-9]\\.\\b(2[3-9]\\|[3-9][0-9])\\b.*","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EVENT_URL","EnvType":"string","EnvValue":"http://localhost:3000/notify","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXECUTE_WIRE_NIL_CHECKER","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXPOSE_CI_METRICS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FEATURE_RESTART_WORKLOAD_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FEATURE_RESTART_WORKLOAD_WORKER_POOL_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FORCE_SECURITY_SCANNING","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GITOPS_REPO_PREFIX","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GO_RUNTIME_ENV","EnvType":"string","EnvValue":"production","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_ORG_ID","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_PASSWORD","EnvType":"string","EnvValue":"prom-operator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_PORT","EnvType":"string","EnvValue":"8090","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_USERNAME","EnvType":"string","EnvValue":"admin","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"HIDE_IMAGE_TAGGING_HARD_DELETE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IGNORE_AUTOCOMPLETE_AUTH_CHECK","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"INSTALLED_MODULES","EnvType":"","EnvValue":"","EnvDescription":"List of installed modules given in helm values/yaml are written in cm and used by devtron to know which modules are given","Example":"security.trivy,security.clair","Deprecated":"false"},{"Env":"INSTALLER_CRD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_GROUP_NAME","EnvType":"string","EnvValue":"installer.devtron.ai","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_RESOURCE","EnvType":"string","EnvValue":"installers","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_VERSION","EnvType":"string","EnvValue":"v1alpha1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IS_AIR_GAP_ENVIRONMENT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"JwtExpirationTime","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST","EnvType":"int","EnvValue":"25","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_IDLE_CONN_TIMEOUT","EnvType":"int","EnvValue":"300","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_KEEPALIVE","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_TIMEOUT","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TLS_HANDSHAKE_TIMEOUT","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE","EnvType":"int","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_SEND_MSG_SIZE","EnvType":"int","EnvValue":"4","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LENS_TIMEOUT","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LENS_URL","EnvType":"string","EnvValue":"http://lens-milandevtron-service:80","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LIMIT_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LIMIT_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOGGER_DEV_MODE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_SESSION_PER_USER","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MODULE_METADATA_API_URL","EnvType":"string","EnvValue":"https://api.devtron.ai/module?name=%s","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MODULE_STATUS_HANDLING_CRON_DURATION_MIN","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NOTIFICATION_MEDIUM","EnvType":"NotificationMedium","EnvValue":"rest","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"OTEL_COLLECTOR_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PARALLELISM_LIMIT_FOR_TAG_PROCESSING","EnvType":"int","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PLUGIN_NAME","EnvType":"string","EnvValue":"Pull images from container repository","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PROPAGATE_EXTRA_LABELS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PROXY_SERVICE_CONFIG","EnvType":"string","EnvValue":"{}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REQ_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REQ_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RESTRICT_TERMINAL_ACCESS_FOR_NON_SUPER_USER","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUNTIME_CONFIG_LOCAL_DEV","EnvType":"LocalDevMode","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_FORMAT","EnvType":"string","EnvValue":"@{{%s}}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_HANDLE_PRIMITIVES","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_NAME_REGEX","EnvType":"string","EnvValue":"^[a-zA-Z][a-zA-Z0-9_-]{0,62}[a-zA-Z0-9]$","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SOCKET_DISCONNECT_DELAY_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SOCKET_HEARTBEAT_SECONDS","EnvType":"int","EnvValue":"25","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SYSTEM_VAR_PREFIX","EnvType":"string","EnvValue":"DEVTRON_","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"default","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_INACTIVE_DURATION_IN_MINS","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_STATUS_SYNC_In_SECS","EnvType":"int","EnvValue":"600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_APP","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_LOG_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_PASSWORD","EnvType":"string","EnvValue":"postgrespw","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_PORT","EnvType":"string","EnvValue":"55000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_USER","EnvType":"string","EnvValue":"postgres","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TIMEOUT_FOR_FAILED_CI_BUILD","EnvType":"string","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TIMEOUT_IN_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USER_SESSION_DURATION_SECONDS","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_ARTIFACT_LISTING_API_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_CUSTOM_HTTP_TRANSPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_GIT_CLI","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_RBAC_CREATION_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"VARIABLE_CACHE_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"VARIABLE_EXPRESSION_REGEX","EnvType":"string","EnvValue":"@{{([^}]+)}}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"WEBHOOK_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"GITOPS","Fields":[{"Env":"ACD_CM","EnvType":"string","EnvValue":"argocd-cm","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ACD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ACD_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ACD_USERNAME","EnvType":"string","EnvValue":"admin","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GITOPS_SECRET_NAME","EnvType":"string","EnvValue":"devtron-gitops-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RESOURCE_LIST_FOR_REPLICAS","EnvType":"string","EnvValue":"Deployment,Rollout,StatefulSet,ReplicaSet","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RESOURCE_LIST_FOR_REPLICAS_BATCH_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"INFRA_SETUP","Fields":[{"Env":"DASHBOARD_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DASHBOARD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DASHBOARD_PORT","EnvType":"string","EnvValue":"3000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_HOST","EnvType":"string","EnvValue":"http://localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_PORT","EnvType":"string","EnvValue":"5556","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_PROTOCOL","EnvType":"string","EnvValue":"REST","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_TIMEOUT","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_URL","EnvType":"string","EnvValue":"127.0.0.1:7070","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"HELM_CLIENT_URL","EnvType":"string","EnvValue":"127.0.0.1:50051","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"POSTGRES","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"Application name","Example":"","Deprecated":"false"},{"Env":"CASBIN_DATABASE","EnvType":"string","EnvValue":"casbin","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"address of postgres service","Example":"postgresql-postgresql.devtroncd","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"postgres database to be made connection with","Example":"orchestrator, casbin, git_sensor, lens","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"{password}","EnvDescription":"password for postgres, associated with PG_USER","Example":"confidential ;)","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"port of postgresql service","Example":"5432","Deprecated":"false"},{"Env":"PG_READ_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"postgres","EnvDescription":"user for postgres","Example":"postgres","Deprecated":"false"},{"Env":"PG_WRITE_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"RBAC","Fields":[{"Env":"ENFORCER_CACHE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENFORCER_CACHE_EXPIRATION_IN_SEC","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENFORCER_MAX_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_CASBIN_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"}]}] \ No newline at end of file +[{"Category":"CD","Fields":[{"Env":"ARGO_APP_MANUAL_SYNC_TIME","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_HELM_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_PIPELINE_STATUS_TIMEOUT_DURATION","EnvType":"string","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS","EnvType":"int","EnvValue":"12","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_CHART_ARGO_CD_INSTALL_REQUEST_TIMEOUT","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT","EnvType":"int","EnvValue":"6","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXPOSE_CD_METRICS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FEATURE_MIGRATE_ARGOCD_APPLICATION_ENABLE","EnvType":"bool","EnvValue":"false","EnvDescription":"enable migration of external argocd application to devtron pipeline","Example":"","Deprecated":"false"},{"Env":"HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME","EnvType":"string","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IS_INTERNAL_USE","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MIGRATE_DEPLOYMENT_CONFIG_DATA","EnvType":"bool","EnvValue":"false","EnvDescription":"migrate deployment config data from charts table to deployment_config table","Example":"","Deprecated":"false"},{"Env":"PIPELINE_DEGRADED_TIME","EnvType":"string","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_DEVTRON_APP","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_EXTERNAL_HELM_APP","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_HELM_APP","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_LINKED_HELM_APP","EnvType":"int","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUN_HELM_INSTALL_IN_ASYNC_MODE_HELM_APPS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SHOULD_CHECK_NAMESPACE_ON_CLONE","EnvType":"bool","EnvValue":"false","EnvDescription":"should we check if namespace exists or not while cloning app","Example":"","Deprecated":"false"},{"Env":"USE_DEPLOYMENT_CONFIG_DATA","EnvType":"bool","EnvValue":"false","EnvDescription":"use deployment config data from deployment_config table","Example":"","Deprecated":"true"}]},{"Category":"CI_RUNNER","Fields":[{"Env":"AZURE_ACCOUNT_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_ACCOUNT_NAME","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_CACHE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_LOG","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_CONNECTION_INSECURE","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_URL","EnvType":"string","EnvValue":"http://devtron-minio.devtroncd:9000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BASE_LOG_LOCATION_PATH","EnvType":"string","EnvValue":"/home/devtron/","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_GCP_CREDENTIALS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_PROVIDER","EnvType":"","EnvValue":"S3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ACCESS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_BUCKET_VERSIONED","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT_INSECURE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_SECRET_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_CACHE_PATH","EnvType":"string","EnvValue":"/var/lib/devtron/buildx","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_K8S_DRIVER_OPTIONS","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_PROVENANCE_MODE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILD_LOG_TTL_VALUE_IN_SECS","EnvType":"int","EnvValue":"3600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CACHE_LIMIT","EnvType":"int64","EnvValue":"5000000000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_DEFAULT_ADDRESS_POOL_BASE_CIDR","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_DEFAULT_ADDRESS_POOL_SIZE","EnvType":"int","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_LIMIT_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_LIMIT_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"dedicated","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"ci","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_REQ_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_REQ_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_WORKFLOW_EXECUTOR_TYPE","EnvType":"","EnvValue":"AWF","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_WORKFLOW_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"cd-runner","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_DEFAULT_ADDRESS_POOL_BASE_CIDR","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_DEFAULT_ADDRESS_POOL_SIZE","EnvType":"int","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_IGNORE_DOCKER_CACHE","EnvType":"bool","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_LOGS_KEY_PREFIX","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_RUNNER_DOCKER_MTU_VALUE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_SUCCESS_AUTO_TRIGGER_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_VOLUME_MOUNTS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_WORKFLOW_EXECUTOR_TYPE","EnvType":"","EnvValue":"AWF","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_ARTIFACT_KEY_LOCATION","EnvType":"string","EnvValue":"arsenal-v1/ci-artifacts","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_BUCKET","EnvType":"string","EnvValue":"devtron-pro-ci-logs","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_KEY_PREFIX","EnvType":"string","EnvValue":"arsenal-v1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET","EnvType":"string","EnvValue":"ci-caching","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET_REGION","EnvType":"string","EnvValue":"us-east-2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_ARTIFACT_KEY_LOCATION","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_LOGS_BUCKET_REGION","EnvType":"string","EnvValue":"us-east-2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_NAMESPACE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_TIMEOUT","EnvType":"int64","EnvValue":"3600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CI_IMAGE","EnvType":"string","EnvValue":"686244538589.dkr.ecr.us-east-2.amazonaws.com/cirunner:47","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtron-ci","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_TARGET_PLATFORM","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DOCKER_BUILD_CACHE_PATH","EnvType":"string","EnvValue":"/var/lib/docker","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_BUILD_CONTEXT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_WORKFLOW_EXECUTION_STAGE","EnvType":"bool","EnvValue":"true","EnvDescription":"if enabled then we will display build stages separately for CI/Job/Pre-Post CD","Example":"true","Deprecated":"false"},{"Env":"EXTERNAL_BLOB_STORAGE_CM_NAME","EnvType":"string","EnvValue":"blob-storage-cm","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_BLOB_STORAGE_SECRET_NAME","EnvType":"string","EnvValue":"blob-storage-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"dedicated","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"ci","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_API_SECRET","EnvType":"string","EnvValue":"devtroncd-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_PAYLOAD","EnvType":"string","EnvValue":"{\"ciProjectDetails\":[{\"gitRepository\":\"https://github.com/vikram1601/getting-started-nodejs.git\",\"checkoutPath\":\"./abc\",\"commitHash\":\"239077135f8cdeeccb7857e2851348f558cb53d3\",\"commitTime\":\"2022-10-30T20:00:00\",\"branch\":\"master\",\"message\":\"Update README.md\",\"author\":\"User Name \"}],\"dockerImage\":\"445808685819.dkr.ecr.us-east-2.amazonaws.com/orch:23907713-2\"}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_WEB_HOOK_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IGNORE_CM_CS_IN_CI_JOB","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_RETRY_COUNT","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_RETRY_INTERVAL","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCANNER_ENDPOINT","EnvType":"string","EnvValue":"http://image-scanner-new-demo-devtroncd-service.devtroncd:80","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_MAX_RETRIES","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_RETRY_DELAY","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IN_APP_LOGGING_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_CD_WORKFLOW_RUNNER_RETRIES","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_CI_WORKFLOW_RETRIES","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MODE","EnvType":"string","EnvValue":"DEV","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"localhost:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ORCH_HOST","EnvType":"string","EnvValue":"http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ORCH_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PRE_CI_CACHE_PATH","EnvType":"string","EnvValue":"/devtroncd-cache","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SHOW_DOCKER_BUILD_ARGS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SKIP_CI_JOB_BUILD_CACHE_PUSH_PULL","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SKIP_CREATING_ECR_REPO","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TERMINATION_GRACE_PERIOD_SECS","EnvType":"int","EnvValue":"180","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_ARTIFACT_LISTING_QUERY_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_BLOB_STORAGE_CONFIG_IN_CI_WORKFLOW","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_BUILDX","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_DOCKER_API_TO_GET_DIGEST","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_EXTERNAL_NODE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_IMAGE_TAG_FROM_GIT_PROVIDER_FOR_TAG_BASED_BUILD","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"WF_CONTROLLER_INSTANCE_ID","EnvType":"string","EnvValue":"devtron-runner","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"WORKFLOW_CACHE_CONFIG","EnvType":"string","EnvValue":"{}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"WORKFLOW_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"ci-runner","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"DEVTRON","Fields":[{"Env":"-","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ADDITIONAL_NODE_GROUP_LABELS","EnvType":"","EnvValue":"","EnvDescription":"Add comma separated list of additional node group labels to default labels","Example":"karpenter.sh/nodepool,cloud.google.com/gke-nodepool","Deprecated":"false"},{"Env":"APP_SYNC_IMAGE","EnvType":"string","EnvValue":"quay.io/devtron/chart-sync:1227622d-132-3775","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_JOB_RESOURCES_OBJ","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"chart-sync","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_SHUTDOWN_WAIT_DURATION","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_AUTO_SYNC_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_GIT_COMMIT_RETRY_COUNT_ON_CONFLICT","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_GIT_COMMIT_RETRY_DELAY_ON_CONFLICT","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_REPO_REGISTER_RETRY_COUNT","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_REPO_REGISTER_RETRY_DELAY","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ASYNC_BUILDX_CACHE_EXPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BATCH_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_CACHE_MODE_MIN","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_PORT","EnvType":"string","EnvValue":"8000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CExpirationTime","EnvType":"int","EnvValue":"600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_TRIGGER_CRON_TIME","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_WORKFLOW_STATUS_UPDATE_CRON","EnvType":"string","EnvValue":"*/5 * * * *","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLI_CMD_TIMEOUT_GLOBAL_SECONDS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLUSTER_STATUS_CRON_TIME","EnvType":"int","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_TIMEOUT","EnvType":"float64","EnvValue":"3600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_BOM_URL","EnvType":"string","EnvValue":"https://raw.githubusercontent.com/devtron-labs/devtron/%s/charts/devtron/devtron-bom.yaml","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_DEX_SECRET_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_CHART_NAME","EnvType":"string","EnvValue":"devtron-operator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_NAME","EnvType":"string","EnvValue":"devtron","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_REPO_NAME","EnvType":"string","EnvValue":"devtron","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_REPO_URL","EnvType":"string","EnvValue":"https://helm.devtron.ai","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_INSTALLATION_TYPE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_MODULES_IDENTIFIER_IN_HELM_VALUES","EnvType":"string","EnvValue":"installer.modules","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_SECRET_NAME","EnvType":"string","EnvValue":"devtron-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_VERSION_IDENTIFIER_IN_HELM_VALUES","EnvType":"string","EnvValue":"installer.release","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_CID","EnvType":"string","EnvValue":"example-app","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_CLIENT_ID","EnvType":"string","EnvValue":"argo-cd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_CSTOREKEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_JWTKEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_RURL","EnvType":"string","EnvValue":"http://127.0.0.1:8080/callback","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_SCOPES","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_SECRET","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ECR_REPO_NAME_PREFIX","EnvType":"string","EnvValue":"test/","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_ASYNC_ARGO_CD_INSTALL_DEVTRON_CHART","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_ASYNC_INSTALL_DEVTRON_CHART","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EPHEMERAL_SERVER_VERSION_REGEX","EnvType":"string","EnvValue":"v[1-9]\\.\\b(2[3-9]\\|[3-9][0-9])\\b.*","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EVENT_URL","EnvType":"string","EnvValue":"http://localhost:3000/notify","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXECUTE_WIRE_NIL_CHECKER","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXPOSE_CI_METRICS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FEATURE_RESTART_WORKLOAD_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FEATURE_RESTART_WORKLOAD_WORKER_POOL_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FORCE_SECURITY_SCANNING","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GITOPS_REPO_PREFIX","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GO_RUNTIME_ENV","EnvType":"string","EnvValue":"production","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_ORG_ID","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_PASSWORD","EnvType":"string","EnvValue":"prom-operator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_PORT","EnvType":"string","EnvValue":"8090","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_USERNAME","EnvType":"string","EnvValue":"admin","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"HIDE_IMAGE_TAGGING_HARD_DELETE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IGNORE_AUTOCOMPLETE_AUTH_CHECK","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"INSTALLED_MODULES","EnvType":"","EnvValue":"","EnvDescription":"List of installed modules given in helm values/yaml are written in cm and used by devtron to know which modules are given","Example":"security.trivy,security.clair","Deprecated":"false"},{"Env":"INSTALLER_CRD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_GROUP_NAME","EnvType":"string","EnvValue":"installer.devtron.ai","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_RESOURCE","EnvType":"string","EnvValue":"installers","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_VERSION","EnvType":"string","EnvValue":"v1alpha1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IS_AIR_GAP_ENVIRONMENT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"JwtExpirationTime","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST","EnvType":"int","EnvValue":"25","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_IDLE_CONN_TIMEOUT","EnvType":"int","EnvValue":"300","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_KEEPALIVE","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_TIMEOUT","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TLS_HANDSHAKE_TIMEOUT","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE","EnvType":"int","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_SEND_MSG_SIZE","EnvType":"int","EnvValue":"4","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LENS_TIMEOUT","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LENS_URL","EnvType":"string","EnvValue":"http://lens-milandevtron-service:80","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LIMIT_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LIMIT_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOGGER_DEV_MODE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_SESSION_PER_USER","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MODULE_METADATA_API_URL","EnvType":"string","EnvValue":"https://api.devtron.ai/module?name=%s","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MODULE_STATUS_HANDLING_CRON_DURATION_MIN","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NOTIFICATION_MEDIUM","EnvType":"NotificationMedium","EnvValue":"rest","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"OTEL_COLLECTOR_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PARALLELISM_LIMIT_FOR_TAG_PROCESSING","EnvType":"int","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PLUGIN_NAME","EnvType":"string","EnvValue":"Pull images from container repository","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PROPAGATE_EXTRA_LABELS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PROXY_SERVICE_CONFIG","EnvType":"string","EnvValue":"{}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REQ_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REQ_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RESTRICT_TERMINAL_ACCESS_FOR_NON_SUPER_USER","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUNTIME_CONFIG_LOCAL_DEV","EnvType":"LocalDevMode","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_FORMAT","EnvType":"string","EnvValue":"@{{%s}}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_HANDLE_PRIMITIVES","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_NAME_REGEX","EnvType":"string","EnvValue":"^[a-zA-Z][a-zA-Z0-9_-]{0,62}[a-zA-Z0-9]$","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SOCKET_DISCONNECT_DELAY_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SOCKET_HEARTBEAT_SECONDS","EnvType":"int","EnvValue":"25","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SYSTEM_VAR_PREFIX","EnvType":"string","EnvValue":"DEVTRON_","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"default","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_INACTIVE_DURATION_IN_MINS","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_STATUS_SYNC_In_SECS","EnvType":"int","EnvValue":"600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_APP","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_LOG_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_PASSWORD","EnvType":"string","EnvValue":"postgrespw","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_PORT","EnvType":"string","EnvValue":"55000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_USER","EnvType":"string","EnvValue":"postgres","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TIMEOUT_FOR_FAILED_CI_BUILD","EnvType":"string","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TIMEOUT_IN_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USER_SESSION_DURATION_SECONDS","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_ARTIFACT_LISTING_API_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_CUSTOM_HTTP_TRANSPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_GIT_CLI","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_RBAC_CREATION_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"VARIABLE_CACHE_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"VARIABLE_EXPRESSION_REGEX","EnvType":"string","EnvValue":"@{{([^}]+)}}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"WEBHOOK_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"GITOPS","Fields":[{"Env":"ACD_CM","EnvType":"string","EnvValue":"argocd-cm","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ACD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ACD_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ACD_USERNAME","EnvType":"string","EnvValue":"admin","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GITOPS_SECRET_NAME","EnvType":"string","EnvValue":"devtron-gitops-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RESOURCE_LIST_FOR_REPLICAS","EnvType":"string","EnvValue":"Deployment,Rollout,StatefulSet,ReplicaSet","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RESOURCE_LIST_FOR_REPLICAS_BATCH_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"INFRA_SETUP","Fields":[{"Env":"DASHBOARD_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DASHBOARD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DASHBOARD_PORT","EnvType":"string","EnvValue":"3000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_HOST","EnvType":"string","EnvValue":"http://localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_PORT","EnvType":"string","EnvValue":"5556","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_PROTOCOL","EnvType":"string","EnvValue":"REST","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_TIMEOUT","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_URL","EnvType":"string","EnvValue":"127.0.0.1:7070","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"HELM_CLIENT_URL","EnvType":"string","EnvValue":"127.0.0.1:50051","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"POSTGRES","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"Application name","Example":"","Deprecated":"false"},{"Env":"CASBIN_DATABASE","EnvType":"string","EnvValue":"casbin","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"address of postgres service","Example":"postgresql-postgresql.devtroncd","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"postgres database to be made connection with","Example":"orchestrator, casbin, git_sensor, lens","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"{password}","EnvDescription":"password for postgres, associated with PG_USER","Example":"confidential ;)","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"port of postgresql service","Example":"5432","Deprecated":"false"},{"Env":"PG_READ_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"postgres","EnvDescription":"user for postgres","Example":"postgres","Deprecated":"false"},{"Env":"PG_WRITE_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"RBAC","Fields":[{"Env":"ENFORCER_CACHE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENFORCER_CACHE_EXPIRATION_IN_SEC","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENFORCER_MAX_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_CASBIN_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"}]}] \ No newline at end of file diff --git a/env_gen.md b/env_gen.md index f247bf8a1f..63d998590a 100644 --- a/env_gen.md +++ b/env_gen.md @@ -168,6 +168,7 @@ | DEX_CSTOREKEY | string | | | | false | | DEX_JWTKEY | string | | | | false | | DEX_RURL | string |http://127.0.0.1:8080/callback | | | false | + | DEX_SCOPES | | | | | false | | DEX_SECRET | string | | | | false | | DEX_URL | string | | | | false | | ECR_REPO_NAME_PREFIX | string |test/ | | | false | diff --git a/pkg/eventProcessor/bean/workflowEventBean.go b/pkg/eventProcessor/bean/workflowEventBean.go index ddf532fd72..3f401845c4 100644 --- a/pkg/eventProcessor/bean/workflowEventBean.go +++ b/pkg/eventProcessor/bean/workflowEventBean.go @@ -120,3 +120,9 @@ type CiCdStatus struct { DevtronAdministratorInstance string `json:"devtronAdministratorInstance"` *v1alpha1.WorkflowStatus } + +func NewCiCdStatus() CiCdStatus { + return CiCdStatus{ + WorkflowStatus: &v1alpha1.WorkflowStatus{}, + } +} diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index f921103d84..12dd7155eb 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -392,7 +392,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeHibernateBulkAction() error { func (impl *WorkflowEventProcessorImpl) SubscribeCIWorkflowStatusUpdate() error { callback := func(msg *model.PubSubMsg) { - wfStatus := bean.CiCdStatus{} + wfStatus := bean.NewCiCdStatus() err := json.Unmarshal([]byte(msg.Data), &wfStatus) if err != nil { impl.logger.Errorw("error while unmarshalling wf status update", "err", err, "msg", msg.Data) @@ -432,7 +432,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCIWorkflowStatusUpdate() error // add required logging here var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) { - wfStatus := bean.CiCdStatus{} + wfStatus := bean.NewCiCdStatus() err := json.Unmarshal([]byte(msg.Data), &wfStatus) if err != nil { return "error while unmarshalling wf status update", []interface{}{"err", err, "msg", msg.Data} @@ -453,7 +453,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCIWorkflowStatusUpdate() error func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error { callback := func(msg *model.PubSubMsg) { - wfStatus := bean.CiCdStatus{} + wfStatus := bean.NewCiCdStatus() err := json.Unmarshal([]byte(msg.Data), &wfStatus) if err != nil { impl.logger.Error("Error while unmarshalling wfStatus json object", "error", err) @@ -530,7 +530,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error // add required logging here var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) { - wfStatus := bean.CiCdStatus{} + wfStatus := bean.NewCiCdStatus() err := json.Unmarshal([]byte(msg.Data), &wfStatus) if err != nil { return "error while unmarshalling wfStatus json object", []interface{}{"error", err} diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 533a39515b..210616b1b7 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -30,7 +30,7 @@ import ( bean3 "github.com/devtron-labs/devtron/pkg/cluster/bean" repository3 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" - bean6 "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" + eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" "github.com/devtron-labs/devtron/pkg/pipeline/constants" util2 "github.com/devtron-labs/devtron/pkg/pipeline/util" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" @@ -71,7 +71,7 @@ const ( ) type CdHandler interface { - UpdateWorkflow(workflowStatus bean6.CiCdStatus) (int, string, bool, error) + UpdateWorkflow(workflowStatus eventProcessorBean.CiCdStatus) (int, string, bool, error) GetCdBuildHistory(appId int, environmentId int, pipelineId int, offset int, size int) ([]pipelineBean.CdWorkflowWithArtifact, error) GetRunningWorkflowLogs(environmentId int, pipelineId int, workflowId int) (*bufio.Reader, func() error, error) FetchCdWorkflowDetails(appId int, environmentId int, pipelineId int, buildId int) (types.WorkflowResponse, error) @@ -267,7 +267,7 @@ func (impl *CdHandlerImpl) handleForceAbortCaseForCdStage(workflowRunner *pipeli return nil } -func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus bean6.CiCdStatus) (int, string, bool, error) { +func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus eventProcessorBean.CiCdStatus) (int, string, bool, error) { wfStatusRs := impl.extractWorkflowStatus(workflowStatus) workflowName, status, podStatus, message, podName := wfStatusRs.WorkflowName, wfStatusRs.Status, wfStatusRs.PodStatus, wfStatusRs.Message, wfStatusRs.PodName impl.Logger.Debugw("cd workflow status update event for", "wf ", workflowName, "status", status) @@ -331,7 +331,7 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus bean6.CiCdStatus) (int, return savedWorkflow.Id, savedWorkflow.Status, false, nil } -func (impl *CdHandlerImpl) extractWorkflowStatus(workflowStatus bean6.CiCdStatus) *types.WorkflowStatus { +func (impl *CdHandlerImpl) extractWorkflowStatus(workflowStatus eventProcessorBean.CiCdStatus) *types.WorkflowStatus { workflowName := "" status := string(workflowStatus.Phase) podStatus := "Pending" diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 1c60af4c2c..42eea8b72d 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -25,7 +25,7 @@ import ( "github.com/devtron-labs/common-lib/utils/workFlow" "github.com/devtron-labs/devtron/internal/sql/constants" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" - bean6 "github.com/devtron-labs/devtron/pkg/auth/user/bean" + userBean "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/bean/common" "github.com/devtron-labs/devtron/pkg/build/artifacts/imageTagging" bean4 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" @@ -33,7 +33,7 @@ import ( bean5 "github.com/devtron-labs/devtron/pkg/cluster/bean" "github.com/devtron-labs/devtron/pkg/cluster/environment" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" - bean6 "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" + eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" constants2 "github.com/devtron-labs/devtron/pkg/pipeline/constants" util3 "github.com/devtron-labs/devtron/pkg/pipeline/util" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" @@ -77,7 +77,7 @@ import ( type CiHandler interface { HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) - CheckAndReTriggerCI(workflowStatus bean6.CiCdStatus) error + CheckAndReTriggerCI(workflowStatus eventProcessorBean.CiCdStatus) error FetchMaterialsByPipelineId(pipelineId int, showAll bool) ([]pipelineConfig.CiPipelineMaterialResponse, error) FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId int, gitMaterialId int, showAll bool) ([]pipelineConfig.CiPipelineMaterialResponse, error) FetchWorkflowDetails(appId int, pipelineId int, buildId int) (types.WorkflowResponse, error) @@ -91,7 +91,7 @@ type CiHandler interface { GetBuildHistory(pipelineId int, appId int, offset int, size int) ([]types.WorkflowResponse, error) DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) - UpdateWorkflow(workflowStatus bean6.CiCdStatus) (int, bool, error) + UpdateWorkflow(workflowStatus eventProcessorBean.CiCdStatus) (int, bool, error) FetchCiStatusForTriggerView(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) FetchCiStatusForTriggerViewV1(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) @@ -174,7 +174,7 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline return cih } -func (impl *CiHandlerImpl) CheckAndReTriggerCI(workflowStatus bean6.CiCdStatus) error { +func (impl *CiHandlerImpl) CheckAndReTriggerCI(workflowStatus eventProcessorBean.CiCdStatus) error { //return if re-trigger feature is disabled if !impl.config.WorkflowRetriesEnabled() { @@ -225,7 +225,7 @@ func (impl *CiHandlerImpl) reTriggerCi(retryCount int, refCiWorkflow *pipelineCo } trigger := types.Trigger{} - trigger.BuildTriggerObject(refCiWorkflow, ciMaterials, bean6.SYSTEM_USER_ID, true, nil, "") + trigger.BuildTriggerObject(refCiWorkflow, ciMaterials, userBean.SYSTEM_USER_ID, true, nil, "") _, err = impl.ciService.TriggerCiPipeline(trigger) if err != nil { @@ -1113,7 +1113,7 @@ func (impl *CiHandlerImpl) GetHistoricBuildLogs(workflowId int, ciWorkflow *pipe return resp, err } -func ExtractWorkflowStatus(workflowStatus bean6.CiCdStatus) (string, string, string, string, string, string) { +func ExtractWorkflowStatus(workflowStatus eventProcessorBean.CiCdStatus) (string, string, string, string, string, string) { workflowName := "" status := string(workflowStatus.Phase) podStatus := "" @@ -1143,7 +1143,7 @@ func ExtractWorkflowStatus(workflowStatus bean6.CiCdStatus) (string, string, str return workflowName, status, podStatus, message, logLocation, podName } -func (impl *CiHandlerImpl) extractPodStatusAndWorkflow(workflowStatus bean6.CiCdStatus) (string, string, *pipelineConfig.CiWorkflow, error) { +func (impl *CiHandlerImpl) extractPodStatusAndWorkflow(workflowStatus eventProcessorBean.CiCdStatus) (string, string, *pipelineConfig.CiWorkflow, error) { workflowName, status, _, message, _, _ := ExtractWorkflowStatus(workflowStatus) if workflowName == "" { impl.Logger.Errorw("extract workflow status, invalid wf name", "workflowName", workflowName, "status", status, "message", message) @@ -1178,7 +1178,7 @@ func (impl *CiHandlerImpl) getRefWorkflowAndCiRetryCount(savedWorkflow *pipeline return retryCount, savedWorkflow, err } -func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus bean6.CiCdStatus) (int, bool, error) { +func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus eventProcessorBean.CiCdStatus) (int, bool, error) { workflowName, status, podStatus, message, _, podName := ExtractWorkflowStatus(workflowStatus) if workflowName == "" { impl.Logger.Errorw("extract workflow status, invalid wf name", "workflowName", workflowName, "status", status, "podStatus", podStatus, "message", message) From 48773ff35912b061c23c853ca469d9afb0e9c801 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Fri, 4 Apr 2025 00:04:38 +0530 Subject: [PATCH 15/37] fix: updated DevtronAdministratorInstanceLabelKey for system executor --- pkg/pipeline/bean/WorkflowTemplate.go | 18 ++++++++++-------- pkg/pipeline/executors/WorkflowUtils.go | 9 +++++---- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/pkg/pipeline/bean/WorkflowTemplate.go b/pkg/pipeline/bean/WorkflowTemplate.go index d39968fa2e..abc8be4588 100644 --- a/pkg/pipeline/bean/WorkflowTemplate.go +++ b/pkg/pipeline/bean/WorkflowTemplate.go @@ -52,14 +52,16 @@ type WorkflowTemplate struct { } const ( - CI_WORKFLOW_NAME = "ci" - CI_WORKFLOW_WITH_STAGES = "ci-stages-with-env" - CiStage = "CI" - JobStage = "JOB" - CdStage = "CD" - CD_WORKFLOW_NAME = "cd" - CD_WORKFLOW_WITH_STAGES = "cd-stages-with-env" - WorkflowGenerateNamePrefix = "devtron.ai/generate-name-prefix" + CI_WORKFLOW_NAME = "ci" + CI_WORKFLOW_WITH_STAGES = "ci-stages-with-env" + CiStage = "CI" + JobStage = "JOB" + CdStage = "CD" + CD_WORKFLOW_NAME = "cd" + CD_WORKFLOW_WITH_STAGES = "cd-stages-with-env" + WorkflowGenerateNamePrefix = "devtron.ai/generate-name-prefix" + DevtronLabelPurposeKey = "devtron.ai/purpose" + DevtronLabelPurposeWorkflow = "workflow" ) func (workflowTemplate *WorkflowTemplate) GetEntrypoint() string { diff --git a/pkg/pipeline/executors/WorkflowUtils.go b/pkg/pipeline/executors/WorkflowUtils.go index f59022d797..c08bfd8efd 100644 --- a/pkg/pipeline/executors/WorkflowUtils.go +++ b/pkg/pipeline/executors/WorkflowUtils.go @@ -298,9 +298,10 @@ func CheckIfReTriggerRequired(status, message, workflowRunnerStatus string) bool func GetWorkflowLabelsForSystemExecutor(workflowTemplate bean.WorkflowTemplate) map[string]string { return map[string]string{ - DEVTRON_WORKFLOW_LABEL_KEY: DEVTRON_WORKFLOW_LABEL_VALUE, - "devtron.ai/purpose": "workflow", - informerBean.WorkflowTypeLabelKey: workflowTemplate.WorkflowType, - bean.WorkflowGenerateNamePrefix: workflowTemplate.WorkflowNamePrefix, + DEVTRON_WORKFLOW_LABEL_KEY: DEVTRON_WORKFLOW_LABEL_VALUE, + bean.DevtronLabelPurposeKey: bean.DevtronLabelPurposeWorkflow, + informerBean.WorkflowTypeLabelKey: workflowTemplate.WorkflowType, + bean.WorkflowGenerateNamePrefix: workflowTemplate.WorkflowNamePrefix, + informerBean.DevtronAdministratorInstanceLabelKey: workflowTemplate.DevtronInstanceUID, } } From 756be6791bca09155b81df7c10272ca0427a1da3 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Fri, 4 Apr 2025 02:17:03 +0530 Subject: [PATCH 16/37] fix: reverted base64 encoding --- .../in/WorkflowEventProcessorService.go | 19 ++++--------------- pkg/pipeline/WorkflowService.go | 3 +-- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index 12dd7155eb..ed466604d5 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -18,7 +18,6 @@ package in import ( "context" - "encoding/base64" "encoding/json" "errors" "fmt" @@ -404,13 +403,8 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCIWorkflowStatusUpdate() error impl.logger.Errorw("error in getting UCID", "err", err) return } - decodedUCID, err := base64.StdEncoding.DecodeString(wfStatus.DevtronAdministratorInstance) - if err != nil { - impl.logger.Errorw("error in decoding UCID", "err", err) - return - } - if string(decodedUCID) != devtronUCID { - impl.logger.Warnw("mis match in UCID. skipping...", "decodedUCID", string(decodedUCID), "devtronUCID", devtronUCID) + if wfStatus.DevtronAdministratorInstance != devtronUCID { + impl.logger.Warnw("mis match in UCID. skipping...", "devtronAdministratorInstance", wfStatus.DevtronAdministratorInstance, "devtronUCID", devtronUCID) return } } @@ -465,13 +459,8 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error impl.logger.Errorw("error in getting UCID", "err", err) return } - decodedUCID, err := base64.StdEncoding.DecodeString(wfStatus.DevtronAdministratorInstance) - if err != nil { - impl.logger.Errorw("error in decoding UCID", "err", err) - return - } - if string(decodedUCID) != devtronUCID { - impl.logger.Warnw("mis match in UCID. skipping...", "decodedUCID", string(decodedUCID), "devtronUCID", devtronUCID) + if wfStatus.DevtronAdministratorInstance != devtronUCID { + impl.logger.Warnw("mis match in UCID. skipping...", "devtronAdministratorInstance", wfStatus.DevtronAdministratorInstance, "devtronUCID", devtronUCID) return } } diff --git a/pkg/pipeline/WorkflowService.go b/pkg/pipeline/WorkflowService.go index 660350a271..5aadc259bb 100644 --- a/pkg/pipeline/WorkflowService.go +++ b/pkg/pipeline/WorkflowService.go @@ -18,7 +18,6 @@ package pipeline import ( "context" - "encoding/base64" "encoding/json" "errors" "fmt" @@ -223,7 +222,7 @@ func (impl *WorkflowServiceImpl) createWorkflowTemplate(workflowRequest *types.W impl.Logger.Errorw("error in getting UCID", "err", err) return bean3.WorkflowTemplate{}, err } - workflowTemplate.DevtronInstanceUID = base64.StdEncoding.EncodeToString([]byte(devtronUCID)) + workflowTemplate.DevtronInstanceUID = devtronUCID return workflowTemplate, nil } From 3c42cd7b1189efc1f223fa1cfb05cafd08c763ef Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Fri, 4 Apr 2025 03:12:43 +0530 Subject: [PATCH 17/37] fix: ci/ cd workflow msg update --- pkg/pipeline/CdHandler.go | 10 ++++++---- pkg/pipeline/CiHandler.go | 30 ++++++++++++++++-------------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 210616b1b7..1bb887f9ba 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -295,10 +295,12 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus eventProcessorBean.CiCd savedWorkflow.CdArtifactLocation = cdArtifactLocation savedWorkflow.PodStatus = podStatus if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.Status) { - savedWorkflow.Message = message - // NOTE: we are doing this to fix where a pre-cd / post-cd workflow message becomes larger than 1000, and in db we had set the charter limit to 1000 - if len(message) > 1000 { - savedWorkflow.Message = message[:1000] + if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.PodStatus) { + savedWorkflow.Message = message + // NOTE: we are doing this to fix where a pre-cd / post-cd workflow message becomes larger than 1000, and in db we had set the charter limit to 1000 + if len(message) > 1000 { + savedWorkflow.Message = message[:1000] + } } savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time } else { diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 42eea8b72d..9b8596cd91 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -24,7 +24,7 @@ import ( "github.com/devtron-labs/common-lib/utils" "github.com/devtron-labs/common-lib/utils/workFlow" "github.com/devtron-labs/devtron/internal/sql/constants" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" + cdWorkflowBean "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" userBean "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/bean/common" "github.com/devtron-labs/devtron/pkg/build/artifacts/imageTagging" @@ -666,8 +666,8 @@ func (impl *CiHandlerImpl) CancelBuild(workflowId int, forceAbort bool) (int, er return workflow.Id, nil } - workflow.Status = cdWorkflow.WorkflowCancel - if workflow.ExecutorType == cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM { + workflow.Status = cdWorkflowBean.WorkflowCancel + if workflow.ExecutorType == cdWorkflowBean.WORKFLOW_EXECUTOR_TYPE_SYSTEM { workflow.PodStatus = "Failed" workflow.Message = constants2.TERMINATE_MESSAGE } @@ -710,7 +710,7 @@ func (impl *CiHandlerImpl) handleForceAbortCaseForCi(workflow *pipelineConfig.Ci } func (impl *CiHandlerImpl) updateWorkflowForForceAbort(workflow *pipelineConfig.CiWorkflow) error { - workflow.Status = cdWorkflow.WorkflowCancel + workflow.Status = cdWorkflowBean.WorkflowCancel workflow.PodStatus = string(bean.Failed) workflow.Message = constants2.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE err := impl.ciService.UpdateCiWorkflowWithStage(workflow) @@ -878,7 +878,7 @@ func (impl *CiHandlerImpl) getWorkflowLogs(ciWorkflow *pipelineConfig.CiWorkflow if logStream == nil || err != nil { if !ciWorkflow.BlobStorageEnabled { return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "logs-not-stored-in-repository"} - } else if string(v1alpha1.NodeSucceeded) == ciWorkflow.Status || string(v1alpha1.NodeError) == ciWorkflow.Status || string(v1alpha1.NodeFailed) == ciWorkflow.Status || ciWorkflow.Status == cdWorkflow.WorkflowCancel { + } else if string(v1alpha1.NodeSucceeded) == ciWorkflow.Status || string(v1alpha1.NodeError) == ciWorkflow.Status || string(v1alpha1.NodeFailed) == ciWorkflow.Status || ciWorkflow.Status == cdWorkflowBean.WorkflowCancel { impl.Logger.Debugw("pod is not live", "podName", ciWorkflow.PodName, "err", err) return impl.getLogsFromRepository(ciWorkflow, clusterConfig, isExt) } @@ -1200,22 +1200,24 @@ func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus eventProcessorBean.CiCd ciArtifactLocation := fmt.Sprintf(ciArtifactLocationFormat, savedWorkflow.Id, savedWorkflow.Id) if impl.stateChanged(status, podStatus, message, workflowStatus.FinishedAt.Time, savedWorkflow) { - if savedWorkflow.Status != cdWorkflow.WorkflowCancel { + if savedWorkflow.Status != cdWorkflowBean.WorkflowCancel { savedWorkflow.Status = status } savedWorkflow.PodStatus = podStatus - if !slices.Contains(cdWorkflow.WfrTerminalStatusList, savedWorkflow.Status) { - savedWorkflow.Message = message - // NOTE: we are doing this for a quick fix where ci pending message become larger than 250 and in db we had set the charter limit to 250 - if len(message) > 250 { - savedWorkflow.Message = message[:250] + if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.Status) { + if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.PodStatus) { + savedWorkflow.Message = message + // NOTE: we are doing this for a quick fix where ci pending message become larger than 250 and in db we had set the charter limit to 250 + if len(message) > 250 { + savedWorkflow.Message = message[:250] + } } savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time } else { impl.Logger.Warnw("cd stage already in terminal state. skipped message and finishedOn from being updated", "wfId", savedWorkflow.Id, "status", savedWorkflow.Status, "message", message, "finishedOn", workflowStatus.FinishedAt.Time) } - if savedWorkflow.ExecutorType == cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM && savedWorkflow.Status == cdWorkflow.WorkflowCancel { + if savedWorkflow.ExecutorType == cdWorkflowBean.WORKFLOW_EXECUTOR_TYPE_SYSTEM && savedWorkflow.Status == cdWorkflowBean.WorkflowCancel { savedWorkflow.PodStatus = "Failed" savedWorkflow.Message = constants2.TERMINATE_MESSAGE } @@ -1708,7 +1710,7 @@ func (impl *CiHandlerImpl) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuil // skip this and process for next ci workflow } } - if ciWorkflow.ExecutorType == cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM { + if ciWorkflow.ExecutorType == cdWorkflowBean.WORKFLOW_EXECUTOR_TYPE_SYSTEM { if wf.Status == string(v1alpha1.WorkflowFailed) { isPodDeleted = true } @@ -1724,7 +1726,7 @@ func (impl *CiHandlerImpl) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuil ciWorkflow.Status = "Failed" ciWorkflow.PodStatus = "Failed" if isPodDeleted { - ciWorkflow.Message = cdWorkflow.POD_DELETED_MESSAGE + ciWorkflow.Message = cdWorkflowBean.POD_DELETED_MESSAGE //error logging handled inside handlePodDeleted impl.handlePodDeleted(ciWorkflow) } else { From 23befaaadf2b982907da3053c45ebb030823a606 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Fri, 4 Apr 2025 10:27:28 +0530 Subject: [PATCH 18/37] fix: ci/ cd workflow msg update --- pkg/pipeline/CdHandler.go | 18 +++++++++--------- pkg/pipeline/CiHandler.go | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 1bb887f9ba..87d338263e 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -294,18 +294,18 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus eventProcessorBean.CiCd } savedWorkflow.CdArtifactLocation = cdArtifactLocation savedWorkflow.PodStatus = podStatus - if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.Status) { - if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.PodStatus) { - savedWorkflow.Message = message - // NOTE: we are doing this to fix where a pre-cd / post-cd workflow message becomes larger than 1000, and in db we had set the charter limit to 1000 - if len(message) > 1000 { - savedWorkflow.Message = message[:1000] - } + if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.PodStatus) { + savedWorkflow.Message = message + // NOTE: we are doing this to fix where a pre-cd / post-cd workflow message becomes larger than 1000, and in db we had set the charter limit to 1000 + if len(message) > 1000 { + savedWorkflow.Message = message[:1000] + } + if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.Status) { + savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time } - savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time } else { impl.Logger.Warnw("cd stage already in terminal state. skipped message and finishedOn from being updated", - "wfId", savedWorkflow.Id, "status", savedWorkflow.Status, "message", message, "finishedOn", workflowStatus.FinishedAt.Time) + "wfId", savedWorkflow.Id, "podStatus", savedWorkflow.PodStatus, "status", savedWorkflow.Status, "message", message, "finishedOn", workflowStatus.FinishedAt.Time) } savedWorkflow.Name = workflowName // removed log location from here since we are saving it at trigger diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 9b8596cd91..8bffdea516 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -1204,18 +1204,18 @@ func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus eventProcessorBean.CiCd savedWorkflow.Status = status } savedWorkflow.PodStatus = podStatus - if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.Status) { - if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.PodStatus) { - savedWorkflow.Message = message - // NOTE: we are doing this for a quick fix where ci pending message become larger than 250 and in db we had set the charter limit to 250 - if len(message) > 250 { - savedWorkflow.Message = message[:250] - } + if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.PodStatus) { + savedWorkflow.Message = message + // NOTE: we are doing this for a quick fix where ci pending message become larger than 250 and in db we had set the charter limit to 250 + if len(message) > 250 { + savedWorkflow.Message = message[:250] + } + if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.Status) { + savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time } - savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time } else { impl.Logger.Warnw("cd stage already in terminal state. skipped message and finishedOn from being updated", - "wfId", savedWorkflow.Id, "status", savedWorkflow.Status, "message", message, "finishedOn", workflowStatus.FinishedAt.Time) + "wfId", savedWorkflow.Id, "podStatus", savedWorkflow.PodStatus, "status", savedWorkflow.Status, "message", message, "finishedOn", workflowStatus.FinishedAt.Time) } if savedWorkflow.ExecutorType == cdWorkflowBean.WORKFLOW_EXECUTOR_TYPE_SYSTEM && savedWorkflow.Status == cdWorkflowBean.WorkflowCancel { savedWorkflow.PodStatus = "Failed" From 87a2f2593b66a1dcd05fc3718866eafb56e85b55 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Fri, 4 Apr 2025 11:26:19 +0530 Subject: [PATCH 19/37] matrics for retrigger failed event --- internal/middleware/instrument.go | 5 ++++ .../in/WorkflowEventProcessorService.go | 29 ++++++++++++------- pkg/pipeline/CdHandler.go | 12 ++++---- pkg/pipeline/CiHandler.go | 8 ++--- 4 files changed, 33 insertions(+), 21 deletions(-) diff --git a/internal/middleware/instrument.go b/internal/middleware/instrument.go index 0bbb8693e3..7fcc0d9ebf 100644 --- a/internal/middleware/instrument.go +++ b/internal/middleware/instrument.go @@ -117,6 +117,11 @@ var TerminalSessionDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Help: "duration of each terminal session", }, []string{"podName", "namespace", "clusterId"}) +var ReTriggerFailedCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "re_trigger_failed_counter", + Help: "ci/ pre cd/ post cd workflow re-trigger failed counter", +}, []string{"workflowType", "workflowId"}) + // prometheusMiddleware implements mux.MiddlewareFunc. func PrometheusMiddleware(next http.Handler) http.Handler { // prometheus.MustRegister(requestCounter) diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index ed466604d5..8388295dd5 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -27,6 +27,7 @@ import ( "github.com/devtron-labs/common-lib/utils/registry" apiBean "github.com/devtron-labs/devtron/api/bean" client "github.com/devtron-labs/devtron/client/events" + "github.com/devtron-labs/devtron/internal/middleware" "github.com/devtron-labs/devtron/internal/sql/constants" "github.com/devtron-labs/devtron/internal/sql/models" "github.com/devtron-labs/devtron/internal/sql/repository" @@ -409,7 +410,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCIWorkflowStatusUpdate() error } } // update the ci workflow status - _, stateChanged, err := impl.ciHandler.UpdateWorkflow(wfStatus) + ciWfId, stateChanged, err := impl.ciHandler.UpdateWorkflow(wfStatus) if err != nil { impl.logger.Errorw("error on update workflow status", "msg", msg.Data, "err", err) return @@ -418,9 +419,12 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCIWorkflowStatusUpdate() error // check if we need to re-trigger the ci err = impl.ciHandler.CheckAndReTriggerCI(wfStatus) if err != nil { + middleware.ReTriggerFailedCounter.WithLabelValues("CI", strconv.Itoa(ciWfId)).Inc() impl.logger.Errorw("error in checking and re triggering ci", "wfStatus", wfStatus, "err", err) return } + } else { + impl.logger.Debugw("no state change detected for the ci workflow status update, ignoring this event", "workflowRunnerId", ciWfId, "wfStatus", wfStatus) } } @@ -439,7 +443,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCIWorkflowStatusUpdate() error err := impl.pubSubClient.Subscribe(pubsub.WORKFLOW_STATUS_UPDATE_TOPIC, callback, loggerFunc) if err != nil { - impl.logger.Error("err", err) + impl.logger.Error("error in subscribing to ci workflow status update topic", "err", err) return err } return nil @@ -464,10 +468,10 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error return } } - wfrId, wfrStatus, stateChanged, err := impl.cdHandler.UpdateWorkflow(wfStatus) - impl.logger.Debugw("cd UpdateWorkflow for wfStatus", "wfrId", wfrId, "wfrStatus", wfrStatus, "wfStatus", wfStatus) + wfrId, status, stateChanged, err := impl.cdHandler.UpdateWorkflow(wfStatus) + impl.logger.Debugw("cd UpdateWorkflow for wfStatus", "wfrId", wfrId, "status", status, "wfStatus", wfStatus) if err != nil { - impl.logger.Errorw("error in cd workflow status update", "wfrId", wfrId, "wfrStatus", wfrStatus, "wfStatus", wfStatus, "err", err) + impl.logger.Errorw("error in cd workflow status update", "wfrId", wfrId, "status", status, "wfStatus", wfStatus, "err", err) return } @@ -478,7 +482,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error return } - if wfrStatus == string(v1alpha1.NodeFailed) || wfrStatus == string(v1alpha1.NodeError) { + if wfr.Status == string(v1alpha1.NodeFailed) || wfr.Status == string(v1alpha1.NodeError) { if len(wfr.ImagePathReservationIds) > 0 { err := impl.cdHandler.DeactivateImageReservationPathsOnFailure(wfr.ImagePathReservationIds) if err != nil { @@ -491,13 +495,16 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error wfStatusInEvent := string(wfStatus.Phase) if wfStatusInEvent == string(v1alpha1.NodeSucceeded) || wfStatusInEvent == string(v1alpha1.NodeFailed) || wfStatusInEvent == string(v1alpha1.NodeError) { // the re-trigger should only happen when we get a pod deleted event. - if wfr != nil && executors.CheckIfReTriggerRequired(wfrStatus, wfStatus.Message, wfr.Status) { + if executors.CheckIfReTriggerRequired(status, wfStatus.Message, wfr.Status) { err = impl.workflowDagExecutor.HandleCdStageReTrigger(wfr) if err != nil { // check if this log required or not - impl.logger.Errorw("error in HandleCdStageReTrigger", "workflowRunnerId", wfr.Id, "workflowStatus", wfrStatus, "workflowStatusMessage", wfStatus.Message, "error", err) + workflowType := fmt.Sprintf("%s-CD", wfr.WorkflowType) + middleware.ReTriggerFailedCounter.WithLabelValues(workflowType, strconv.Itoa(wfrId)).Inc() + impl.logger.Errorw("error in HandleCdStageReTrigger", "workflowRunnerId", wfr.Id, "status", status, "message", wfStatus.Message, "error", err) + return } - impl.logger.Debugw("re-triggered cd stage", "workflowRunnerId", wfr.Id, "workflowStatus", wfrStatus, "workflowStatusMessage", wfStatus.Message) + impl.logger.Debugw("re-triggered cd stage", "workflowRunnerId", wfr.Id, "status", status, "message", wfStatus.Message) } else { // we send this notification on *workflow_runner* status, both success and failure // during workflow runner failure, particularly when failure occurred due to pod deletion , we get two events from kubewatch. @@ -513,7 +520,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error } } } else { - impl.logger.Debugw("no state change detected for the cd workflow status update, ignoring this event", "workflowRunnerId", wfrId, "wfrStatus", wfrStatus) + impl.logger.Debugw("no state change detected for the cd workflow status update, ignoring this event", "workflowRunnerId", wfrId, "status", status) } } @@ -530,7 +537,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error err := impl.pubSubClient.Subscribe(pubsub.CD_WORKFLOW_STATUS_UPDATE, callback, loggerFunc) if err != nil { - impl.logger.Error("err", err) + impl.logger.Error("error in subscribing to cd workflow status update topic", "err", err) return err } return nil diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 87d338263e..4889e361b6 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -289,11 +289,6 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus eventProcessorBean.CiCd cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() cdArtifactLocation := fmt.Sprintf(cdArtifactLocationFormat, savedWorkflow.CdWorkflowId, savedWorkflow.Id) if impl.stateChanged(status, podStatus, message, workflowStatus.FinishedAt.Time, savedWorkflow) { - if savedWorkflow.Status != cdWorkflowBean.WorkflowCancel { - savedWorkflow.Status = status - } - savedWorkflow.CdArtifactLocation = cdArtifactLocation - savedWorkflow.PodStatus = podStatus if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.PodStatus) { savedWorkflow.Message = message // NOTE: we are doing this to fix where a pre-cd / post-cd workflow message becomes larger than 1000, and in db we had set the charter limit to 1000 @@ -307,6 +302,11 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus eventProcessorBean.CiCd impl.Logger.Warnw("cd stage already in terminal state. skipped message and finishedOn from being updated", "wfId", savedWorkflow.Id, "podStatus", savedWorkflow.PodStatus, "status", savedWorkflow.Status, "message", message, "finishedOn", workflowStatus.FinishedAt.Time) } + if savedWorkflow.Status != cdWorkflowBean.WorkflowCancel { + savedWorkflow.Status = status + } + savedWorkflow.CdArtifactLocation = cdArtifactLocation + savedWorkflow.PodStatus = podStatus savedWorkflow.Name = workflowName // removed log location from here since we are saving it at trigger savedWorkflow.PodName = podName @@ -330,7 +330,7 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus eventProcessorBean.CiCd } return savedWorkflow.Id, savedWorkflow.Status, true, nil } - return savedWorkflow.Id, savedWorkflow.Status, false, nil + return savedWorkflow.Id, status, false, nil } func (impl *CdHandlerImpl) extractWorkflowStatus(workflowStatus eventProcessorBean.CiCdStatus) *types.WorkflowStatus { diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 8bffdea516..8fb4a4b8da 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -1200,10 +1200,6 @@ func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus eventProcessorBean.CiCd ciArtifactLocation := fmt.Sprintf(ciArtifactLocationFormat, savedWorkflow.Id, savedWorkflow.Id) if impl.stateChanged(status, podStatus, message, workflowStatus.FinishedAt.Time, savedWorkflow) { - if savedWorkflow.Status != cdWorkflowBean.WorkflowCancel { - savedWorkflow.Status = status - } - savedWorkflow.PodStatus = podStatus if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.PodStatus) { savedWorkflow.Message = message // NOTE: we are doing this for a quick fix where ci pending message become larger than 250 and in db we had set the charter limit to 250 @@ -1217,6 +1213,10 @@ func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus eventProcessorBean.CiCd impl.Logger.Warnw("cd stage already in terminal state. skipped message and finishedOn from being updated", "wfId", savedWorkflow.Id, "podStatus", savedWorkflow.PodStatus, "status", savedWorkflow.Status, "message", message, "finishedOn", workflowStatus.FinishedAt.Time) } + if savedWorkflow.Status != cdWorkflowBean.WorkflowCancel { + savedWorkflow.Status = status + } + savedWorkflow.PodStatus = podStatus if savedWorkflow.ExecutorType == cdWorkflowBean.WORKFLOW_EXECUTOR_TYPE_SYSTEM && savedWorkflow.Status == cdWorkflowBean.WorkflowCancel { savedWorkflow.PodStatus = "Failed" savedWorkflow.Message = constants2.TERMINATE_MESSAGE From 2a76464e1bb64ce0a232f7498cdc9a9e4153ab71 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Fri, 4 Apr 2025 11:28:08 +0530 Subject: [PATCH 20/37] updated logger --- pkg/eventProcessor/in/WorkflowEventProcessorService.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index 8388295dd5..9816c31920 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -504,7 +504,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error impl.logger.Errorw("error in HandleCdStageReTrigger", "workflowRunnerId", wfr.Id, "status", status, "message", wfStatus.Message, "error", err) return } - impl.logger.Debugw("re-triggered cd stage", "workflowRunnerId", wfr.Id, "status", status, "message", wfStatus.Message) + impl.logger.Infow("re-triggered cd stage", "workflowRunnerId", wfr.Id, "status", status, "message", wfStatus.Message) } else { // we send this notification on *workflow_runner* status, both success and failure // during workflow runner failure, particularly when failure occurred due to pod deletion , we get two events from kubewatch. From e9ac5319a779230c3779dd814c0a1950cf0d9fb9 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Sun, 6 Apr 2025 23:41:34 +0530 Subject: [PATCH 21/37] ci service ent oss completed --- .../CIPipelineRepository_ent.go | 8 + pkg/build/trigger/Service.go | 221 +++++++++--------- pkg/build/trigger/Service_ent.go | 82 ++++++- pkg/pipeline/types/Workflow_ent.go | 2 +- 4 files changed, 205 insertions(+), 108 deletions(-) create mode 100644 internal/sql/repository/pipelineConfig/CIPipelineRepository_ent.go diff --git a/internal/sql/repository/pipelineConfig/CIPipelineRepository_ent.go b/internal/sql/repository/pipelineConfig/CIPipelineRepository_ent.go new file mode 100644 index 0000000000..3a1538c1f8 --- /dev/null +++ b/internal/sql/repository/pipelineConfig/CIPipelineRepository_ent.go @@ -0,0 +1,8 @@ +package pipelineConfig + +import "github.com/devtron-labs/devtron/pkg/bean/common" + +func (p *CiPipeline) GetWorkflowCacheConfig() common.WorkflowCacheConfigType { + //in oss, there is no pipeline level workflow cache config, so we pass inherit to get the app level config + return common.WorkflowCacheConfigInherit +} diff --git a/pkg/build/trigger/Service.go b/pkg/build/trigger/Service.go index 48f217c40e..e35f2a3058 100644 --- a/pkg/build/trigger/Service.go +++ b/pkg/build/trigger/Service.go @@ -2,7 +2,6 @@ package trigger import ( "context" - "encoding/json" "errors" "fmt" "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" @@ -63,6 +62,8 @@ type Service interface { HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) TriggerCiPipeline(trigger types.Trigger) (int, error) + + StartCiWorkflowAndPrepareWfRequest(trigger types.Trigger) (*pipelineConfig.CiPipeline, map[string]string, *pipelineConfig.CiWorkflow, *types.WorkflowRequest, error) } type BuildxCacheFlags struct { @@ -569,15 +570,60 @@ func SetGitCommitValuesForBuildingCommitHash(ciMaterial *pipelineConfig.CiPipeli } func (impl *ServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { - impl.Logger.Debug("ci pipeline manual trigger") - ciMaterials, err := impl.GetCiMaterials(trigger.PipelineId, trigger.CiMaterials) + pipeline, variableSnapshot, savedCiWf, workflowRequest, err := impl.StartCiWorkflowAndPrepareWfRequest(trigger) if err != nil { return 0, err } + workflowRequest.CiPipelineType = trigger.PipelineType + err = impl.executeCiPipeline(workflowRequest) + if err != nil { + impl.Logger.Errorw("error in executing ci pipeline", "err", err) + dbErr := impl.markCurrentCiWorkflowFailed(savedCiWf, err) + if dbErr != nil { + impl.Logger.Errorw("update ci workflow error", "err", dbErr) + } + return 0, err + } + impl.Logger.Debugw("ci triggered", " pipeline ", trigger.PipelineId) + + var variableSnapshotHistories = sliceUtil.GetBeansPtr( + repository4.GetSnapshotBean(savedCiWf.Id, repository4.HistoryReferenceTypeCIWORKFLOW, variableSnapshot)) + if len(variableSnapshotHistories) > 0 { + err = impl.scopedVariableManager.SaveVariableHistoriesForTrigger(variableSnapshotHistories, trigger.TriggeredBy) + if err != nil { + impl.Logger.Errorf("Not able to save variable snapshot for CI trigger %s", err) + } + } + + middleware.CiTriggerCounter.WithLabelValues(pipeline.App.AppName, pipeline.Name).Inc() + go impl.ciService.WriteCITriggerEvent(trigger, pipeline, workflowRequest) + return savedCiWf.Id, err +} + +func (impl *ServiceImpl) GetCiMaterials(pipelineId int, ciMaterials []*pipelineConfig.CiPipelineMaterial) ([]*pipelineConfig.CiPipelineMaterial, error) { + if !(len(ciMaterials) == 0) { + return ciMaterials, nil + } else { + ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(pipelineId) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return nil, err + } + impl.Logger.Debug("ciMaterials for pipeline trigger ", ciMaterials) + return ciMaterials, nil + } +} + +func (impl *ServiceImpl) StartCiWorkflowAndPrepareWfRequest(trigger types.Trigger) (*pipelineConfig.CiPipeline, map[string]string, *pipelineConfig.CiWorkflow, *types.WorkflowRequest, error) { + impl.Logger.Debugw("ci pipeline manual trigger", "request", trigger) + ciMaterials, err := impl.GetCiMaterials(trigger.PipelineId, trigger.CiMaterials) + if err != nil { + return nil, nil, nil, nil, err + } ciPipelineScripts, err := impl.ciPipelineRepository.FindCiScriptsByCiPipelineId(trigger.PipelineId) if err != nil && !util.IsErrNoRows(err) { - return 0, err + return nil, nil, nil, nil, err } var pipeline *pipelineConfig.CiPipeline @@ -592,9 +638,15 @@ func (impl *ServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { ciWorkflowConfigNamespace := impl.config.GetDefaultNamespace() envModal, isJob, err := impl.getEnvironmentForJob(pipeline, trigger) if err != nil { - return 0, err + return nil, nil, nil, nil, err } if isJob && envModal != nil { + + err = impl.checkArgoSetupRequirement(envModal) + if err != nil { + return nil, nil, nil, nil, err + } + ciWorkflowConfigNamespace = envModal.Namespace // This will be populated for jobs running in selected environment @@ -613,15 +665,14 @@ func (impl *ServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { AppName: pipeline.App.AppName, } } - - savedCiWf, err := impl.saveNewWorkflow(pipeline, ciWorkflowConfigNamespace, trigger.CommitHashes, trigger.TriggeredBy, trigger.EnvironmentId, isJob, trigger.ReferenceCiWorkflowId) + savedCiWf, err := impl.saveNewWorkflowForCITrigger(pipeline, ciWorkflowConfigNamespace, trigger.CommitHashes, trigger.TriggeredBy, ciMaterials, trigger.EnvironmentId, isJob, trigger.ReferenceCiWorkflowId) if err != nil { impl.Logger.Errorw("could not save new workflow", "err", err) - return 0, err + return nil, nil, nil, nil, err } - // preCiSteps, postCiSteps, refPluginsData, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(pipeline.Id, ciEvent) request := pipelineConfigBean.NewBuildPrePostStepDataReq(pipeline.Id, pipelineConfigBean.CiStage, scope) + request = updateBuildPrePostStepDataReq(request, trigger) prePostAndRefPluginResponse, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(request) if err != nil { impl.Logger.Errorw("error in getting pre steps data for wf request", "err", err, "ciPipelineId", pipeline.Id) @@ -629,7 +680,7 @@ func (impl *ServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { if dbErr != nil { impl.Logger.Errorw("saving workflow error", "err", dbErr) } - return 0, err + return nil, nil, nil, nil, err } preCiSteps := prePostAndRefPluginResponse.PreStageSteps postCiSteps := prePostAndRefPluginResponse.PostStageSteps @@ -639,15 +690,14 @@ func (impl *ServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { if len(preCiSteps) == 0 && isJob { errMsg := fmt.Sprintf("No tasks are configured in this job pipeline") validationErr := util.NewApiError(http.StatusNotFound, errMsg, errMsg) - - return 0, validationErr + return nil, nil, nil, nil, validationErr } // get env variables of git trigger data and add it in the extraEnvVariables gitTriggerEnvVariables, _, err := impl.ciCdPipelineOrchestrator.GetGitCommitEnvVarDataForCICDStage(savedCiWf.GitTriggers) if err != nil { impl.Logger.Errorw("error in getting gitTrigger env data for stage", "gitTriggers", savedCiWf.GitTriggers, "err", err) - return 0, err + return nil, nil, nil, nil, err } for k, v := range gitTriggerEnvVariables { @@ -657,7 +707,7 @@ func (impl *ServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { workflowRequest, err := impl.buildWfRequestForCiPipeline(pipeline, trigger, ciMaterials, savedCiWf, ciWorkflowConfigNamespace, ciPipelineScripts, preCiSteps, postCiSteps, refPluginsData, isJob) if err != nil { impl.Logger.Errorw("make workflow req", "err", err) - return 0, err + return nil, nil, nil, nil, err } err = impl.handleRuntimeParamsValidations(trigger, ciMaterials, workflowRequest) if err != nil { @@ -667,102 +717,48 @@ func (impl *ServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { if err1 != nil { impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag") } - return 0, err + return nil, nil, nil, nil, err } workflowRequest.Scope = scope workflowRequest.AppId = pipeline.AppId - workflowRequest.BuildxCacheModeMin = impl.buildxCacheFlags.BuildxCacheModeMin - workflowRequest.AsyncBuildxCacheExport = impl.buildxCacheFlags.AsyncBuildxCacheExport - if impl.config != nil && impl.config.BuildxK8sDriverOptions != "" { - err = impl.setBuildxK8sDriverData(workflowRequest) - if err != nil { - impl.Logger.Errorw("error in setBuildxK8sDriverData", "BUILDX_K8S_DRIVER_OPTIONS", impl.config.BuildxK8sDriverOptions, "err", err) - return 0, err - } - } - - // savedCiWf.LogLocation = impl.ciCdConfig.CiDefaultBuildLogsKeyPrefix + "/" + workflowRequest.WorkflowNamePrefix + "/main.log" - savedCiWf.LogLocation = fmt.Sprintf("%s/%s/main.log", impl.config.GetDefaultBuildLogsKeyPrefix(), workflowRequest.WorkflowNamePrefix) - err = impl.updateCiWorkflow(workflowRequest, savedCiWf) - - appLabels, err := impl.appCrudOperationService.GetLabelsByAppId(pipeline.AppId) - if err != nil { - return 0, err - } - workflowRequest.AppLabels = appLabels workflowRequest.Env = envModal if isJob { workflowRequest.Type = pipelineConfigBean.JOB_WORKFLOW_PIPELINE_TYPE } else { workflowRequest.Type = pipelineConfigBean.CI_WORKFLOW_PIPELINE_TYPE } - - workflowRequest.CiPipelineType = trigger.PipelineType - err = impl.executeCiPipeline(workflowRequest) + workflowRequest, err = impl.updateWorkflowRequestWithBuildCacheData(workflowRequest, scope) if err != nil { - impl.Logger.Errorw("error in executing ci pipeline", "err", err) - dbErr := impl.markCurrentCiWorkflowFailed(savedCiWf, err) - if dbErr != nil { - impl.Logger.Errorw("update ci workflow error", "err", dbErr) - } - return 0, err + impl.Logger.Errorw("error, updateWorkflowRequestWithBuildCacheData", "workflowRequest", workflowRequest, "err", err) + return nil, nil, nil, nil, err } - impl.Logger.Debugw("ci triggered", " pipeline ", trigger.PipelineId) - - var variableSnapshotHistories = sliceUtil.GetBeansPtr( - repository4.GetSnapshotBean(savedCiWf.Id, repository4.HistoryReferenceTypeCIWORKFLOW, variableSnapshot)) - if len(variableSnapshotHistories) > 0 { - err = impl.scopedVariableManager.SaveVariableHistoriesForTrigger(variableSnapshotHistories, trigger.TriggeredBy) + if impl.canSetK8sDriverData(workflowRequest) { + err = impl.setBuildxK8sDriverData(workflowRequest) if err != nil { - impl.Logger.Errorf("Not able to save variable snapshot for CI trigger %s", err) + impl.Logger.Errorw("error in setBuildxK8sDriverData", "BUILDX_K8S_DRIVER_OPTIONS", impl.config.BuildxK8sDriverOptions, "err", err) + return nil, nil, nil, nil, err } } - - middleware.CiTriggerCounter.WithLabelValues(pipeline.App.AppName, pipeline.Name).Inc() - go impl.ciService.WriteCITriggerEvent(trigger, pipeline, workflowRequest) - return savedCiWf.Id, err -} - -func (impl *ServiceImpl) GetCiMaterials(pipelineId int, ciMaterials []*pipelineConfig.CiPipelineMaterial) ([]*pipelineConfig.CiPipelineMaterial, error) { - if !(len(ciMaterials) == 0) { - return ciMaterials, nil - } else { - ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(pipelineId) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return nil, err - } - impl.Logger.Debug("ciMaterials for pipeline trigger ", ciMaterials) - return ciMaterials, nil + savedCiWf.LogLocation = fmt.Sprintf("%s/%s/main.log", impl.config.GetDefaultBuildLogsKeyPrefix(), workflowRequest.WorkflowNamePrefix) + err = impl.updateCiWorkflow(workflowRequest, savedCiWf) + appLabels, err := impl.appCrudOperationService.GetLabelsByAppId(pipeline.AppId) + if err != nil { + return nil, nil, nil, nil, err } + workflowRequest.AppLabels = appLabels + workflowRequest = impl.updateWorkflowRequestWithEntSupportData(workflowRequest) + return pipeline, variableSnapshot, savedCiWf, workflowRequest, nil } func (impl *ServiceImpl) setBuildxK8sDriverData(workflowRequest *types.WorkflowRequest) error { - ciBuildConfig := workflowRequest.CiBuildConfig - if ciBuildConfig != nil { - if dockerBuildConfig := ciBuildConfig.DockerBuildConfig; dockerBuildConfig != nil { - k8sDriverOptions, err := impl.getK8sDriverOptions() - if err != nil { - errMsg := "error in parsing BUILDX_K8S_DRIVER_OPTIONS from the devtron-cm, " - err = errors.New(errMsg + "error : " + err.Error()) - impl.Logger.Errorw(errMsg, "err", err) - } - dockerBuildConfig.BuildxK8sDriverOptions = k8sDriverOptions - - } - } - return nil -} - -func (impl *ServiceImpl) getK8sDriverOptions() ([]map[string]string, error) { - buildxK8sDriverOptions := make([]map[string]string, 0) - err := json.Unmarshal([]byte(impl.config.BuildxK8sDriverOptions), &buildxK8sDriverOptions) + dockerBuildConfig := workflowRequest.CiBuildConfig.DockerBuildConfig + k8sDriverOptions, err := impl.getK8sDriverOptions(workflowRequest, dockerBuildConfig.TargetPlatform) if err != nil { - return nil, err - } else { - return buildxK8sDriverOptions, nil + impl.Logger.Errorw("error in parsing BUILDX_K8S_DRIVER_OPTIONS from the devtron-cm", "err", err) } + dockerBuildConfig.BuildxK8sDriverOptions = k8sDriverOptions + return nil } func (impl *ServiceImpl) getEnvironmentForJob(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger) (*repository6.Environment, bool, error) { @@ -796,9 +792,14 @@ func (impl *ServiceImpl) BuildPayload(trigger types.Trigger, pipeline *pipelineC return payload } -func (impl *ServiceImpl) saveNewWorkflow(pipeline *pipelineConfig.CiPipeline, ciWorkflowConfigNamespace string, - commitHashes map[int]pipelineConfig.GitCommit, userId int32, EnvironmentId int, isJob bool, refCiWorkflowId int) (wf *pipelineConfig.CiWorkflow, error error) { +func (impl *ServiceImpl) saveNewWorkflowForCITrigger(pipeline *pipelineConfig.CiPipeline, ciWorkflowConfigNamespace string, + commitHashes map[int]pipelineConfig.GitCommit, userId int32, ciMaterials []*pipelineConfig.CiPipelineMaterial, EnvironmentId int, isJob bool, refCiWorkflowId int) (*pipelineConfig.CiWorkflow, error) { + isCiTriggerBlocked, err := impl.checkIfCITriggerIsBlocked(pipeline, ciMaterials, isJob) + if err != nil { + impl.Logger.Errorw("error, checkIfCITriggerIsBlocked", "pipelineId", pipeline.Id, "err", err) + return &pipelineConfig.CiWorkflow{}, err + } ciWorkflow := &pipelineConfig.CiWorkflow{ Name: pipeline.Name + "-" + strconv.Itoa(pipeline.Id), Status: cdWorkflow.WorkflowStarting, // starting CIStage @@ -817,7 +818,10 @@ func (impl *ServiceImpl) saveNewWorkflow(pipeline *pipelineConfig.CiPipeline, ci ciWorkflow.Namespace = ciWorkflowConfigNamespace ciWorkflow.EnvironmentId = EnvironmentId } - err := impl.ciService.SaveCiWorkflowWithStage(ciWorkflow) + if isCiTriggerBlocked { + return impl.handleWFIfCITriggerIsBlocked(ciWorkflow) + } + err = impl.ciService.SaveCiWorkflowWithStage(ciWorkflow) if err != nil { impl.Logger.Errorw("saving workflow error", "err", err) return &pipelineConfig.CiWorkflow{}, err @@ -882,7 +886,12 @@ func (impl *ServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.Ci CaCert: ciMaterial.GitMaterial.GitProvider.CaCert, }, } - + var err error + ciProjectDetail, err = impl.updateCIProjectDetailWithCloningMode(pipeline.AppId, ciMaterial, ciProjectDetail) + if err != nil { + impl.Logger.Errorw("error, updateCIProjectDetailWithCloningMode", "pipelineId", pipeline.Id, "err", err) + return nil, err + } if ciMaterial.Type == constants.SOURCE_TYPE_WEBHOOK { webhookData := commitHashForPipelineId.WebhookData ciProjectDetail.WebhookData = pipelineConfig.WebhookData{ @@ -1072,15 +1081,8 @@ func (impl *ServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.Ci checkoutPath = filepath.Join(checkoutPath, buildPackConfig.ProjectPath) } - defaultTargetPlatform := impl.config.DefaultTargetPlatform - useBuildx := impl.config.UseBuildx - if ciBuildConfigBean.DockerBuildConfig != nil { - if ciBuildConfigBean.DockerBuildConfig.TargetPlatform == "" && useBuildx { - ciBuildConfigBean.DockerBuildConfig.TargetPlatform = defaultTargetPlatform - ciBuildConfigBean.DockerBuildConfig.UseBuildx = useBuildx - } - ciBuildConfigBean.DockerBuildConfig.BuildxProvenanceMode = impl.config.BuildxProvenanceMode + ciBuildConfigBean = impl.updateCIBuildConfig(ciBuildConfigBean) } workflowRequest := &types.WorkflowRequest{ @@ -1117,16 +1119,15 @@ func (impl *ServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.Ci WorkflowExecutor: impl.config.GetWorkflowExecutorType(), Type: pipelineConfigBean.CI_WORKFLOW_PIPELINE_TYPE, CiArtifactLastFetch: trigger.CiArtifactLastFetch, - RegistryDestinationImageMap: registryDestinationImageMap, RegistryCredentialMap: registryCredentialMap, PluginArtifactStage: pluginArtifactStage, ImageScanMaxRetries: impl.config.ImageScanMaxRetries, ImageScanRetryDelay: impl.config.ImageScanRetryDelay, UseDockerApiToGetDigest: impl.config.UseDockerApiToGetDigest, + RegistryDestinationImageMap: registryDestinationImageMap, } - workflowRequest.SetAwsInspectorConfig("") - //in oss, there is no pipeline level workflow cache config, so we pass inherit to get the app level config - workflowCacheConfig := impl.ciCdPipelineOrchestrator.GetWorkflowCacheConfig(pipeline.App.AppType, trigger.PipelineType, common.WorkflowCacheConfigInherit) + workflowRequest.SetEntOnlyFields(trigger, impl.config) + workflowCacheConfig := impl.ciCdPipelineOrchestrator.GetWorkflowCacheConfig(pipeline.App.AppType, trigger.PipelineType, pipeline.GetWorkflowCacheConfig()) workflowRequest.IgnoreDockerCachePush = !workflowCacheConfig.Value workflowRequest.IgnoreDockerCachePull = !workflowCacheConfig.Value impl.Logger.Debugw("Ignore Cache values", "IgnoreDockerCachePush", workflowRequest.IgnoreDockerCachePush, "IgnoreDockerCachePull", workflowRequest.IgnoreDockerCachePull) @@ -1151,6 +1152,11 @@ func (impl *ServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.Ci } } if dockerRegistry != nil { + workflowRequest, err = impl.updateWorkflowRequestWithRemoteConnConf(dockerRegistry, workflowRequest) + if err != nil { + impl.Logger.Errorw("error occurred updating workflow request", "dockerRegistryId", dockerRegistry.Id, "err", err) + return nil, err + } workflowRequest.DockerRegistryId = dockerRegistry.Id workflowRequest.DockerRegistryType = string(dockerRegistry.RegistryType) workflowRequest.DockerImageTag = dockerImageTag @@ -1457,6 +1463,11 @@ func (impl *ServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, c impl.Logger.Errorw("ci artifact already exists with same image name", "artifact", externalCiArtifact) return fmt.Errorf("ci artifact already exists with same image name") } + workflowRequest, err = impl.updateWorkflowRequestForDigestPull(trigger.PipelineId, workflowRequest) + if err != nil { + impl.Logger.Errorw("error in updating workflow request", "err", err) + return err + } } else { artifactExists, err = impl.ciArtifactRepository.IfArtifactExistByImageDigest(imageDigest, externalCiArtifact, trigger.PipelineId) if err != nil { diff --git a/pkg/build/trigger/Service_ent.go b/pkg/build/trigger/Service_ent.go index cd22daa921..0844a5986b 100644 --- a/pkg/build/trigger/Service_ent.go +++ b/pkg/build/trigger/Service_ent.go @@ -1,12 +1,20 @@ package trigger import ( + "encoding/json" "github.com/devtron-labs/common-lib/imageScan/bean" + repository3 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/util" bean3 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/bean/common" - bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + bean2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" + pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" + "github.com/devtron-labs/devtron/pkg/resourceQualifiers" + "net/http" ) func (impl *ServiceImpl) updateRuntimeParamsForAutoCI(ciPipelineId int, runtimeParameters *common.RuntimeParameters) (*common.RuntimeParameters, error) { @@ -21,6 +29,76 @@ func (impl *ServiceImpl) fetchImageScanExecutionMedium() (*repository.ScanToolMe return &repository.ScanToolMetadata{}, "", nil } -func (impl *ServiceImpl) fetchImageScanExecutionStepsForWfRequest(scanToolMetadata *repository.ScanToolMetadata) ([]*types.ImageScanningSteps, []*bean2.RefPluginObject, error) { +func (impl *ServiceImpl) fetchImageScanExecutionStepsForWfRequest(scanToolMetadata *repository.ScanToolMetadata) ([]*types.ImageScanningSteps, []*pipelineConfigBean.RefPluginObject, error) { return nil, nil, nil } + +func (impl *ServiceImpl) checkIfCITriggerIsBlocked(pipeline *pipelineConfig.CiPipeline, + ciMaterials []*pipelineConfig.CiPipelineMaterial, isJob bool) (bool, error) { + return false, nil +} + +func (impl *ServiceImpl) handleWFIfCITriggerIsBlocked(ciWorkflow *pipelineConfig.CiWorkflow) (*pipelineConfig.CiWorkflow, error) { + impl.Logger.Errorw("cannot trigger pipeline, blocked by mandatory plugin policy", "ciPipelineId", ciWorkflow.CiPipelineId) + return &pipelineConfig.CiWorkflow{}, util.GetApiErrorAdapter(http.StatusInternalServerError, "500", "Invalid flow access, corrupt data possibility", "Invalid flow access, corrupt data possibility") +} + +func (impl *ServiceImpl) checkArgoSetupRequirement(envModal *repository2.Environment) error { + return nil +} + +func (impl *ServiceImpl) updateWorkflowRequestForDigestPull(pipelineId int, workflowRequest *types.WorkflowRequest) (*types.WorkflowRequest, error) { + return workflowRequest, nil +} + +func (impl *ServiceImpl) updateCIProjectDetailWithCloningMode(appId int, ciMaterial *pipelineConfig.CiPipelineMaterial, + ciProjectDetail pipelineConfigBean.CiProjectDetails) (pipelineConfigBean.CiProjectDetails, error) { + return ciProjectDetail, nil +} + +func (impl *ServiceImpl) updateWorkflowRequestWithRemoteConnConf(dockerRegistry *repository3.DockerArtifactStore, + workflowRequest *types.WorkflowRequest) (*types.WorkflowRequest, error) { + return workflowRequest, nil +} + +func (impl *ServiceImpl) updateWorkflowRequestWithEntSupportData(workflowRequest *types.WorkflowRequest) *types.WorkflowRequest { + return workflowRequest +} + +func (impl *ServiceImpl) updateWorkflowRequestWithBuildCacheData(workflowRequest *types.WorkflowRequest, + scope resourceQualifiers.Scope) (*types.WorkflowRequest, error) { + workflowRequest.BuildxCacheModeMin = impl.buildxCacheFlags.BuildxCacheModeMin + workflowRequest.AsyncBuildxCacheExport = impl.buildxCacheFlags.AsyncBuildxCacheExport + return workflowRequest, nil +} + +func (impl *ServiceImpl) canSetK8sDriverData(workflowRequest *types.WorkflowRequest) bool { + return impl.config != nil && impl.config.BuildxK8sDriverOptions != "" && workflowRequest.CiBuildConfig != nil && + workflowRequest.CiBuildConfig.DockerBuildConfig != nil +} + +func (impl *ServiceImpl) getK8sDriverOptions(workflowRequest *types.WorkflowRequest, targetPlatforms string) ([]map[string]string, error) { + buildxK8sDriverOptions := make([]map[string]string, 0) + err := json.Unmarshal([]byte(impl.config.BuildxK8sDriverOptions), &buildxK8sDriverOptions) + if err != nil { + return nil, err + } + return buildxK8sDriverOptions, nil +} + +func (impl *ServiceImpl) updateCIBuildConfig(ciBuildConfigBean *bean2.CiBuildConfigBean) *bean2.CiBuildConfigBean { + defaultTargetPlatform := impl.config.DefaultTargetPlatform + useBuildx := impl.config.UseBuildx + if ciBuildConfigBean.DockerBuildConfig != nil { + if ciBuildConfigBean.DockerBuildConfig.TargetPlatform == "" && useBuildx { + ciBuildConfigBean.DockerBuildConfig.TargetPlatform = defaultTargetPlatform + ciBuildConfigBean.DockerBuildConfig.UseBuildx = useBuildx + } + ciBuildConfigBean.DockerBuildConfig.BuildxProvenanceMode = impl.config.BuildxProvenanceMode + } + return ciBuildConfigBean +} + +func updateBuildPrePostStepDataReq(req *pipelineConfigBean.BuildPrePostStepDataRequest, trigger types.Trigger) *pipelineConfigBean.BuildPrePostStepDataRequest { + return req +} diff --git a/pkg/pipeline/types/Workflow_ent.go b/pkg/pipeline/types/Workflow_ent.go index 2a5c03e78f..ecf78b9e1a 100644 --- a/pkg/pipeline/types/Workflow_ent.go +++ b/pkg/pipeline/types/Workflow_ent.go @@ -30,6 +30,6 @@ func (workflowRequest *WorkflowRequest) SetImageScanningSteps(imageScanningSteps return } -func (workflowRequest *WorkflowRequest) SetAwsInspectorConfig(awsInspectorConfig string) { +func (workflowRequest *WorkflowRequest) SetEntOnlyFields(trigger Trigger, ciConfig *CiConfig) { return } From f6391d776bc2f88aaaa7cb3689f14e1d32b0cf2c Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Mon, 7 Apr 2025 00:02:56 +0530 Subject: [PATCH 22/37] added app to subdirectory for categorising entity --- .../ArgoApplicationRestHandler.go | 4 ++-- api/argoApplication/wire_argoApplication.go | 16 ++++++++-------- api/helm-app/HelmAppRestHandler.go | 4 ++-- api/k8s/application/k8sApplicationRestHandler.go | 4 ++-- cmd/external-app/wire_gen.go | 6 +++--- pkg/app/AppService.go | 2 +- .../InstalledAppDeploymentTypeChangeService.go | 4 ++-- .../FullMode/resource/ResourceTreeService.go | 4 ++-- .../status/resourceTree/ResourceTreeService.go | 4 ++-- .../trigger/devtronApps/TriggerService_ent1.go | 2 +- pkg/k8s/K8sCommonService.go | 2 +- pkg/k8s/application/k8sApplicationService.go | 2 +- .../argoApplication/ArgoApplicationService.go | 6 +++--- .../ArgoApplicationServiceExtended.go | 4 ++-- pkg/{ => nucleus}/argoApplication/bean/bean.go | 0 .../helper/deploymentStatusHelper.go | 0 .../argoApplication/helper/helper.go | 2 +- .../read/ArgoApplicationReadService.go | 4 ++-- .../read/config/ArgoApplicationConfigService.go | 4 ++-- pkg/terminal/terminalSesion.go | 2 +- wire_gen.go | 6 +++--- 21 files changed, 41 insertions(+), 41 deletions(-) rename pkg/{ => nucleus}/argoApplication/ArgoApplicationService.go (97%) rename pkg/{ => nucleus}/argoApplication/ArgoApplicationServiceExtended.go (99%) rename pkg/{ => nucleus}/argoApplication/bean/bean.go (100%) rename pkg/{ => nucleus}/argoApplication/helper/deploymentStatusHelper.go (100%) rename pkg/{ => nucleus}/argoApplication/helper/helper.go (98%) rename pkg/{ => nucleus}/argoApplication/read/ArgoApplicationReadService.go (98%) rename pkg/{ => nucleus}/argoApplication/read/config/ArgoApplicationConfigService.go (97%) diff --git a/api/argoApplication/ArgoApplicationRestHandler.go b/api/argoApplication/ArgoApplicationRestHandler.go index edcbabdca9..27d9a8632e 100644 --- a/api/argoApplication/ArgoApplicationRestHandler.go +++ b/api/argoApplication/ArgoApplicationRestHandler.go @@ -20,9 +20,9 @@ import ( "context" "errors" "github.com/devtron-labs/devtron/api/restHandler/common" - "github.com/devtron-labs/devtron/pkg/argoApplication" - "github.com/devtron-labs/devtron/pkg/argoApplication/read" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read" "go.uber.org/zap" "net/http" "strconv" diff --git a/api/argoApplication/wire_argoApplication.go b/api/argoApplication/wire_argoApplication.go index e3c7381725..6373e274f8 100644 --- a/api/argoApplication/wire_argoApplication.go +++ b/api/argoApplication/wire_argoApplication.go @@ -17,9 +17,9 @@ package argoApplication import ( - "github.com/devtron-labs/devtron/pkg/argoApplication" - "github.com/devtron-labs/devtron/pkg/argoApplication/read" - "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" + argoApplication2 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read/config" "github.com/google/wire" ) @@ -30,9 +30,9 @@ var ArgoApplicationWireSetFull = wire.NewSet( config.NewArgoApplicationConfigServiceImpl, wire.Bind(new(config.ArgoApplicationConfigService), new(*config.ArgoApplicationConfigServiceImpl)), - argoApplication.NewArgoApplicationServiceImpl, - argoApplication.NewArgoApplicationServiceExtendedServiceImpl, - wire.Bind(new(argoApplication.ArgoApplicationService), new(*argoApplication.ArgoApplicationServiceExtendedImpl)), + argoApplication2.NewArgoApplicationServiceImpl, + argoApplication2.NewArgoApplicationServiceExtendedServiceImpl, + wire.Bind(new(argoApplication2.ArgoApplicationService), new(*argoApplication2.ArgoApplicationServiceExtendedImpl)), NewArgoApplicationRestHandlerImpl, wire.Bind(new(ArgoApplicationRestHandler), new(*ArgoApplicationRestHandlerImpl)), @@ -48,8 +48,8 @@ var ArgoApplicationWireSetEA = wire.NewSet( config.NewArgoApplicationConfigServiceImpl, wire.Bind(new(config.ArgoApplicationConfigService), new(*config.ArgoApplicationConfigServiceImpl)), - argoApplication.NewArgoApplicationServiceImpl, - wire.Bind(new(argoApplication.ArgoApplicationService), new(*argoApplication.ArgoApplicationServiceImpl)), + argoApplication2.NewArgoApplicationServiceImpl, + wire.Bind(new(argoApplication2.ArgoApplicationService), new(*argoApplication2.ArgoApplicationServiceImpl)), NewArgoApplicationRestHandlerImpl, wire.Bind(new(ArgoApplicationRestHandler), new(*ArgoApplicationRestHandlerImpl)), diff --git a/api/helm-app/HelmAppRestHandler.go b/api/helm-app/HelmAppRestHandler.go index a21cf730ff..feab53cc95 100644 --- a/api/helm-app/HelmAppRestHandler.go +++ b/api/helm-app/HelmAppRestHandler.go @@ -24,12 +24,12 @@ import ( service2 "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/EAMode" - "github.com/devtron-labs/devtron/pkg/argoApplication" - "github.com/devtron-labs/devtron/pkg/argoApplication/helper" clusterBean "github.com/devtron-labs/devtron/pkg/cluster/bean" clientErrors "github.com/devtron-labs/devtron/pkg/errors" "github.com/devtron-labs/devtron/pkg/fluxApplication" bean2 "github.com/devtron-labs/devtron/pkg/k8s/application/bean" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/pipeline" "net/http" "strconv" diff --git a/api/k8s/application/k8sApplicationRestHandler.go b/api/k8s/application/k8sApplicationRestHandler.go index dd68e18e15..55d42ed15c 100644 --- a/api/k8s/application/k8sApplicationRestHandler.go +++ b/api/k8s/application/k8sApplicationRestHandler.go @@ -33,8 +33,6 @@ import ( client "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/api/restHandler/common" util2 "github.com/devtron-labs/devtron/internal/util" - "github.com/devtron-labs/devtron/pkg/argoApplication/helper" - "github.com/devtron-labs/devtron/pkg/argoApplication/read" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/auth/user" bean4 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" @@ -44,6 +42,8 @@ import ( application2 "github.com/devtron-labs/devtron/pkg/k8s/application" bean2 "github.com/devtron-labs/devtron/pkg/k8s/application/bean" bean3 "github.com/devtron-labs/devtron/pkg/k8s/bean" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read" "github.com/devtron-labs/devtron/pkg/terminal" "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/rbac" diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index e827637c0a..7e1c13fc04 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -75,9 +75,6 @@ import ( "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/common" "github.com/devtron-labs/devtron/pkg/appStore/values/repository" service4 "github.com/devtron-labs/devtron/pkg/appStore/values/service" - "github.com/devtron-labs/devtron/pkg/argoApplication" - read9 "github.com/devtron-labs/devtron/pkg/argoApplication/read" - config3 "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/attributes" "github.com/devtron-labs/devtron/pkg/auth/authentication" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" @@ -118,6 +115,9 @@ import ( read7 "github.com/devtron-labs/devtron/pkg/module/read" "github.com/devtron-labs/devtron/pkg/module/repo" "github.com/devtron-labs/devtron/pkg/module/store" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" + read9 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read" + config3 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool" repository11 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 5bb33db0c8..5d44b6f4a4 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -29,7 +29,6 @@ import ( internalUtil "github.com/devtron-labs/devtron/internal/util" bean3 "github.com/devtron-labs/devtron/pkg/app/bean" installedAppReader "github.com/devtron-labs/devtron/pkg/appStore/installedApp/read" - "github.com/devtron-labs/devtron/pkg/argoApplication/helper" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" bean2 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" @@ -39,6 +38,7 @@ import ( bean6 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/bean" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/read" bean4 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/workflow/cd" "net/url" "strconv" diff --git a/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go b/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go index 2d713500d6..1895a3975b 100644 --- a/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go +++ b/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go @@ -37,8 +37,6 @@ import ( deployment2 "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/EAMode/deployment" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/FullMode/deployment" util2 "github.com/devtron-labs/devtron/pkg/appStore/util" - "github.com/devtron-labs/devtron/pkg/argoApplication" - bean4 "github.com/devtron-labs/devtron/pkg/argoApplication/bean" "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/cluster" "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" @@ -49,6 +47,8 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" bean2 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/k8s" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" + bean4 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" util3 "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" "go.uber.org/zap" diff --git a/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go b/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go index 5a4cd57313..082cb6856e 100644 --- a/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go +++ b/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go @@ -37,12 +37,12 @@ import ( "github.com/devtron-labs/devtron/pkg/appStore/bean" appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/repository" - "github.com/devtron-labs/devtron/pkg/argoApplication" - bean3 "github.com/devtron-labs/devtron/pkg/argoApplication/bean" "github.com/devtron-labs/devtron/pkg/deployment/common" bean2 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/k8s" application3 "github.com/devtron-labs/devtron/pkg/k8s/application" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" + bean3 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" util3 "github.com/devtron-labs/devtron/pkg/util" util2 "github.com/devtron-labs/devtron/util" "github.com/tidwall/gjson" diff --git a/pkg/deployment/deployedApp/status/resourceTree/ResourceTreeService.go b/pkg/deployment/deployedApp/status/resourceTree/ResourceTreeService.go index b6fd6b5f69..4f5ea83ca5 100644 --- a/pkg/deployment/deployedApp/status/resourceTree/ResourceTreeService.go +++ b/pkg/deployment/deployedApp/status/resourceTree/ResourceTreeService.go @@ -34,12 +34,12 @@ import ( "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" "github.com/devtron-labs/devtron/pkg/appStatus" - argoApplication2 "github.com/devtron-labs/devtron/pkg/argoApplication" - bean2 "github.com/devtron-labs/devtron/pkg/argoApplication/bean" read2 "github.com/devtron-labs/devtron/pkg/cluster/environment/read" commonBean "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/k8s" application2 "github.com/devtron-labs/devtron/pkg/k8s/application" + argoApplication2 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" + bean2 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" util2 "github.com/devtron-labs/devtron/util" "go.uber.org/zap" "strconv" diff --git a/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go b/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go index f008f72adc..6c54cf9c57 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go @@ -8,11 +8,11 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/app" bean4 "github.com/devtron-labs/devtron/pkg/app/bean" - "github.com/devtron-labs/devtron/pkg/argoApplication/helper" bean2 "github.com/devtron-labs/devtron/pkg/bean" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/deployment/manifest/publish" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/pipeline/repository" "helm.sh/helm/v3/pkg/chart" ) diff --git a/pkg/k8s/K8sCommonService.go b/pkg/k8s/K8sCommonService.go index 0c3a2a9401..d448dd2311 100644 --- a/pkg/k8s/K8sCommonService.go +++ b/pkg/k8s/K8sCommonService.go @@ -25,12 +25,12 @@ import ( "github.com/devtron-labs/devtron/api/bean/AppView" helmBean "github.com/devtron-labs/devtron/api/helm-app/service/bean" internalUtil "github.com/devtron-labs/devtron/internal/util" - "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" bean2 "github.com/devtron-labs/devtron/pkg/cluster/bean" bean4 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" "github.com/devtron-labs/devtron/pkg/cluster/read" bean3 "github.com/devtron-labs/devtron/pkg/k8s/application/bean" bean5 "github.com/devtron-labs/devtron/pkg/k8s/bean" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read/config" "github.com/devtron-labs/devtron/util" "go.opentelemetry.io/otel" "go.uber.org/zap" diff --git a/pkg/k8s/application/k8sApplicationService.go b/pkg/k8s/application/k8sApplicationService.go index 39588ef15e..7ab5a6b6b7 100644 --- a/pkg/k8s/application/k8sApplicationService.go +++ b/pkg/k8s/application/k8sApplicationService.go @@ -25,7 +25,6 @@ import ( "github.com/devtron-labs/devtron/api/helm-app/gRPC" client "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/api/helm-app/service/bean" - "github.com/devtron-labs/devtron/pkg/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" bean5 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" "github.com/devtron-labs/devtron/pkg/cluster/read" @@ -33,6 +32,7 @@ import ( "github.com/devtron-labs/devtron/pkg/fluxApplication" bean2 "github.com/devtron-labs/devtron/pkg/fluxApplication/bean" bean4 "github.com/devtron-labs/devtron/pkg/k8s/bean" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" "io" v1 "k8s.io/client-go/kubernetes/typed/core/v1" "net/http" diff --git a/pkg/argoApplication/ArgoApplicationService.go b/pkg/nucleus/argoApplication/ArgoApplicationService.go similarity index 97% rename from pkg/argoApplication/ArgoApplicationService.go rename to pkg/nucleus/argoApplication/ArgoApplicationService.go index 48cd31c56d..4bd4f893bc 100644 --- a/pkg/argoApplication/ArgoApplicationService.go +++ b/pkg/nucleus/argoApplication/ArgoApplicationService.go @@ -25,15 +25,15 @@ import ( "github.com/devtron-labs/devtron/api/helm-app/service" argoApplication "github.com/devtron-labs/devtron/client/argocdServer/bean" util2 "github.com/devtron-labs/devtron/internal/util" - "github.com/devtron-labs/devtron/pkg/argoApplication/bean" - "github.com/devtron-labs/devtron/pkg/argoApplication/helper" - "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/cluster/adapter" clusterRepository "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/deployment/common" commonBean "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/k8s/application" k8s2 "github.com/devtron-labs/devtron/pkg/k8s/bean" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read/config" "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/sliceUtil" "go.uber.org/zap" diff --git a/pkg/argoApplication/ArgoApplicationServiceExtended.go b/pkg/nucleus/argoApplication/ArgoApplicationServiceExtended.go similarity index 99% rename from pkg/argoApplication/ArgoApplicationServiceExtended.go rename to pkg/nucleus/argoApplication/ArgoApplicationServiceExtended.go index 0cfb81b2d3..2643bbe432 100644 --- a/pkg/argoApplication/ArgoApplicationServiceExtended.go +++ b/pkg/nucleus/argoApplication/ArgoApplicationServiceExtended.go @@ -28,9 +28,9 @@ import ( argoApplication "github.com/devtron-labs/devtron/client/argocdServer/bean" util2 "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app/appDetails/adapter" - "github.com/devtron-labs/devtron/pkg/argoApplication/bean" - "github.com/devtron-labs/devtron/pkg/argoApplication/read" "github.com/devtron-labs/devtron/pkg/cluster" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read" util3 "github.com/devtron-labs/devtron/pkg/util" "github.com/devtron-labs/devtron/util" "google.golang.org/grpc" diff --git a/pkg/argoApplication/bean/bean.go b/pkg/nucleus/argoApplication/bean/bean.go similarity index 100% rename from pkg/argoApplication/bean/bean.go rename to pkg/nucleus/argoApplication/bean/bean.go diff --git a/pkg/argoApplication/helper/deploymentStatusHelper.go b/pkg/nucleus/argoApplication/helper/deploymentStatusHelper.go similarity index 100% rename from pkg/argoApplication/helper/deploymentStatusHelper.go rename to pkg/nucleus/argoApplication/helper/deploymentStatusHelper.go diff --git a/pkg/argoApplication/helper/helper.go b/pkg/nucleus/argoApplication/helper/helper.go similarity index 98% rename from pkg/argoApplication/helper/helper.go rename to pkg/nucleus/argoApplication/helper/helper.go index 6197e150b4..f68840f416 100644 --- a/pkg/argoApplication/helper/helper.go +++ b/pkg/nucleus/argoApplication/helper/helper.go @@ -4,8 +4,8 @@ import ( "fmt" "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/api/helm-app/gRPC" - "github.com/devtron-labs/devtron/pkg/argoApplication/bean" "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" "strconv" "strings" ) diff --git a/pkg/argoApplication/read/ArgoApplicationReadService.go b/pkg/nucleus/argoApplication/read/ArgoApplicationReadService.go similarity index 98% rename from pkg/argoApplication/read/ArgoApplicationReadService.go rename to pkg/nucleus/argoApplication/read/ArgoApplicationReadService.go index b0f30ccb8f..d95521ba17 100644 --- a/pkg/argoApplication/read/ArgoApplicationReadService.go +++ b/pkg/nucleus/argoApplication/read/ArgoApplicationReadService.go @@ -24,11 +24,11 @@ import ( k8sCommonBean "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/api/helm-app/gRPC" "github.com/devtron-labs/devtron/api/helm-app/service" - "github.com/devtron-labs/devtron/pkg/argoApplication/bean" - "github.com/devtron-labs/devtron/pkg/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/cluster/adapter" clusterRepository "github.com/devtron-labs/devtron/pkg/cluster/repository" clientErrors "github.com/devtron-labs/devtron/pkg/errors" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" "go.uber.org/zap" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/pkg/argoApplication/read/config/ArgoApplicationConfigService.go b/pkg/nucleus/argoApplication/read/config/ArgoApplicationConfigService.go similarity index 97% rename from pkg/argoApplication/read/config/ArgoApplicationConfigService.go rename to pkg/nucleus/argoApplication/read/config/ArgoApplicationConfigService.go index 0c5a8bf819..95aee5d25c 100644 --- a/pkg/argoApplication/read/config/ArgoApplicationConfigService.go +++ b/pkg/nucleus/argoApplication/read/config/ArgoApplicationConfigService.go @@ -6,10 +6,10 @@ import ( "fmt" "github.com/devtron-labs/common-lib/utils/k8s" k8sCommonBean "github.com/devtron-labs/common-lib/utils/k8s/commonBean" - "github.com/devtron-labs/devtron/pkg/argoApplication/bean" - "github.com/devtron-labs/devtron/pkg/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/cluster/adapter" clusterRepository "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" "go.uber.org/zap" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/pkg/terminal/terminalSesion.go b/pkg/terminal/terminalSesion.go index b2b5f2841a..07549991f7 100644 --- a/pkg/terminal/terminalSesion.go +++ b/pkg/terminal/terminalSesion.go @@ -26,13 +26,13 @@ import ( "github.com/caarlos0/env" "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/internal/middleware" - "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/cluster" "github.com/devtron-labs/devtron/pkg/cluster/bean" "github.com/devtron-labs/devtron/pkg/cluster/environment" bean2 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" "github.com/devtron-labs/devtron/pkg/cluster/read" "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read/config" errors1 "github.com/juju/errors" "go.uber.org/zap" "io" diff --git a/wire_gen.go b/wire_gen.go index a59599dfb9..a1365d92a7 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -130,9 +130,6 @@ import ( "github.com/devtron-labs/devtron/pkg/appStore/values/repository" service4 "github.com/devtron-labs/devtron/pkg/appStore/values/service" appWorkflow2 "github.com/devtron-labs/devtron/pkg/appWorkflow" - "github.com/devtron-labs/devtron/pkg/argoApplication" - read22 "github.com/devtron-labs/devtron/pkg/argoApplication/read" - config2 "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/asyncProvider" "github.com/devtron-labs/devtron/pkg/attributes" "github.com/devtron-labs/devtron/pkg/auth/authentication" @@ -229,6 +226,9 @@ import ( "github.com/devtron-labs/devtron/pkg/module/repo" "github.com/devtron-labs/devtron/pkg/module/store" "github.com/devtron-labs/devtron/pkg/notifier" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" + read22 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read" + config2 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/history" From 414712aa5c1dcb163c80891332e8d67dbb2daf50 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Mon, 7 Apr 2025 09:36:27 +0530 Subject: [PATCH 23/37] cd handler and ci handle extracted, executor workflow service executed --- Wire.go | 6 +- .../configure/BuildPipelineRestHandler.go | 8 +- .../DeploymentPipelineRestHandler.go | 6 +- .../configure/PipelineConfigRestHandler.go | 8 +- pkg/build/trigger/Service.go | 440 ++++++++++++++++- .../devtronApps/PostStageTriggerService.go | 2 +- .../devtronApps/PrePostCommonService.go | 337 +++++++++++++ .../devtronApps/PreStageTriggerService.go | 2 +- .../trigger/devtronApps/TriggerService.go | 25 +- pkg/{pipeline => executor}/WorkflowService.go | 8 +- .../WorkflowServiceIT_test.go | 7 +- .../WorkflowService_test.go | 2 +- pkg/executor/wire_executor.go | 8 + pkg/pipeline/BlobStorageConfigService.go | 7 +- pkg/pipeline/CdHandler.go | 342 +------------- pkg/pipeline/CiHandler.go | 444 +----------------- pkg/pipeline/PipelineStageService.go | 5 + pkg/workflow/dag/WorkflowDagExecutor.go | 5 +- wire_gen.go | 23 +- 19 files changed, 864 insertions(+), 821 deletions(-) create mode 100644 pkg/deployment/trigger/devtronApps/PrePostCommonService.go rename pkg/{pipeline => executor}/WorkflowService.go (99%) rename pkg/{pipeline => executor}/WorkflowServiceIT_test.go (99%) rename pkg/{pipeline => executor}/WorkflowService_test.go (99%) create mode 100644 pkg/executor/wire_executor.go diff --git a/Wire.go b/Wire.go index 3401d2d623..1106c3e7dd 100644 --- a/Wire.go +++ b/Wire.go @@ -144,6 +144,7 @@ import ( "github.com/devtron-labs/devtron/pkg/deploymentGroup" "github.com/devtron-labs/devtron/pkg/dockerRegistry" "github.com/devtron-labs/devtron/pkg/eventProcessor" + "github.com/devtron-labs/devtron/pkg/executor" "github.com/devtron-labs/devtron/pkg/generateManifest" "github.com/devtron-labs/devtron/pkg/gitops" "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" @@ -218,7 +219,7 @@ func InitializeApp() (*App, error) { userResource.UserResourceWireSet, policyGovernance.PolicyGovernanceWireSet, resourceScan.ScanningResultWireSet, - + executor.ExecutorWireSet, // -------wireset end ---------- // ------- gitSensor.GetConfig, @@ -477,9 +478,6 @@ func InitializeApp() (*App, error) { util.IntValidator, types.GetCiCdConfig, - pipeline.NewWorkflowServiceImpl, - wire.Bind(new(pipeline.WorkflowService), new(*pipeline.WorkflowServiceImpl)), - pipeline.NewCiServiceImpl, wire.Bind(new(pipeline.CiService), new(*pipeline.CiServiceImpl)), diff --git a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go index 9e749ec207..78ddc57454 100644 --- a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go @@ -974,7 +974,7 @@ func (handler *PipelineConfigRestHandlerImpl) DownloadCiWorkflowArtifacts(w http } //RBAC - file, err := handler.ciHandler.DownloadCiWorkflowArtifacts(pipelineId, buildId) + file, err := handler.ciTriggerService.DownloadCiWorkflowArtifacts(pipelineId, buildId) defer file.Close() if err != nil { handler.Logger.Errorw("service err, DownloadCiWorkflowArtifacts", "err", err, "pipelineId", pipelineId, "buildId", buildId) @@ -1029,7 +1029,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetHistoricBuildLogs(w http.Respon return } //RBAC - resp, err := handler.ciHandler.GetHistoricBuildLogs(workflowId, nil) + resp, err := handler.ciTriggerService.GetHistoricBuildLogs(workflowId, nil) if err != nil { handler.Logger.Errorw("service err, GetHistoricBuildLogs", "err", err, "pipelineId", pipelineId, "workflowId", workflowId) common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) @@ -1168,7 +1168,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetBuildLogs(w http.ResponseWriter return } } - logsReader, cleanUp, err := handler.ciHandler.GetRunningWorkflowLogs(workflowId) + logsReader, cleanUp, err := handler.ciTriggerService.GetRunningWorkflowLogs(workflowId) if err != nil { handler.Logger.Errorw("service err, GetBuildLogs", "err", err, "pipelineId", pipelineId, "workflowId", workflowId, "lastEventId", lastEventId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) @@ -1611,7 +1611,7 @@ func (handler *PipelineConfigRestHandlerImpl) CancelWorkflow(w http.ResponseWrit //RBAC - resp, err := handler.ciHandler.CancelBuild(workflowId, forceAbort) + resp, err := handler.ciTriggerService.CancelBuild(workflowId, forceAbort) if err != nil { handler.Logger.Errorw("service err, CancelWorkflow", "err", err, "workflowId", workflowId, "pipelineId", pipelineId) if util.IsErrNoRows(err) { diff --git a/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go index 52728940d5..a856e40c56 100644 --- a/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go @@ -1643,7 +1643,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetPrePostDeploymentLogs(w http.Re } //RBAC CHECK - logsReader, cleanUp, err := handler.cdHandler.GetRunningWorkflowLogs(environmentId, pipelineId, workflowId) + logsReader, cleanUp, err := handler.cdTriggerService.GetRunningWorkflowLogs(environmentId, pipelineId, workflowId) if err != nil { handler.Logger.Errorw("service err, GetPrePostDeploymentLogs", "err", err, "appId", appId, "environmentId", environmentId, "pipelineId", pipelineId, "workflowId", workflowId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) @@ -1768,7 +1768,7 @@ func (handler *PipelineConfigRestHandlerImpl) DownloadArtifacts(w http.ResponseW } //RBAC CHECK - file, err := handler.cdHandler.DownloadCdWorkflowArtifacts(buildId) + file, err := handler.cdTriggerService.DownloadCdWorkflowArtifacts(buildId) defer file.Close() if err != nil { @@ -1952,7 +1952,7 @@ func (handler *PipelineConfigRestHandlerImpl) CancelStage(w http.ResponseWriter, } //RBAC - resp, err := handler.cdHandler.CancelStage(workflowRunnerId, forceAbort, userId) + resp, err := handler.cdTriggerService.CancelStage(workflowRunnerId, forceAbort, userId) if err != nil { handler.Logger.Errorw("service err, CancelStage", "err", err, "pipelineId", pipelineId, "workflowRunnerId", workflowRunnerId) if util.IsErrNoRows(err) { diff --git a/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go b/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go index 0980abae24..11de4523aa 100644 --- a/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go +++ b/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go @@ -33,6 +33,7 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" security2 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/read" read3 "github.com/devtron-labs/devtron/pkg/team/read" @@ -140,6 +141,7 @@ type PipelineConfigRestHandlerImpl struct { environmentRepository repository2.EnvironmentRepository chartReadService read5.ChartReadService ciTriggerService trigger.Service + cdTriggerService devtronApps.TriggerService } func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger *zap.SugaredLogger, @@ -174,7 +176,9 @@ func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger teamReadService read3.TeamReadService, EnvironmentRepository repository2.EnvironmentRepository, chartReadService read5.ChartReadService, - ciTriggerService trigger.Service) *PipelineConfigRestHandlerImpl { + ciTriggerService trigger.Service, + cdTriggerService devtronApps.TriggerService, +) *PipelineConfigRestHandlerImpl { envConfig := &PipelineRestHandlerEnvConfig{} err := env.Parse(envConfig) if err != nil { @@ -216,6 +220,8 @@ func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger teamReadService: teamReadService, environmentRepository: EnvironmentRepository, chartReadService: chartReadService, + ciTriggerService: ciTriggerService, + cdTriggerService: cdTriggerService, } } diff --git a/pkg/build/trigger/Service.go b/pkg/build/trigger/Service.go index e35f2a3058..0812c485ba 100644 --- a/pkg/build/trigger/Service.go +++ b/pkg/build/trigger/Service.go @@ -1,6 +1,7 @@ package trigger import ( + "bufio" "context" "errors" "fmt" @@ -9,6 +10,7 @@ import ( blob_storage "github.com/devtron-labs/common-lib/blob-storage" "github.com/devtron-labs/common-lib/utils" bean4 "github.com/devtron-labs/common-lib/utils/bean" + "github.com/devtron-labs/common-lib/utils/k8s" commonBean "github.com/devtron-labs/common-lib/workflow" client "github.com/devtron-labs/devtron/client/events" "github.com/devtron-labs/devtron/client/gitSensor" @@ -31,13 +33,20 @@ import ( "github.com/devtron-labs/devtron/pkg/build/pipeline" buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" + "github.com/devtron-labs/devtron/pkg/cluster" + adapter2 "github.com/devtron-labs/devtron/pkg/cluster/adapter" + clusterBean "github.com/devtron-labs/devtron/pkg/cluster/bean" + "github.com/devtron-labs/devtron/pkg/cluster/environment" repository6 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" + "github.com/devtron-labs/devtron/pkg/executor" pipeline2 "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/adapter" pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" + constants2 "github.com/devtron-labs/devtron/pkg/pipeline/constants" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" + util2 "github.com/devtron-labs/devtron/pkg/pipeline/util" "github.com/devtron-labs/devtron/pkg/plugin" bean2 "github.com/devtron-labs/devtron/pkg/plugin/bean" repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository" @@ -47,7 +56,10 @@ import ( "github.com/devtron-labs/devtron/util/sliceUtil" "github.com/go-pg/pg" "go.uber.org/zap" + "io/ioutil" + "k8s.io/client-go/rest" "net/http" + "os" "path" "path/filepath" "slices" @@ -61,9 +73,14 @@ type Service interface { CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) - TriggerCiPipeline(trigger types.Trigger) (int, error) + //TriggerCiPipeline(trigger types.Trigger) (int, error) StartCiWorkflowAndPrepareWfRequest(trigger types.Trigger) (*pipelineConfig.CiPipeline, map[string]string, *pipelineConfig.CiWorkflow, *types.WorkflowRequest, error) + + CancelBuild(workflowId int, forceAbort bool) (int, error) + GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, func() error, error) + GetHistoricBuildLogs(workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) + DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) } type BuildxCacheFlags struct { @@ -73,7 +90,7 @@ type BuildxCacheFlags struct { type ServiceImpl struct { Logger *zap.SugaredLogger - workflowService pipeline2.WorkflowService + workflowService executor.WorkflowService ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository ciPipelineRepository pipelineConfig.CiPipelineRepository ciArtifactRepository repository5.CiArtifactRepository @@ -94,9 +111,14 @@ type ServiceImpl struct { ciService pipeline2.CiService ciWorkflowRepository pipelineConfig.CiWorkflowRepository gitSensorClient gitSensor.Client + ciLogService pipeline2.CiLogService + blobConfigStorageService pipeline2.BlobStorageConfigService + clusterService cluster.ClusterService + envService environment.EnvironmentService + K8sUtil *k8s.K8sServiceImpl } -func NewServiceImpl(Logger *zap.SugaredLogger, workflowService pipeline2.WorkflowService, +func NewServiceImpl(Logger *zap.SugaredLogger, workflowService executor.WorkflowService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, ciPipelineRepository pipelineConfig.CiPipelineRepository, ciArtifactRepository repository5.CiArtifactRepository, @@ -114,6 +136,11 @@ func NewServiceImpl(Logger *zap.SugaredLogger, workflowService pipeline2.Workflo ciService pipeline2.CiService, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, gitSensorClient gitSensor.Client, + ciLogService pipeline2.CiLogService, + blobConfigStorageService pipeline2.BlobStorageConfigService, + clusterService cluster.ClusterService, + envService environment.EnvironmentService, + K8sUtil *k8s.K8sServiceImpl, ) *ServiceImpl { buildxCacheFlags := &BuildxCacheFlags{} err := env.Parse(buildxCacheFlags) @@ -142,6 +169,11 @@ func NewServiceImpl(Logger *zap.SugaredLogger, workflowService pipeline2.Workflo ciService: ciService, ciWorkflowRepository: ciWorkflowRepository, gitSensorClient: gitSensorClient, + ciLogService: ciLogService, + blobConfigStorageService: blobConfigStorageService, + clusterService: clusterService, + envService: envService, + K8sUtil: K8sUtil, } config, err := types.GetCiConfig() if err != nil { @@ -227,7 +259,7 @@ func (impl *ServiceImpl) reTriggerCi(retryCount int, refCiWorkflow *pipelineConf "runtimeParameters", trigger.RuntimeParameters, "err", err) return err } - _, err = impl.TriggerCiPipeline(trigger) + _, err = impl.triggerCiPipeline(trigger) if err != nil { impl.Logger.Errorw("error occurred in re-triggering ciWorkflow", "triggerDetails", trigger, "err", err) @@ -265,7 +297,7 @@ func (impl *ServiceImpl) HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) PipelineType: ciTriggerRequest.PipelineType, CiArtifactLastFetch: createdOn, } - id, err := impl.TriggerCiPipeline(trigger) + id, err := impl.triggerCiPipeline(trigger) if err != nil { return 0, err @@ -318,7 +350,7 @@ func (impl *ServiceImpl) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRe TriggeredBy: gitCiTriggerRequest.TriggeredBy, RuntimeParameters: runtimeParams, } - id, err := impl.TriggerCiPipeline(trigger) + id, err := impl.triggerCiPipeline(trigger) if err != nil { return 0, err } @@ -569,7 +601,7 @@ func SetGitCommitValuesForBuildingCommitHash(ciMaterial *pipelineConfig.CiPipeli return newGitCommit } -func (impl *ServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { +func (impl *ServiceImpl) triggerCiPipeline(trigger types.Trigger) (int, error) { pipeline, variableSnapshot, savedCiWf, workflowRequest, err := impl.StartCiWorkflowAndPrepareWfRequest(trigger) if err != nil { return 0, err @@ -1529,3 +1561,397 @@ func (impl *ServiceImpl) markCurrentCiWorkflowFailed(savedCiWf *pipelineConfig.C } return nil } + +func (impl *ServiceImpl) CancelBuild(workflowId int, forceAbort bool) (int, error) { + workflow, err := impl.ciWorkflowRepository.FindById(workflowId) + if err != nil { + impl.Logger.Errorw("error in finding ci-workflow by workflow id", "ciWorkflowId", workflowId, "err", err) + return 0, err + } + isExt := workflow.Namespace != constants2.DefaultCiWorkflowNamespace + var env *repository6.Environment + var restConfig *rest.Config + if isExt { + restConfig, err = impl.getRestConfig(workflow) + if err != nil { + return 0, err + } + } + // Terminate workflow + cancelWfDtoRequest := &types.CancelWfRequestDto{ + ExecutorType: workflow.ExecutorType, + WorkflowName: workflow.Name, + Namespace: workflow.Namespace, + RestConfig: restConfig, + IsExt: isExt, + Environment: env, + } + // Terminate workflow + err = impl.workflowService.TerminateWorkflow(cancelWfDtoRequest) + if err != nil && forceAbort { + impl.Logger.Errorw("error in terminating workflow, with force abort flag flag as true", "workflowName", workflow.Name, "err", err) + + cancelWfDtoRequest.WorkflowGenerateName = fmt.Sprintf("%d-%s", workflowId, workflow.Name) + err1 := impl.workflowService.TerminateDanglingWorkflows(cancelWfDtoRequest) + if err1 != nil { + impl.Logger.Errorw("error in terminating dangling workflows", "cancelWfDtoRequest", cancelWfDtoRequest, "err", err) + // ignoring error here in case of force abort, confirmed from product + } + } else if err != nil && strings.Contains(err.Error(), "cannot find workflow") { + return 0, &util.ApiError{Code: "200", HttpStatusCode: http.StatusBadRequest, UserMessage: err.Error()} + } else if err != nil { + impl.Logger.Errorw("cannot terminate wf", "err", err) + return 0, err + } + if forceAbort { + err = impl.handleForceAbortCaseForCi(workflow, forceAbort) + if err != nil { + impl.Logger.Errorw("error in handleForceAbortCaseForCi", "forceAbortFlag", forceAbort, "workflow", workflow, "err", err) + return 0, err + } + return workflow.Id, nil + } + + workflow.Status = cdWorkflow.WorkflowCancel + if workflow.ExecutorType == cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM { + workflow.PodStatus = "Failed" + workflow.Message = constants2.TERMINATE_MESSAGE + } + err = impl.ciService.UpdateCiWorkflowWithStage(workflow) + if err != nil { + impl.Logger.Errorw("cannot update deleted workflow status, but wf deleted", "err", err) + return 0, err + } + imagePathReservationId := workflow.ImagePathReservationId + err = impl.customTagService.DeactivateImagePathReservation(imagePathReservationId) + if err != nil { + impl.Logger.Errorw("error in marking image tag unreserved", "err", err) + return 0, err + } + imagePathReservationIds := workflow.ImagePathReservationIds + if len(imagePathReservationIds) > 0 { + err = impl.customTagService.DeactivateImagePathReservationByImageIds(imagePathReservationIds) + if err != nil { + impl.Logger.Errorw("error in marking image tag unreserved", "err", err) + return 0, err + } + } + return workflow.Id, nil +} + +func (impl *ServiceImpl) handleForceAbortCaseForCi(workflow *pipelineConfig.CiWorkflow, forceAbort bool) error { + isWorkflowInNonTerminalStage := workflow.Status == string(v1alpha1.NodePending) || workflow.Status == string(v1alpha1.NodeRunning) + if !isWorkflowInNonTerminalStage { + if forceAbort { + return impl.updateWorkflowForForceAbort(workflow) + } else { + return &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "cannot cancel build, build not in progress"} + } + } + //this arises when someone deletes the workflow in resource browser and wants to force abort a ci + if workflow.Status == string(v1alpha1.NodeRunning) && forceAbort { + return impl.updateWorkflowForForceAbort(workflow) + } + return nil +} + +func (impl *ServiceImpl) updateWorkflowForForceAbort(workflow *pipelineConfig.CiWorkflow) error { + workflow.Status = cdWorkflow.WorkflowCancel + workflow.PodStatus = string(bean.Failed) + workflow.Message = constants2.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE + err := impl.ciService.UpdateCiWorkflowWithStage(workflow) + if err != nil { + impl.Logger.Errorw("error in updating workflow status", "err", err) + return err + } + return nil +} + +func (impl *ServiceImpl) getRestConfig(workflow *pipelineConfig.CiWorkflow) (*rest.Config, error) { + env, err := impl.envRepository.FindById(workflow.EnvironmentId) + if err != nil { + impl.Logger.Errorw("could not fetch stage env", "err", err) + return nil, err + } + + clusterBean := adapter2.GetClusterBean(*env.Cluster) + + clusterConfig := clusterBean.GetClusterConfig() + restConfig, err := impl.K8sUtil.GetRestConfigByCluster(clusterConfig) + if err != nil { + impl.Logger.Errorw("error in getting rest config by cluster id", "err", err) + return nil, err + } + return restConfig, nil +} + +func (impl *ServiceImpl) GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, func() error, error) { + ciWorkflow, err := impl.ciWorkflowRepository.FindById(workflowId) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return nil, nil, err + } + return impl.getWorkflowLogs(ciWorkflow) +} + +func (impl *ServiceImpl) getWorkflowLogs(ciWorkflow *pipelineConfig.CiWorkflow) (*bufio.Reader, func() error, error) { + if string(v1alpha1.NodePending) == ciWorkflow.PodStatus { + return bufio.NewReader(strings.NewReader("")), func() error { return nil }, nil + } + ciLogRequest := types.BuildLogRequest{ + PodName: ciWorkflow.PodName, + Namespace: ciWorkflow.Namespace, + } + isExt := false + clusterConfig := &k8s.ClusterConfig{} + if ciWorkflow.EnvironmentId != 0 { + env, err := impl.envRepository.FindById(ciWorkflow.EnvironmentId) + if err != nil { + return nil, nil, err + } + var clusterBean clusterBean.ClusterBean + if env != nil && env.Cluster != nil { + clusterBean = adapter2.GetClusterBean(*env.Cluster) + } + clusterConfig = clusterBean.GetClusterConfig() + isExt = true + } + + logStream, cleanUp, err := impl.ciLogService.FetchRunningWorkflowLogs(ciLogRequest, clusterConfig, isExt) + if logStream == nil || err != nil { + if !ciWorkflow.BlobStorageEnabled { + return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "logs-not-stored-in-repository"} + } else if string(v1alpha1.NodeSucceeded) == ciWorkflow.Status || string(v1alpha1.NodeError) == ciWorkflow.Status || string(v1alpha1.NodeFailed) == ciWorkflow.Status || ciWorkflow.Status == cdWorkflow.WorkflowCancel { + impl.Logger.Debugw("pod is not live", "podName", ciWorkflow.PodName, "err", err) + return impl.getLogsFromRepository(ciWorkflow, clusterConfig, isExt) + } + if err != nil { + impl.Logger.Errorw("err on fetch workflow logs", "err", err) + return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: err.Error()} + } else if logStream == nil { + return nil, cleanUp, fmt.Errorf("no logs found for pod %s", ciWorkflow.PodName) + } + } + logReader := bufio.NewReader(logStream) + return logReader, cleanUp, err +} + +func (impl *ServiceImpl) getLogsFromRepository(ciWorkflow *pipelineConfig.CiWorkflow, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { + impl.Logger.Debug("getting historic logs", "ciWorkflowId", ciWorkflow.Id) + ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() + ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion + logsFilePath := impl.config.GetDefaultBuildLogsKeyPrefix() + "/" + ciWorkflow.Name + "/main.log" // this is for backward compatibilty + if strings.Contains(ciWorkflow.LogLocation, "main.log") { + logsFilePath = ciWorkflow.LogLocation + } + ciLogRequest := types.BuildLogRequest{ + PipelineId: ciWorkflow.CiPipelineId, + WorkflowId: ciWorkflow.Id, + PodName: ciWorkflow.PodName, + LogsFilePath: logsFilePath, + CloudProvider: impl.config.CloudProvider, + AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, + AccountName: impl.config.AzureAccountName, + BlobContainerName: impl.config.AzureBlobContainerCiLog, + AccountKey: impl.config.AzureAccountKey, + }, + AwsS3BaseConfig: &blob_storage.AwsS3BaseConfig{ + AccessKey: impl.config.BlobStorageS3AccessKey, + Passkey: impl.config.BlobStorageS3SecretKey, + EndpointUrl: impl.config.BlobStorageS3Endpoint, + IsInSecure: impl.config.BlobStorageS3EndpointInsecure, + BucketName: ciConfigLogsBucket, + Region: ciConfigCiCacheRegion, + VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, + }, + GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ + BucketName: ciConfigLogsBucket, + CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, + }, + } + useExternalBlobStorage := pipeline2.IsExternalBlobStorageEnabled(isExt, impl.config.UseBlobStorageConfigInCiWorkflow) + if useExternalBlobStorage { + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) + if err != nil { + impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, nil, err + } + rq := &ciLogRequest + rq.SetBuildLogRequest(cmConfig, secretConfig) + } + oldLogsStream, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, ciLogRequest) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return nil, nil, err + } + logReader := bufio.NewReader(oldLogsStream) + return logReader, cleanUp, err +} + +func (impl *ServiceImpl) GetHistoricBuildLogs(workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) { + var err error + if ciWorkflow == nil { + ciWorkflow, err = impl.ciWorkflowRepository.FindById(workflowId) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return nil, err + } + } + ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() + ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion + ciLogRequest := types.BuildLogRequest{ + PipelineId: ciWorkflow.CiPipelineId, + WorkflowId: ciWorkflow.Id, + PodName: ciWorkflow.PodName, + LogsFilePath: ciWorkflow.LogLocation, + CloudProvider: impl.config.CloudProvider, + AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, + AccountName: impl.config.AzureAccountName, + BlobContainerName: impl.config.AzureBlobContainerCiLog, + AccountKey: impl.config.AzureAccountKey, + }, + AwsS3BaseConfig: &blob_storage.AwsS3BaseConfig{ + AccessKey: impl.config.BlobStorageS3AccessKey, + Passkey: impl.config.BlobStorageS3SecretKey, + EndpointUrl: impl.config.BlobStorageS3Endpoint, + IsInSecure: impl.config.BlobStorageS3EndpointInsecure, + BucketName: ciConfigLogsBucket, + Region: ciConfigCiCacheRegion, + VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, + }, + GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ + BucketName: ciConfigLogsBucket, + CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, + }, + } + useExternalBlobStorage := pipeline2.IsExternalBlobStorageEnabled(ciWorkflow.IsExternalRunInJobType(), impl.config.UseBlobStorageConfigInCiWorkflow) + if useExternalBlobStorage { + envBean, err := impl.envService.FindById(ciWorkflow.EnvironmentId) + if err != nil { + impl.Logger.Errorw("error in getting envBean by envId", "err", err, "envId", ciWorkflow.EnvironmentId) + return nil, err + } + clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(envBean.ClusterId) + if err != nil { + impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig by clusterId", "err", err, "clusterId", envBean.ClusterId) + return nil, err + } + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) + if err != nil { + impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, err + } + rq := &ciLogRequest + rq.SetBuildLogRequest(cmConfig, secretConfig) + } + logsFile, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, ciLogRequest) + logs, err := ioutil.ReadFile(logsFile.Name()) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return map[string]string{}, err + } + logStr := string(logs) + resp := make(map[string]string) + resp["logs"] = logStr + defer cleanUp() + return resp, err +} + +func (impl *ServiceImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) { + ciWorkflow, err := impl.ciWorkflowRepository.FindById(buildId) + if err != nil { + impl.Logger.Errorw("unable to fetch ciWorkflow", "err", err) + return nil, err + } + useExternalBlobStorage := pipeline2.IsExternalBlobStorageEnabled(ciWorkflow.IsExternalRunInJobType(), impl.config.UseBlobStorageConfigInCiWorkflow) + if !ciWorkflow.BlobStorageEnabled { + return nil, errors.New("logs-not-stored-in-repository") + } + + if ciWorkflow.CiPipelineId != pipelineId { + impl.Logger.Error("invalid request, wf not in pipeline") + return nil, errors.New("invalid request, wf not in pipeline") + } + + ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() + item := strconv.Itoa(ciWorkflow.Id) + ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion + azureBlobConfig := &blob_storage.AzureBlobBaseConfig{ + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, + AccountName: impl.config.AzureAccountName, + BlobContainerName: impl.config.AzureBlobContainerCiLog, + AccountKey: impl.config.AzureAccountKey, + } + awsS3BaseConfig := &blob_storage.AwsS3BaseConfig{ + AccessKey: impl.config.BlobStorageS3AccessKey, + Passkey: impl.config.BlobStorageS3SecretKey, + EndpointUrl: impl.config.BlobStorageS3Endpoint, + IsInSecure: impl.config.BlobStorageS3EndpointInsecure, + BucketName: ciConfigLogsBucket, + Region: ciConfigCiCacheRegion, + VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, + } + gcpBlobBaseConfig := &blob_storage.GcpBlobBaseConfig{ + BucketName: ciConfigLogsBucket, + CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, + } + + ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() + key := fmt.Sprintf(ciArtifactLocationFormat, ciWorkflow.Id, ciWorkflow.Id) + if len(ciWorkflow.CiArtifactLocation) != 0 && util2.IsValidUrlSubPath(ciWorkflow.CiArtifactLocation) { + key = ciWorkflow.CiArtifactLocation + } else if util2.IsValidUrlSubPath(key) { + impl.ciWorkflowRepository.MigrateCiArtifactLocation(ciWorkflow.Id, key) + } + baseLogLocationPathConfig := impl.config.BaseLogLocationPath + blobStorageService := blob_storage.NewBlobStorageServiceImpl(nil) + destinationKey := filepath.Clean(filepath.Join(baseLogLocationPathConfig, item)) + request := &blob_storage.BlobStorageRequest{ + StorageType: impl.config.CloudProvider, + SourceKey: key, + DestinationKey: destinationKey, + AzureBlobBaseConfig: azureBlobConfig, + AwsS3BaseConfig: awsS3BaseConfig, + GcpBlobBaseConfig: gcpBlobBaseConfig, + } + if useExternalBlobStorage { + envBean, err := impl.envService.FindById(ciWorkflow.EnvironmentId) + if err != nil { + impl.Logger.Errorw("error in getting envBean by envId", "err", err, "envId", ciWorkflow.EnvironmentId) + return nil, err + } + clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(envBean.ClusterId) + if err != nil { + impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig by clusterId", "err", err, "clusterId", envBean.ClusterId) + return nil, err + } + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) + if err != nil { + impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, err + } + request = pipeline2.UpdateRequestWithExtClusterCmAndSecret(request, cmConfig, secretConfig) + } + _, numBytes, err := blobStorageService.Get(request) + if err != nil { + impl.Logger.Errorw("error occurred while downloading file", "request", request, "error", err) + return nil, errors.New("failed to download resource") + } + + file, err := os.Open(destinationKey) + if err != nil { + impl.Logger.Errorw("unable to open file", "file", item, "err", err) + return nil, errors.New("unable to open file") + } + + impl.Logger.Infow("Downloaded ", "filename", file.Name(), "bytes", numBytes) + return file, nil +} diff --git a/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go index 8a11b06a6b..defe63b75d 100644 --- a/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go @@ -123,7 +123,7 @@ func (impl *TriggerServiceImpl) TriggerPostStage(request bean.TriggerRequest) (* return nil, err } - _, jobHelmPackagePath, err := impl.cdWorkflowService.SubmitWorkflow(cdStageWorkflowRequest) + _, jobHelmPackagePath, err := impl.workflowService.SubmitWorkflow(cdStageWorkflowRequest) if err != nil { impl.logger.Errorw("error in submitting workflow", "err", err, "workflowId", cdStageWorkflowRequest.WorkflowId, "pipeline", pipeline, "env", env) runner.Status = cdWorkflow.WorkflowFailed diff --git a/pkg/deployment/trigger/devtronApps/PrePostCommonService.go b/pkg/deployment/trigger/devtronApps/PrePostCommonService.go new file mode 100644 index 0000000000..d1cf588032 --- /dev/null +++ b/pkg/deployment/trigger/devtronApps/PrePostCommonService.go @@ -0,0 +1,337 @@ +package devtronApps + +import ( + "bufio" + "errors" + "fmt" + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + blob_storage "github.com/devtron-labs/common-lib/blob-storage" + "github.com/devtron-labs/common-lib/utils/k8s" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + cdWorkflow2 "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" + "github.com/devtron-labs/devtron/internal/util" + bean2 "github.com/devtron-labs/devtron/pkg/bean" + "github.com/devtron-labs/devtron/pkg/cluster/adapter" + bean3 "github.com/devtron-labs/devtron/pkg/cluster/bean" + "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/pipeline/constants" + "github.com/devtron-labs/devtron/pkg/pipeline/types" + util2 "github.com/devtron-labs/devtron/pkg/pipeline/util" + "k8s.io/client-go/rest" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "time" +) + +func (impl *TriggerServiceImpl) CancelStage(workflowRunnerId int, forceAbort bool, userId int32) (int, error) { + workflowRunner, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(workflowRunnerId) + if err != nil { + impl.logger.Errorw("err", "err", err) + return 0, err + } + pipeline, err := impl.pipelineRepository.FindById(workflowRunner.CdWorkflow.PipelineId) + if err != nil { + impl.logger.Errorw("error while fetching cd pipeline", "err", err) + return 0, err + } + + env, err := impl.envRepository.FindById(pipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("could not fetch stage env", "err", err) + return 0, err + } + + var clusterBean bean3.ClusterBean + if env != nil && env.Cluster != nil { + clusterBean = adapter.GetClusterBean(*env.Cluster) + } + clusterConfig := clusterBean.GetClusterConfig() + var isExtCluster bool + if workflowRunner.WorkflowType == types.PRE { + isExtCluster = pipeline.RunPreStageInEnv + } else if workflowRunner.WorkflowType == types.POST { + isExtCluster = pipeline.RunPostStageInEnv + } + var restConfig *rest.Config + if isExtCluster { + restConfig, err = impl.K8sUtil.GetRestConfigByCluster(clusterConfig) + if err != nil { + impl.logger.Errorw("error in getting rest config by cluster id", "err", err) + return 0, err + } + } + // Terminate workflow + cancelWfDtoRequest := &types.CancelWfRequestDto{ + ExecutorType: workflowRunner.ExecutorType, + WorkflowName: workflowRunner.Name, + Namespace: workflowRunner.Namespace, + RestConfig: restConfig, + IsExt: isExtCluster, + Environment: nil, + } + err = impl.workflowService.TerminateWorkflow(cancelWfDtoRequest) + if err != nil && forceAbort { + impl.logger.Errorw("error in terminating workflow, with force abort flag as true", "workflowName", workflowRunner.Name, "err", err) + cancelWfDtoRequest.WorkflowGenerateName = fmt.Sprintf("%d-%s", workflowRunnerId, workflowRunner.Name) + err1 := impl.workflowService.TerminateDanglingWorkflows(cancelWfDtoRequest) + if err1 != nil { + impl.logger.Errorw("error in terminating dangling workflows", "cancelWfDtoRequest", cancelWfDtoRequest, "err", err) + // ignoring error here in case of force abort, confirmed from product + } + } else if err != nil && strings.Contains(err.Error(), "cannot find workflow") { + return 0, &util.ApiError{Code: "200", HttpStatusCode: http.StatusBadRequest, UserMessage: err.Error()} + } else if err != nil { + impl.logger.Error("cannot terminate wf runner", "err", err) + return 0, err + } + if forceAbort { + err = impl.handleForceAbortCaseForCdStage(workflowRunner, forceAbort) + if err != nil { + impl.logger.Errorw("error in handleForceAbortCaseForCdStage", "forceAbortFlag", forceAbort, "workflowRunner", workflowRunner, "err", err) + return 0, err + } + return workflowRunner.Id, nil + } + if len(workflowRunner.ImagePathReservationIds) > 0 { + err := impl.customTagService.DeactivateImagePathReservationByImageIds(workflowRunner.ImagePathReservationIds) + if err != nil { + impl.logger.Errorw("error in deactivating image path reservation ids", "err", err) + return 0, err + } + } + workflowRunner.Status = cdWorkflow2.WorkflowCancel + workflowRunner.UpdatedOn = time.Now() + workflowRunner.UpdatedBy = userId + err = impl.cdWorkflowRunnerService.UpdateCdWorkflowRunnerWithStage(workflowRunner) + if err != nil { + impl.logger.Error("cannot update deleted workflow runner status, but wf deleted", "err", err) + return 0, err + } + return workflowRunner.Id, nil +} + +func (impl *TriggerServiceImpl) updateWorkflowRunnerForForceAbort(workflowRunner *pipelineConfig.CdWorkflowRunner) error { + workflowRunner.Status = cdWorkflow2.WorkflowCancel + workflowRunner.PodStatus = string(bean2.Failed) + workflowRunner.Message = constants.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE + err := impl.cdWorkflowRunnerService.UpdateCdWorkflowRunnerWithStage(workflowRunner) + if err != nil { + impl.logger.Errorw("error in updating workflow status in cd workflow runner in force abort case", "err", err) + return err + } + return nil +} + +func (impl *TriggerServiceImpl) handleForceAbortCaseForCdStage(workflowRunner *pipelineConfig.CdWorkflowRunner, forceAbort bool) error { + isWorkflowInNonTerminalStage := workflowRunner.Status == string(v1alpha1.NodePending) || workflowRunner.Status == string(v1alpha1.NodeRunning) + if !isWorkflowInNonTerminalStage { + if forceAbort { + return impl.updateWorkflowRunnerForForceAbort(workflowRunner) + } else { + return &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "cannot cancel stage, stage not in progress"} + } + } + //this arises when someone deletes the workflow in resource browser and wants to force abort a cd stage(pre/post) + if workflowRunner.Status == string(v1alpha1.NodeRunning) && forceAbort { + return impl.updateWorkflowRunnerForForceAbort(workflowRunner) + } + return nil +} + +func (impl *TriggerServiceImpl) DownloadCdWorkflowArtifacts(buildId int) (*os.File, error) { + wfr, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(buildId) + if err != nil { + impl.logger.Errorw("unable to fetch ciWorkflow", "err", err) + return nil, err + } + useExternalBlobStorage := pipeline.IsExternalBlobStorageEnabled(wfr.IsExternalRun(), impl.config.UseBlobStorageConfigInCdWorkflow) + if !wfr.BlobStorageEnabled { + return nil, errors.New("logs-not-stored-in-repository") + } + + cdConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() + cdConfigCdCacheRegion := impl.config.GetDefaultCdLogsBucketRegion() + + item := strconv.Itoa(wfr.Id) + awsS3BaseConfig := &blob_storage.AwsS3BaseConfig{ + AccessKey: impl.config.BlobStorageS3AccessKey, + Passkey: impl.config.BlobStorageS3SecretKey, + EndpointUrl: impl.config.BlobStorageS3Endpoint, + IsInSecure: impl.config.BlobStorageS3EndpointInsecure, + BucketName: cdConfigLogsBucket, + Region: cdConfigCdCacheRegion, + VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, + } + azureBlobBaseConfig := &blob_storage.AzureBlobBaseConfig{ + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, + AccountKey: impl.config.AzureAccountKey, + AccountName: impl.config.AzureAccountName, + BlobContainerName: impl.config.AzureBlobContainerCiLog, + } + gcpBlobBaseConfig := &blob_storage.GcpBlobBaseConfig{ + BucketName: cdConfigLogsBucket, + CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, + } + cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() + key := fmt.Sprintf(cdArtifactLocationFormat, wfr.CdWorkflow.Id, wfr.Id) + if len(wfr.CdArtifactLocation) != 0 && util2.IsValidUrlSubPath(wfr.CdArtifactLocation) { + key = wfr.CdArtifactLocation + } else if util2.IsValidUrlSubPath(key) { + impl.cdWorkflowRepository.MigrateCdArtifactLocation(wfr.Id, key) + } + baseLogLocationPathConfig := impl.config.BaseLogLocationPath + blobStorageService := blob_storage.NewBlobStorageServiceImpl(nil) + destinationKey := filepath.Clean(filepath.Join(baseLogLocationPathConfig, item)) + request := &blob_storage.BlobStorageRequest{ + StorageType: impl.config.CloudProvider, + SourceKey: key, + DestinationKey: destinationKey, + AzureBlobBaseConfig: azureBlobBaseConfig, + AwsS3BaseConfig: awsS3BaseConfig, + GcpBlobBaseConfig: gcpBlobBaseConfig, + } + if useExternalBlobStorage { + clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(wfr.CdWorkflow.Pipeline.Environment.ClusterId) + if err != nil { + impl.logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig", "err", err, "clusterId", wfr.CdWorkflow.Pipeline.Environment.ClusterId) + return nil, err + } + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, wfr.Namespace) + if err != nil { + impl.logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, err + } + request = pipeline.UpdateRequestWithExtClusterCmAndSecret(request, cmConfig, secretConfig) + } + _, numBytes, err := blobStorageService.Get(request) + if err != nil { + impl.logger.Errorw("error occurred while downloading file", "request", request, "error", err) + return nil, errors.New("failed to download resource") + } + + file, err := os.Open(destinationKey) + if err != nil { + impl.logger.Errorw("unable to open file", "file", item, "err", err) + return nil, errors.New("unable to open file") + } + + impl.logger.Infow("Downloaded ", "name", file.Name(), "bytes", numBytes) + return file, nil +} + +func (impl *TriggerServiceImpl) GetRunningWorkflowLogs(environmentId int, pipelineId int, wfrId int) (*bufio.Reader, func() error, error) { + cdWorkflow, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(wfrId) + if err != nil { + impl.logger.Errorw("error on fetch wf runner", "err", err) + return nil, nil, err + } + + env, err := impl.envRepository.FindById(environmentId) + if err != nil { + impl.logger.Errorw("could not fetch stage env", "err", err) + return nil, nil, err + } + + pipeline, err := impl.pipelineRepository.FindById(cdWorkflow.CdWorkflow.PipelineId) + if err != nil { + impl.logger.Errorw("error while fetching cd pipeline", "err", err) + return nil, nil, err + } + var clusterBean bean3.ClusterBean + if env != nil && env.Cluster != nil { + clusterBean = adapter.GetClusterBean(*env.Cluster) + } + clusterConfig := clusterBean.GetClusterConfig() + var isExtCluster bool + if cdWorkflow.WorkflowType == types.PRE { + isExtCluster = pipeline.RunPreStageInEnv + } else if cdWorkflow.WorkflowType == types.POST { + isExtCluster = pipeline.RunPostStageInEnv + } + return impl.getWorkflowLogs(pipelineId, cdWorkflow, clusterConfig, isExtCluster) +} + +func (impl *TriggerServiceImpl) getWorkflowLogs(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, runStageInEnv bool) (*bufio.Reader, func() error, error) { + cdLogRequest := types.BuildLogRequest{ + PodName: cdWorkflow.PodName, + Namespace: cdWorkflow.Namespace, + } + + logStream, cleanUp, err := impl.ciLogService.FetchRunningWorkflowLogs(cdLogRequest, clusterConfig, runStageInEnv) + if logStream == nil || err != nil { + if !cdWorkflow.BlobStorageEnabled { + return nil, nil, errors.New("logs-not-stored-in-repository") + } else if string(v1alpha1.NodeSucceeded) == cdWorkflow.Status || string(v1alpha1.NodeError) == cdWorkflow.Status || string(v1alpha1.NodeFailed) == cdWorkflow.Status || cdWorkflow.Status == cdWorkflow2.WorkflowCancel { + impl.logger.Debugw("pod is not live", "podName", cdWorkflow.PodName, "err", err) + return impl.getLogsFromRepository(pipelineId, cdWorkflow, clusterConfig, runStageInEnv) + } + if err != nil { + impl.logger.Errorw("err on fetch workflow logs", "err", err) + return nil, nil, err + } else if logStream == nil { + return nil, cleanUp, fmt.Errorf("no logs found for pod %s", cdWorkflow.PodName) + } + } + logReader := bufio.NewReader(logStream) + return logReader, cleanUp, err +} + +func (impl *TriggerServiceImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { + impl.logger.Debug("getting historic logs", "pipelineId", pipelineId) + + cdConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() // TODO -fixme + cdConfigCdCacheRegion := impl.config.GetDefaultCdLogsBucketRegion() + + cdLogRequest := types.BuildLogRequest{ + PipelineId: cdWorkflow.CdWorkflow.PipelineId, + WorkflowId: cdWorkflow.Id, + PodName: cdWorkflow.PodName, + LogsFilePath: cdWorkflow.LogLocation, // impl.ciCdConfig.CiDefaultBuildLogsKeyPrefix + "/" + cdWorkflow.Name + "/main.log", //TODO - fixme + CloudProvider: impl.config.CloudProvider, + AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, + AccountName: impl.config.AzureAccountName, + BlobContainerName: impl.config.AzureBlobContainerCiLog, + AccountKey: impl.config.AzureAccountKey, + }, + AwsS3BaseConfig: &blob_storage.AwsS3BaseConfig{ + AccessKey: impl.config.BlobStorageS3AccessKey, + Passkey: impl.config.BlobStorageS3SecretKey, + EndpointUrl: impl.config.BlobStorageS3Endpoint, + IsInSecure: impl.config.BlobStorageS3EndpointInsecure, + BucketName: cdConfigLogsBucket, + Region: cdConfigCdCacheRegion, + VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, + }, + GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ + BucketName: cdConfigLogsBucket, + CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, + }, + } + useExternalBlobStorage := pipeline.IsExternalBlobStorageEnabled(isExt, impl.config.UseBlobStorageConfigInCdWorkflow) + if useExternalBlobStorage { + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, cdWorkflow.Namespace) + if err != nil { + impl.logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, nil, err + } + rq := &cdLogRequest + rq.SetBuildLogRequest(cmConfig, secretConfig) + } + + impl.logger.Debugw("s3 log req ", "pipelineId", pipelineId, "runnerId", cdWorkflow.Id) + oldLogsStream, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, cdLogRequest) + if err != nil { + impl.logger.Errorw("err", err) + return nil, nil, err + } + logReader := bufio.NewReader(oldLogsStream) + return logReader, cleanUp, err +} diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go index ef11bedf2c..64c373a235 100644 --- a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go @@ -147,7 +147,7 @@ func (impl *TriggerServiceImpl) TriggerPreStage(request bean.TriggerRequest) (*b cdStageWorkflowRequest.Pipeline = pipeline cdStageWorkflowRequest.Env = env cdStageWorkflowRequest.Type = pipelineConfigBean.CD_WORKFLOW_PIPELINE_TYPE - _, jobHelmPackagePath, err := impl.cdWorkflowService.SubmitWorkflow(cdStageWorkflowRequest) + _, jobHelmPackagePath, err := impl.workflowService.SubmitWorkflow(cdStageWorkflowRequest) span.End() if err != nil { runner.Status = cdWorkflow.WorkflowFailed diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/TriggerService.go index 7a5d135045..942761fc47 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService.go @@ -17,6 +17,7 @@ package devtronApps import ( + "bufio" "context" "errors" "fmt" @@ -52,6 +53,7 @@ import ( "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/read" pipeline2 "github.com/devtron-labs/devtron/pkg/build/pipeline" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" + "github.com/devtron-labs/devtron/pkg/cluster" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" repository5 "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/deployment/common" @@ -68,6 +70,7 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/userDeploymentRequest/service" clientErrors "github.com/devtron-labs/devtron/pkg/errors" "github.com/devtron-labs/devtron/pkg/eventProcessor/out" + "github.com/devtron-labs/devtron/pkg/executor" "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" k8s2 "github.com/devtron-labs/devtron/pkg/k8s" "github.com/devtron-labs/devtron/pkg/pipeline" @@ -93,6 +96,7 @@ import ( status2 "google.golang.org/grpc/status" "helm.sh/helm/v3/pkg/chart" "net/http" + "os" "path" "regexp" "strconv" @@ -112,6 +116,10 @@ type TriggerService interface { TriggerAutomaticDeployment(request bean.TriggerRequest) error TriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) + + CancelStage(workflowRunnerId int, forceAbort bool, userId int32) (int, error) + DownloadCdWorkflowArtifacts(buildId int) (*os.File, error) + GetRunningWorkflowLogs(environmentId int, pipelineId int, workflowId int) (*bufio.Reader, func() error, error) } type TriggerServiceImpl struct { @@ -136,7 +144,6 @@ type TriggerServiceImpl struct { pluginInputVariableParser pipeline.PluginInputVariableParser prePostCdScriptHistoryService history.PrePostCdScriptHistoryService scopedVariableManager variables.ScopedVariableCMCSManager - cdWorkflowService pipeline.WorkflowService imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService userService user.UserService gitSensorClient gitSensorClient.Client @@ -174,6 +181,10 @@ type TriggerServiceImpl struct { attributeService attributes.AttributesService clusterRepository repository5.ClusterRepository cdWorkflowRunnerService cd.CdWorkflowRunnerService + clusterService cluster.ClusterService + ciLogService pipeline.CiLogService + workflowService executor.WorkflowService + blobConfigStorageService pipeline.BlobStorageConfigService } func NewTriggerServiceImpl(logger *zap.SugaredLogger, @@ -194,7 +205,6 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, pluginInputVariableParser pipeline.PluginInputVariableParser, prePostCdScriptHistoryService history.PrePostCdScriptHistoryService, scopedVariableManager variables.ScopedVariableCMCSManager, - cdWorkflowService pipeline.WorkflowService, imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService, userService user.UserService, gitSensorClient gitSensorClient.Client, @@ -233,6 +243,10 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, attributeService attributes.AttributesService, clusterRepository repository5.ClusterRepository, cdWorkflowRunnerService cd.CdWorkflowRunnerService, + clusterService cluster.ClusterService, + ciLogService pipeline.CiLogService, + workflowService executor.WorkflowService, + blobConfigStorageService pipeline.BlobStorageConfigService, ) (*TriggerServiceImpl, error) { impl := &TriggerServiceImpl{ logger: logger, @@ -253,7 +267,6 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, pluginInputVariableParser: pluginInputVariableParser, prePostCdScriptHistoryService: prePostCdScriptHistoryService, scopedVariableManager: scopedVariableManager, - cdWorkflowService: cdWorkflowService, imageDigestPolicyService: imageDigestPolicyService, userService: userService, gitSensorClient: gitSensorClient, @@ -297,7 +310,11 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, attributeService: attributeService, cdWorkflowRunnerService: cdWorkflowRunnerService, - clusterRepository: clusterRepository, + clusterRepository: clusterRepository, + clusterService: clusterService, + ciLogService: ciLogService, + workflowService: workflowService, + blobConfigStorageService: blobConfigStorageService, } config, err := types.GetCdConfig() if err != nil { diff --git a/pkg/pipeline/WorkflowService.go b/pkg/executor/WorkflowService.go similarity index 99% rename from pkg/pipeline/WorkflowService.go rename to pkg/executor/WorkflowService.go index 191b973c75..339c566113 100644 --- a/pkg/pipeline/WorkflowService.go +++ b/pkg/executor/WorkflowService.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package pipeline +package executor import ( "context" @@ -33,6 +33,7 @@ import ( "github.com/devtron-labs/devtron/pkg/config/read" v1 "github.com/devtron-labs/devtron/pkg/infraConfig/bean/v1" k8s2 "github.com/devtron-labs/devtron/pkg/k8s" + "github.com/devtron-labs/devtron/pkg/pipeline" bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders" @@ -66,7 +67,7 @@ type WorkflowServiceImpl struct { ciCdConfig *types.CiCdConfig configMapService read.ConfigReadService envRepository repository2.EnvironmentRepository - globalCMCSService GlobalCMCSService + globalCMCSService pipeline.GlobalCMCSService argoWorkflowExecutor executors.ArgoWorkflowExecutor systemWorkflowExecutor executors.SystemWorkflowExecutor k8sUtil *k8s.K8sServiceImpl @@ -80,7 +81,7 @@ func NewWorkflowServiceImpl(Logger *zap.SugaredLogger, envRepository repository2.EnvironmentRepository, ciCdConfig *types.CiCdConfig, configMapService read.ConfigReadService, - globalCMCSService GlobalCMCSService, + globalCMCSService pipeline.GlobalCMCSService, argoWorkflowExecutor executors.ArgoWorkflowExecutor, k8sUtil *k8s.K8sServiceImpl, systemWorkflowExecutor executors.SystemWorkflowExecutor, @@ -378,6 +379,7 @@ func (impl *WorkflowServiceImpl) getWorkflowExecutor(executorType cdWorkflow.Wor impl.Logger.Warnw("workflow executor not found", "type", executorType) return nil } + func (impl *WorkflowServiceImpl) GetWorkflow(executorType cdWorkflow.WorkflowExecutorType, name string, namespace string, restConfig *rest.Config) (*unstructured.UnstructuredList, error) { impl.Logger.Debug("getting wf", name) workflowExecutor := impl.getWorkflowExecutor(executorType) diff --git a/pkg/pipeline/WorkflowServiceIT_test.go b/pkg/executor/WorkflowServiceIT_test.go similarity index 99% rename from pkg/pipeline/WorkflowServiceIT_test.go rename to pkg/executor/WorkflowServiceIT_test.go index c025abe1fe..b1d569487f 100644 --- a/pkg/pipeline/WorkflowServiceIT_test.go +++ b/pkg/executor/WorkflowServiceIT_test.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package pipeline +package executor import ( "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" @@ -35,6 +35,7 @@ import ( "github.com/devtron-labs/devtron/pkg/commonService" k8s2 "github.com/devtron-labs/devtron/pkg/k8s" "github.com/devtron-labs/devtron/pkg/k8s/informer" + "github.com/devtron-labs/devtron/pkg/pipeline" bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/types" @@ -59,10 +60,10 @@ var cmManifest3 = "{\"kind\":\"ConfigMap\",\"apiVersion\":\"v1\",\"metadata\":{\ var cmManifest4 = "{\"kind\":\"ConfigMap\",\"apiVersion\":\"v1\",\"metadata\":{\"name\":\"cm4-5-ci\",\"creationTimestamp\":null,\"ownerReferences\":[{\"apiVersion\":\"argoproj.io/v1alpha1\",\"kind\":\"Workflow\",\"name\":\"{{workflow.name}}\",\"uid\":\"{{workflow.uid}}\",\"blockOwnerDeletion\":true}]},\"data\":{\"key4\":\"value4\"}}" func getWorkflowServiceImpl(t *testing.T) *WorkflowServiceImpl { - logger, dbConnection := getDbConnAndLoggerService(t) + logger, dbConnection := pipeline.getDbConnAndLoggerService(t) ciCdConfig, _ := types.GetCiCdConfig() newGlobalCMCSRepositoryImpl := repository.NewGlobalCMCSRepositoryImpl(logger, dbConnection) - globalCMCSServiceImpl := NewGlobalCMCSServiceImpl(logger, newGlobalCMCSRepositoryImpl) + globalCMCSServiceImpl := pipeline.NewGlobalCMCSServiceImpl(logger, newGlobalCMCSRepositoryImpl) newEnvConfigOverrideRepository := chartConfig.NewEnvConfigOverrideRepository(dbConnection) newConfigMapRepositoryImpl := chartConfig.NewConfigMapRepositoryImpl(logger, dbConnection) newChartRepository := chartRepoRepository.NewChartRepository(dbConnection) diff --git a/pkg/pipeline/WorkflowService_test.go b/pkg/executor/WorkflowService_test.go similarity index 99% rename from pkg/pipeline/WorkflowService_test.go rename to pkg/executor/WorkflowService_test.go index 4b4e937d68..1ebcb9639c 100644 --- a/pkg/pipeline/WorkflowService_test.go +++ b/pkg/executor/WorkflowService_test.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package pipeline +package executor import ( "fmt" diff --git a/pkg/executor/wire_executor.go b/pkg/executor/wire_executor.go new file mode 100644 index 0000000000..a99bf31f61 --- /dev/null +++ b/pkg/executor/wire_executor.go @@ -0,0 +1,8 @@ +package executor + +import "github.com/google/wire" + +var ExecutorWireSet = wire.NewSet( + NewWorkflowServiceImpl, + wire.Bind(new(WorkflowService), new(*WorkflowServiceImpl)), +) diff --git a/pkg/pipeline/BlobStorageConfigService.go b/pkg/pipeline/BlobStorageConfigService.go index fd6141c111..53ea2efde6 100644 --- a/pkg/pipeline/BlobStorageConfigService.go +++ b/pkg/pipeline/BlobStorageConfigService.go @@ -91,7 +91,7 @@ func (impl *BlobStorageConfigServiceImpl) FetchCmAndSecretBlobConfigFromExternal return cmConfig, secretConfig, nil } -func updateRequestWithExtClusterCmAndSecret(request *blob_storage.BlobStorageRequest, cmConfig *bean2.CmBlobStorageConfig, secretConfig *bean2.SecretBlobStorageConfig) *blob_storage.BlobStorageRequest { +func UpdateRequestWithExtClusterCmAndSecret(request *blob_storage.BlobStorageRequest, cmConfig *bean2.CmBlobStorageConfig, secretConfig *bean2.SecretBlobStorageConfig) *blob_storage.BlobStorageRequest { request.StorageType = cmConfig.CloudProvider request.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey @@ -113,3 +113,8 @@ func updateRequestWithExtClusterCmAndSecret(request *blob_storage.BlobStorageReq return request } + +func IsExternalBlobStorageEnabled(isExternalRun bool, useBlobStorageConfigInCdWorkflow bool) bool { + // TODO impl.config.UseBlobStorageConfigInCdWorkflow fetches the live status, we need to check from db as well, we should put useExternalBlobStorage in db + return isExternalRun && !useBlobStorageConfigInCdWorkflow +} diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 230d35c522..6816db9067 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -17,41 +17,30 @@ package pipeline import ( - "bufio" "errors" "fmt" "github.com/devtron-labs/common-lib/utils" bean4 "github.com/devtron-labs/common-lib/utils/bean" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/adapter/cdWorkflow" cdWorkflow2 "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" - bean2 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/build/artifacts/imageTagging" buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" - "github.com/devtron-labs/devtron/pkg/cluster/adapter" - bean3 "github.com/devtron-labs/devtron/pkg/cluster/bean" repository3 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" - "github.com/devtron-labs/devtron/pkg/pipeline/constants" - util2 "github.com/devtron-labs/devtron/pkg/pipeline/util" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" bean5 "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus/bean" "github.com/devtron-labs/devtron/pkg/workflow/cd" - "net/http" - "os" - "path/filepath" "strconv" "strings" "time" "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" - blob_storage "github.com/devtron-labs/common-lib/blob-storage" "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/auth/user" - "github.com/devtron-labs/devtron/pkg/cluster" pipelineBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/types" resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup" @@ -60,7 +49,6 @@ import ( "github.com/go-pg/pg" "go.opentelemetry.io/otel" "go.uber.org/zap" - "k8s.io/client-go/rest" ) const ( @@ -72,11 +60,8 @@ const ( type CdHandler interface { UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, string, bool, error) GetCdBuildHistory(appId int, environmentId int, pipelineId int, offset int, size int) ([]pipelineBean.CdWorkflowWithArtifact, error) - GetRunningWorkflowLogs(environmentId int, pipelineId int, workflowId int) (*bufio.Reader, func() error, error) FetchCdWorkflowDetails(appId int, environmentId int, pipelineId int, buildId int) (types.WorkflowResponse, error) - DownloadCdWorkflowArtifacts(buildId int) (*os.File, error) FetchCdPrePostStageStatus(pipelineId int) ([]pipelineBean.CdWorkflowWithArtifact, error) - CancelStage(workflowRunnerId int, forceAbort bool, userId int32) (int, error) FetchAppWorkflowStatusForTriggerView(appId int) ([]*pipelineConfig.CdWorkflowStatus, error) FetchAppWorkflowStatusForTriggerViewForEnvironment(request resourceGroup2.ResourceGroupingRequest, token string) ([]*pipelineConfig.CdWorkflowStatus, error) FetchAppDeploymentStatusForEnvironments(request resourceGroup2.ResourceGroupingRequest, token string) ([]*pipelineConfig.AppDeploymentStatus, error) @@ -86,7 +71,6 @@ type CdHandler interface { type CdHandlerImpl struct { Logger *zap.SugaredLogger userService user.UserService - ciLogService CiLogService ciArtifactRepository repository.CiArtifactRepository ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository cdWorkflowRepository pipelineConfig.CdWorkflowRepository @@ -97,10 +81,7 @@ type CdHandlerImpl struct { resourceGroupService resourceGroup2.ResourceGroupService imageTaggingService imageTagging.ImageTaggingService k8sUtil *k8s.K8sServiceImpl - workflowService WorkflowService config *types.CdConfig - clusterService cluster.ClusterService - blobConfigStorageService BlobStorageConfigService customTagService CustomTagService deploymentConfigService common2.DeploymentConfigService workflowStageStatusService workflowStatus.WorkFlowStageStatusService @@ -108,15 +89,14 @@ type CdHandlerImpl struct { } func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, - cdWorkflowRepository pipelineConfig.CdWorkflowRepository, ciLogService CiLogService, + cdWorkflowRepository pipelineConfig.CdWorkflowRepository, ciArtifactRepository repository.CiArtifactRepository, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, pipelineRepository pipelineConfig.PipelineRepository, envRepository repository3.EnvironmentRepository, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, enforcerUtil rbac.EnforcerUtil, resourceGroupService resourceGroup2.ResourceGroupService, imageTaggingService imageTagging.ImageTaggingService, k8sUtil *k8s.K8sServiceImpl, - workflowService WorkflowService, clusterService cluster.ClusterService, - blobConfigStorageService BlobStorageConfigService, customTagService CustomTagService, + customTagService CustomTagService, deploymentConfigService common2.DeploymentConfigService, workflowStageStatusService workflowStatus.WorkFlowStageStatusService, cdWorkflowRunnerService cd.CdWorkflowRunnerService, @@ -124,7 +104,6 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, cdh := &CdHandlerImpl{ Logger: Logger, userService: userService, - ciLogService: ciLogService, cdWorkflowRepository: cdWorkflowRepository, ciArtifactRepository: ciArtifactRepository, ciPipelineMaterialRepository: ciPipelineMaterialRepository, @@ -135,9 +114,6 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, resourceGroupService: resourceGroupService, imageTaggingService: imageTaggingService, k8sUtil: k8sUtil, - workflowService: workflowService, - clusterService: clusterService, - blobConfigStorageService: blobConfigStorageService, customTagService: customTagService, deploymentConfigService: deploymentConfigService, workflowStageStatusService: workflowStageStatusService, @@ -151,121 +127,6 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, return cdh } -func (impl *CdHandlerImpl) CancelStage(workflowRunnerId int, forceAbort bool, userId int32) (int, error) { - workflowRunner, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(workflowRunnerId) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return 0, err - } - pipeline, err := impl.pipelineRepository.FindById(workflowRunner.CdWorkflow.PipelineId) - if err != nil { - impl.Logger.Errorw("error while fetching cd pipeline", "err", err) - return 0, err - } - - env, err := impl.envRepository.FindById(pipeline.EnvironmentId) - if err != nil { - impl.Logger.Errorw("could not fetch stage env", "err", err) - return 0, err - } - - var clusterBean bean3.ClusterBean - if env != nil && env.Cluster != nil { - clusterBean = adapter.GetClusterBean(*env.Cluster) - } - clusterConfig := clusterBean.GetClusterConfig() - var isExtCluster bool - if workflowRunner.WorkflowType == types.PRE { - isExtCluster = pipeline.RunPreStageInEnv - } else if workflowRunner.WorkflowType == types.POST { - isExtCluster = pipeline.RunPostStageInEnv - } - var restConfig *rest.Config - if isExtCluster { - restConfig, err = impl.k8sUtil.GetRestConfigByCluster(clusterConfig) - if err != nil { - impl.Logger.Errorw("error in getting rest config by cluster id", "err", err) - return 0, err - } - } - // Terminate workflow - cancelWfDtoRequest := &types.CancelWfRequestDto{ - ExecutorType: workflowRunner.ExecutorType, - WorkflowName: workflowRunner.Name, - Namespace: workflowRunner.Namespace, - RestConfig: restConfig, - IsExt: isExtCluster, - Environment: nil, - } - err = impl.workflowService.TerminateWorkflow(cancelWfDtoRequest) - if err != nil && forceAbort { - impl.Logger.Errorw("error in terminating workflow, with force abort flag as true", "workflowName", workflowRunner.Name, "err", err) - cancelWfDtoRequest.WorkflowGenerateName = fmt.Sprintf("%d-%s", workflowRunnerId, workflowRunner.Name) - err1 := impl.workflowService.TerminateDanglingWorkflows(cancelWfDtoRequest) - if err1 != nil { - impl.Logger.Errorw("error in terminating dangling workflows", "cancelWfDtoRequest", cancelWfDtoRequest, "err", err) - // ignoring error here in case of force abort, confirmed from product - } - } else if err != nil && strings.Contains(err.Error(), "cannot find workflow") { - return 0, &util.ApiError{Code: "200", HttpStatusCode: http.StatusBadRequest, UserMessage: err.Error()} - } else if err != nil { - impl.Logger.Error("cannot terminate wf runner", "err", err) - return 0, err - } - if forceAbort { - err = impl.handleForceAbortCaseForCdStage(workflowRunner, forceAbort) - if err != nil { - impl.Logger.Errorw("error in handleForceAbortCaseForCdStage", "forceAbortFlag", forceAbort, "workflowRunner", workflowRunner, "err", err) - return 0, err - } - return workflowRunner.Id, nil - } - if len(workflowRunner.ImagePathReservationIds) > 0 { - err := impl.customTagService.DeactivateImagePathReservationByImageIds(workflowRunner.ImagePathReservationIds) - if err != nil { - impl.Logger.Errorw("error in deactivating image path reservation ids", "err", err) - return 0, err - } - } - workflowRunner.Status = cdWorkflow2.WorkflowCancel - workflowRunner.UpdatedOn = time.Now() - workflowRunner.UpdatedBy = userId - err = impl.cdWorkflowRunnerService.UpdateCdWorkflowRunnerWithStage(workflowRunner) - if err != nil { - impl.Logger.Error("cannot update deleted workflow runner status, but wf deleted", "err", err) - return 0, err - } - return workflowRunner.Id, nil -} - -func (impl *CdHandlerImpl) updateWorkflowRunnerForForceAbort(workflowRunner *pipelineConfig.CdWorkflowRunner) error { - workflowRunner.Status = cdWorkflow2.WorkflowCancel - workflowRunner.PodStatus = string(bean2.Failed) - workflowRunner.Message = constants.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE - err := impl.cdWorkflowRunnerService.UpdateCdWorkflowRunnerWithStage(workflowRunner) - if err != nil { - impl.Logger.Errorw("error in updating workflow status in cd workflow runner in force abort case", "err", err) - return err - } - return nil -} - -func (impl *CdHandlerImpl) handleForceAbortCaseForCdStage(workflowRunner *pipelineConfig.CdWorkflowRunner, forceAbort bool) error { - isWorkflowInNonTerminalStage := workflowRunner.Status == string(v1alpha1.NodePending) || workflowRunner.Status == string(v1alpha1.NodeRunning) - if !isWorkflowInNonTerminalStage { - if forceAbort { - return impl.updateWorkflowRunnerForForceAbort(workflowRunner) - } else { - return &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "cannot cancel stage, stage not in progress"} - } - } - //this arises when someone deletes the workflow in resource browser and wants to force abort a cd stage(pre/post) - if workflowRunner.Status == string(v1alpha1.NodeRunning) && forceAbort { - return impl.updateWorkflowRunnerForForceAbort(workflowRunner) - } - return nil -} - func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, string, bool, error) { wfStatusRs := impl.extractWorkfowStatus(workflowStatus) workflowName, status, podStatus, message, podName := wfStatusRs.WorkflowName, wfStatusRs.Status, wfStatusRs.PodStatus, wfStatusRs.Message, wfStatusRs.PodName @@ -516,122 +377,6 @@ func (impl *CdHandlerImpl) GetCdBuildHistory(appId int, environmentId int, pipel return cdWorkflowArtifact, nil } -func (impl *CdHandlerImpl) GetRunningWorkflowLogs(environmentId int, pipelineId int, wfrId int) (*bufio.Reader, func() error, error) { - cdWorkflow, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(wfrId) - if err != nil { - impl.Logger.Errorw("error on fetch wf runner", "err", err) - return nil, nil, err - } - - env, err := impl.envRepository.FindById(environmentId) - if err != nil { - impl.Logger.Errorw("could not fetch stage env", "err", err) - return nil, nil, err - } - - pipeline, err := impl.pipelineRepository.FindById(cdWorkflow.CdWorkflow.PipelineId) - if err != nil { - impl.Logger.Errorw("error while fetching cd pipeline", "err", err) - return nil, nil, err - } - var clusterBean bean3.ClusterBean - if env != nil && env.Cluster != nil { - clusterBean = adapter.GetClusterBean(*env.Cluster) - } - clusterConfig := clusterBean.GetClusterConfig() - var isExtCluster bool - if cdWorkflow.WorkflowType == types.PRE { - isExtCluster = pipeline.RunPreStageInEnv - } else if cdWorkflow.WorkflowType == types.POST { - isExtCluster = pipeline.RunPostStageInEnv - } - return impl.getWorkflowLogs(pipelineId, cdWorkflow, clusterConfig, isExtCluster) -} - -func (impl *CdHandlerImpl) getWorkflowLogs(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, runStageInEnv bool) (*bufio.Reader, func() error, error) { - cdLogRequest := types.BuildLogRequest{ - PodName: cdWorkflow.PodName, - Namespace: cdWorkflow.Namespace, - } - - logStream, cleanUp, err := impl.ciLogService.FetchRunningWorkflowLogs(cdLogRequest, clusterConfig, runStageInEnv) - if logStream == nil || err != nil { - if !cdWorkflow.BlobStorageEnabled { - return nil, nil, errors.New("logs-not-stored-in-repository") - } else if string(v1alpha1.NodeSucceeded) == cdWorkflow.Status || string(v1alpha1.NodeError) == cdWorkflow.Status || string(v1alpha1.NodeFailed) == cdWorkflow.Status || cdWorkflow.Status == cdWorkflow2.WorkflowCancel { - impl.Logger.Debugw("pod is not live", "podName", cdWorkflow.PodName, "err", err) - return impl.getLogsFromRepository(pipelineId, cdWorkflow, clusterConfig, runStageInEnv) - } - if err != nil { - impl.Logger.Errorw("err on fetch workflow logs", "err", err) - return nil, nil, err - } else if logStream == nil { - return nil, cleanUp, fmt.Errorf("no logs found for pod %s", cdWorkflow.PodName) - } - } - logReader := bufio.NewReader(logStream) - return logReader, cleanUp, err -} - -func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { - impl.Logger.Debug("getting historic logs", "pipelineId", pipelineId) - - cdConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() // TODO -fixme - cdConfigCdCacheRegion := impl.config.GetDefaultCdLogsBucketRegion() - - cdLogRequest := types.BuildLogRequest{ - PipelineId: cdWorkflow.CdWorkflow.PipelineId, - WorkflowId: cdWorkflow.Id, - PodName: cdWorkflow.PodName, - LogsFilePath: cdWorkflow.LogLocation, // impl.ciCdConfig.CiDefaultBuildLogsKeyPrefix + "/" + cdWorkflow.Name + "/main.log", //TODO - fixme - CloudProvider: impl.config.CloudProvider, - AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ - Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, - AccountName: impl.config.AzureAccountName, - BlobContainerName: impl.config.AzureBlobContainerCiLog, - AccountKey: impl.config.AzureAccountKey, - }, - AwsS3BaseConfig: &blob_storage.AwsS3BaseConfig{ - AccessKey: impl.config.BlobStorageS3AccessKey, - Passkey: impl.config.BlobStorageS3SecretKey, - EndpointUrl: impl.config.BlobStorageS3Endpoint, - IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - BucketName: cdConfigLogsBucket, - Region: cdConfigCdCacheRegion, - VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, - }, - GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ - BucketName: cdConfigLogsBucket, - CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - }, - } - useExternalBlobStorage := isExternalBlobStorageEnabled(isExt, impl.config.UseBlobStorageConfigInCdWorkflow) - if useExternalBlobStorage { - // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - // from them else return. - cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, cdWorkflow.Namespace) - if err != nil { - impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) - return nil, nil, err - } - rq := &cdLogRequest - rq.SetBuildLogRequest(cmConfig, secretConfig) - } - - impl.Logger.Debugw("s3 log req ", "pipelineId", pipelineId, "runnerId", cdWorkflow.Id) - oldLogsStream, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, cdLogRequest) - if err != nil { - impl.Logger.Errorw("err", err) - return nil, nil, err - } - logReader := bufio.NewReader(oldLogsStream) - return logReader, cleanUp, err -} -func isExternalBlobStorageEnabled(isExternalRun bool, useBlobStorageConfigInCdWorkflow bool) bool { - // TODO impl.config.UseBlobStorageConfigInCdWorkflow fetches the live status, we need to check from db as well, we should put useExternalBlobStorage in db - return isExternalRun && !useBlobStorageConfigInCdWorkflow -} - func (impl *CdHandlerImpl) FetchCdWorkflowDetails(appId int, environmentId int, pipelineId int, buildId int) (types.WorkflowResponse, error) { workflowR, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(buildId) if err != nil && err != pg.ErrNoRows { @@ -736,89 +481,6 @@ func (impl *CdHandlerImpl) FetchCdWorkflowDetails(appId int, environmentId int, } -func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(buildId int) (*os.File, error) { - wfr, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(buildId) - if err != nil { - impl.Logger.Errorw("unable to fetch ciWorkflow", "err", err) - return nil, err - } - useExternalBlobStorage := isExternalBlobStorageEnabled(wfr.IsExternalRun(), impl.config.UseBlobStorageConfigInCdWorkflow) - if !wfr.BlobStorageEnabled { - return nil, errors.New("logs-not-stored-in-repository") - } - - cdConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() - cdConfigCdCacheRegion := impl.config.GetDefaultCdLogsBucketRegion() - - item := strconv.Itoa(wfr.Id) - awsS3BaseConfig := &blob_storage.AwsS3BaseConfig{ - AccessKey: impl.config.BlobStorageS3AccessKey, - Passkey: impl.config.BlobStorageS3SecretKey, - EndpointUrl: impl.config.BlobStorageS3Endpoint, - IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - BucketName: cdConfigLogsBucket, - Region: cdConfigCdCacheRegion, - VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, - } - azureBlobBaseConfig := &blob_storage.AzureBlobBaseConfig{ - Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, - AccountKey: impl.config.AzureAccountKey, - AccountName: impl.config.AzureAccountName, - BlobContainerName: impl.config.AzureBlobContainerCiLog, - } - gcpBlobBaseConfig := &blob_storage.GcpBlobBaseConfig{ - BucketName: cdConfigLogsBucket, - CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - } - cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() - key := fmt.Sprintf(cdArtifactLocationFormat, wfr.CdWorkflow.Id, wfr.Id) - if len(wfr.CdArtifactLocation) != 0 && util2.IsValidUrlSubPath(wfr.CdArtifactLocation) { - key = wfr.CdArtifactLocation - } else if util2.IsValidUrlSubPath(key) { - impl.cdWorkflowRepository.MigrateCdArtifactLocation(wfr.Id, key) - } - baseLogLocationPathConfig := impl.config.BaseLogLocationPath - blobStorageService := blob_storage.NewBlobStorageServiceImpl(nil) - destinationKey := filepath.Clean(filepath.Join(baseLogLocationPathConfig, item)) - request := &blob_storage.BlobStorageRequest{ - StorageType: impl.config.CloudProvider, - SourceKey: key, - DestinationKey: destinationKey, - AzureBlobBaseConfig: azureBlobBaseConfig, - AwsS3BaseConfig: awsS3BaseConfig, - GcpBlobBaseConfig: gcpBlobBaseConfig, - } - if useExternalBlobStorage { - clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(wfr.CdWorkflow.Pipeline.Environment.ClusterId) - if err != nil { - impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig", "err", err, "clusterId", wfr.CdWorkflow.Pipeline.Environment.ClusterId) - return nil, err - } - // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - // from them else return. - cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, wfr.Namespace) - if err != nil { - impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) - return nil, err - } - request = updateRequestWithExtClusterCmAndSecret(request, cmConfig, secretConfig) - } - _, numBytes, err := blobStorageService.Get(request) - if err != nil { - impl.Logger.Errorw("error occurred while downloading file", "request", request, "error", err) - return nil, errors.New("failed to download resource") - } - - file, err := os.Open(destinationKey) - if err != nil { - impl.Logger.Errorw("unable to open file", "file", item, "err", err) - return nil, errors.New("unable to open file") - } - - impl.Logger.Infow("Downloaded ", "name", file.Name(), "bytes", numBytes) - return file, nil -} - func (impl *CdHandlerImpl) converterWFR(wfr pipelineConfig.CdWorkflowRunner) pipelineBean.CdWorkflowWithArtifact { workflow := pipelineBean.CdWorkflowWithArtifact{} if wfr.Id > 0 { diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 20d18c93eb..70cb749398 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -17,7 +17,6 @@ package pipeline import ( - "bufio" "context" "errors" "fmt" @@ -26,43 +25,29 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/pkg/build/artifacts/imageTagging" buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" - "github.com/devtron-labs/devtron/pkg/cluster/adapter" - clusterBean "github.com/devtron-labs/devtron/pkg/cluster/bean" - "github.com/devtron-labs/devtron/pkg/cluster/environment" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/pipeline/constants" - util3 "github.com/devtron-labs/devtron/pkg/pipeline/util" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" - "io/ioutil" - "net/http" - "os" - "path/filepath" "regexp" "strconv" "strings" "time" - blob_storage "github.com/devtron-labs/common-lib/blob-storage" - "github.com/devtron-labs/common-lib/utils/k8s" + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" bean2 "github.com/devtron-labs/devtron/api/bean" + client "github.com/devtron-labs/devtron/client/events" "github.com/devtron-labs/devtron/client/gitSensor" + "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/auth/user" - "github.com/devtron-labs/devtron/pkg/cluster" k8sPkg "github.com/devtron-labs/devtron/pkg/k8s" pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/util/rbac" - "k8s.io/client-go/rest" - - "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" - client "github.com/devtron-labs/devtron/client/events" - "github.com/devtron-labs/devtron/internal/sql/repository" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/internal/util" - "github.com/devtron-labs/devtron/pkg/bean" "github.com/go-pg/pg" "go.uber.org/zap" ) @@ -75,13 +60,8 @@ type CiHandler interface { FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId int, gitMaterialId int, showAll bool) ([]buildBean.CiPipelineMaterialResponse, error) FetchWorkflowDetails(appId int, pipelineId int, buildId int) (types.WorkflowResponse, error) FetchArtifactsForCiJob(buildId int) (*types.ArtifactsForCiJob, error) - CancelBuild(workflowId int, forceAbort bool) (int, error) - - GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, func() error, error) - GetHistoricBuildLogs(workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) GetBuildHistory(pipelineId int, appId int, offset int, size int) ([]types.WorkflowResponse, error) - DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, error) FetchCiStatusForTriggerView(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) @@ -99,15 +79,12 @@ type CiHandlerImpl struct { ciService CiService gitSensorClient gitSensor.Client ciWorkflowRepository pipelineConfig.CiWorkflowRepository - workflowService WorkflowService - ciLogService CiLogService ciArtifactRepository repository.CiArtifactRepository userService user.UserService eventClient client.EventClient eventFactory client.EventFactory ciPipelineRepository pipelineConfig.CiPipelineRepository appListingRepository repository.AppListingRepository - K8sUtil *k8s.K8sServiceImpl cdPipelineRepository pipelineConfig.PipelineRepository enforcerUtil rbac.EnforcerUtil resourceGroupService resourceGroup.ResourceGroupService @@ -117,18 +94,14 @@ type CiHandlerImpl struct { appWorkflowRepository appWorkflow.AppWorkflowRepository config *types.CiConfig k8sCommonService k8sPkg.K8sCommonService - clusterService cluster.ClusterService - blobConfigStorageService BlobStorageConfigService - envService environment.EnvironmentService workFlowStageStatusService workflowStatus.WorkFlowStageStatusService } -func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, gitSensorClient gitSensor.Client, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, workflowService WorkflowService, - ciLogService CiLogService, ciArtifactRepository repository.CiArtifactRepository, userService user.UserService, eventClient client.EventClient, eventFactory client.EventFactory, ciPipelineRepository pipelineConfig.CiPipelineRepository, +func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, gitSensorClient gitSensor.Client, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, + ciArtifactRepository repository.CiArtifactRepository, userService user.UserService, eventClient client.EventClient, eventFactory client.EventFactory, ciPipelineRepository pipelineConfig.CiPipelineRepository, appListingRepository repository.AppListingRepository, cdPipelineRepository pipelineConfig.PipelineRepository, enforcerUtil rbac.EnforcerUtil, resourceGroupService resourceGroup.ResourceGroupService, envRepository repository2.EnvironmentRepository, - imageTaggingService imageTagging.ImageTaggingService, k8sCommonService k8sPkg.K8sCommonService, clusterService cluster.ClusterService, blobConfigStorageService BlobStorageConfigService, appWorkflowRepository appWorkflow.AppWorkflowRepository, customTagService CustomTagService, - envService environment.EnvironmentService, workFlowStageStatusService workflowStatus.WorkFlowStageStatusService, - K8sUtil *k8s.K8sServiceImpl, + imageTaggingService imageTagging.ImageTaggingService, k8sCommonService k8sPkg.K8sCommonService, appWorkflowRepository appWorkflow.AppWorkflowRepository, customTagService CustomTagService, + workFlowStageStatusService workflowStatus.WorkFlowStageStatusService, ) *CiHandlerImpl { cih := &CiHandlerImpl{ Logger: Logger, @@ -136,15 +109,12 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline ciPipelineMaterialRepository: ciPipelineMaterialRepository, gitSensorClient: gitSensorClient, ciWorkflowRepository: ciWorkflowRepository, - workflowService: workflowService, - ciLogService: ciLogService, ciArtifactRepository: ciArtifactRepository, userService: userService, eventClient: eventClient, eventFactory: eventFactory, ciPipelineRepository: ciPipelineRepository, appListingRepository: appListingRepository, - K8sUtil: K8sUtil, cdPipelineRepository: cdPipelineRepository, enforcerUtil: enforcerUtil, resourceGroupService: resourceGroupService, @@ -153,9 +123,6 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline customTagService: customTagService, appWorkflowRepository: appWorkflowRepository, k8sCommonService: k8sCommonService, - clusterService: clusterService, - blobConfigStorageService: blobConfigStorageService, - envService: envService, workFlowStageStatusService: workFlowStageStatusService, } config, err := types.GetCiConfig() @@ -443,129 +410,6 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int return ciWorkLowResponses, nil } -func (impl *CiHandlerImpl) CancelBuild(workflowId int, forceAbort bool) (int, error) { - workflow, err := impl.ciWorkflowRepository.FindById(workflowId) - if err != nil { - impl.Logger.Errorw("error in finding ci-workflow by workflow id", "ciWorkflowId", workflowId, "err", err) - return 0, err - } - isExt := workflow.Namespace != constants.DefaultCiWorkflowNamespace - var env *repository2.Environment - var restConfig *rest.Config - if isExt { - restConfig, err = impl.getRestConfig(workflow) - if err != nil { - return 0, err - } - } - // Terminate workflow - cancelWfDtoRequest := &types.CancelWfRequestDto{ - ExecutorType: workflow.ExecutorType, - WorkflowName: workflow.Name, - Namespace: workflow.Namespace, - RestConfig: restConfig, - IsExt: isExt, - Environment: env, - } - // Terminate workflow - err = impl.workflowService.TerminateWorkflow(cancelWfDtoRequest) - if err != nil && forceAbort { - impl.Logger.Errorw("error in terminating workflow, with force abort flag flag as true", "workflowName", workflow.Name, "err", err) - - cancelWfDtoRequest.WorkflowGenerateName = fmt.Sprintf("%d-%s", workflowId, workflow.Name) - err1 := impl.workflowService.TerminateDanglingWorkflows(cancelWfDtoRequest) - if err1 != nil { - impl.Logger.Errorw("error in terminating dangling workflows", "cancelWfDtoRequest", cancelWfDtoRequest, "err", err) - // ignoring error here in case of force abort, confirmed from product - } - } else if err != nil && strings.Contains(err.Error(), "cannot find workflow") { - return 0, &util.ApiError{Code: "200", HttpStatusCode: http.StatusBadRequest, UserMessage: err.Error()} - } else if err != nil { - impl.Logger.Errorw("cannot terminate wf", "err", err) - return 0, err - } - if forceAbort { - err = impl.handleForceAbortCaseForCi(workflow, forceAbort) - if err != nil { - impl.Logger.Errorw("error in handleForceAbortCaseForCi", "forceAbortFlag", forceAbort, "workflow", workflow, "err", err) - return 0, err - } - return workflow.Id, nil - } - - workflow.Status = cdWorkflow.WorkflowCancel - if workflow.ExecutorType == cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM { - workflow.PodStatus = "Failed" - workflow.Message = constants.TERMINATE_MESSAGE - } - err = impl.ciService.UpdateCiWorkflowWithStage(workflow) - if err != nil { - impl.Logger.Errorw("cannot update deleted workflow status, but wf deleted", "err", err) - return 0, err - } - imagePathReservationId := workflow.ImagePathReservationId - err = impl.customTagService.DeactivateImagePathReservation(imagePathReservationId) - if err != nil { - impl.Logger.Errorw("error in marking image tag unreserved", "err", err) - return 0, err - } - imagePathReservationIds := workflow.ImagePathReservationIds - if len(imagePathReservationIds) > 0 { - err = impl.customTagService.DeactivateImagePathReservationByImageIds(imagePathReservationIds) - if err != nil { - impl.Logger.Errorw("error in marking image tag unreserved", "err", err) - return 0, err - } - } - return workflow.Id, nil -} - -func (impl *CiHandlerImpl) handleForceAbortCaseForCi(workflow *pipelineConfig.CiWorkflow, forceAbort bool) error { - isWorkflowInNonTerminalStage := workflow.Status == string(v1alpha1.NodePending) || workflow.Status == string(v1alpha1.NodeRunning) - if !isWorkflowInNonTerminalStage { - if forceAbort { - return impl.updateWorkflowForForceAbort(workflow) - } else { - return &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "cannot cancel build, build not in progress"} - } - } - //this arises when someone deletes the workflow in resource browser and wants to force abort a ci - if workflow.Status == string(v1alpha1.NodeRunning) && forceAbort { - return impl.updateWorkflowForForceAbort(workflow) - } - return nil -} - -func (impl *CiHandlerImpl) updateWorkflowForForceAbort(workflow *pipelineConfig.CiWorkflow) error { - workflow.Status = cdWorkflow.WorkflowCancel - workflow.PodStatus = string(bean.Failed) - workflow.Message = constants.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE - err := impl.ciService.UpdateCiWorkflowWithStage(workflow) - if err != nil { - impl.Logger.Errorw("error in updating workflow status", "err", err) - return err - } - return nil -} - -func (impl *CiHandlerImpl) getRestConfig(workflow *pipelineConfig.CiWorkflow) (*rest.Config, error) { - env, err := impl.envRepository.FindById(workflow.EnvironmentId) - if err != nil { - impl.Logger.Errorw("could not fetch stage env", "err", err) - return nil, err - } - - clusterBean := adapter.GetClusterBean(*env.Cluster) - - clusterConfig := clusterBean.GetClusterConfig() - restConfig, err := impl.K8sUtil.GetRestConfigByCluster(clusterConfig) - if err != nil { - impl.Logger.Errorw("error in getting rest config by cluster id", "err", err) - return nil, err - } - return restConfig, nil -} - func (impl *CiHandlerImpl) FetchWorkflowDetails(appId int, pipelineId int, buildId int) (types.WorkflowResponse, error) { workflow, err := impl.ciWorkflowRepository.FindById(buildId) if err != nil { @@ -670,276 +514,6 @@ func (impl *CiHandlerImpl) FetchArtifactsForCiJob(buildId int) (*types.Artifacts } return artifactsResponse, nil } -func (impl *CiHandlerImpl) GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, func() error, error) { - ciWorkflow, err := impl.ciWorkflowRepository.FindById(workflowId) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return nil, nil, err - } - return impl.getWorkflowLogs(ciWorkflow) -} - -func (impl *CiHandlerImpl) getWorkflowLogs(ciWorkflow *pipelineConfig.CiWorkflow) (*bufio.Reader, func() error, error) { - if string(v1alpha1.NodePending) == ciWorkflow.PodStatus { - return bufio.NewReader(strings.NewReader("")), func() error { return nil }, nil - } - ciLogRequest := types.BuildLogRequest{ - PodName: ciWorkflow.PodName, - Namespace: ciWorkflow.Namespace, - } - isExt := false - clusterConfig := &k8s.ClusterConfig{} - if ciWorkflow.EnvironmentId != 0 { - env, err := impl.envRepository.FindById(ciWorkflow.EnvironmentId) - if err != nil { - return nil, nil, err - } - var clusterBean clusterBean.ClusterBean - if env != nil && env.Cluster != nil { - clusterBean = adapter.GetClusterBean(*env.Cluster) - } - clusterConfig = clusterBean.GetClusterConfig() - isExt = true - } - - logStream, cleanUp, err := impl.ciLogService.FetchRunningWorkflowLogs(ciLogRequest, clusterConfig, isExt) - if logStream == nil || err != nil { - if !ciWorkflow.BlobStorageEnabled { - return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "logs-not-stored-in-repository"} - } else if string(v1alpha1.NodeSucceeded) == ciWorkflow.Status || string(v1alpha1.NodeError) == ciWorkflow.Status || string(v1alpha1.NodeFailed) == ciWorkflow.Status || ciWorkflow.Status == cdWorkflow.WorkflowCancel { - impl.Logger.Debugw("pod is not live", "podName", ciWorkflow.PodName, "err", err) - return impl.getLogsFromRepository(ciWorkflow, clusterConfig, isExt) - } - if err != nil { - impl.Logger.Errorw("err on fetch workflow logs", "err", err) - return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: err.Error()} - } else if logStream == nil { - return nil, cleanUp, fmt.Errorf("no logs found for pod %s", ciWorkflow.PodName) - } - } - logReader := bufio.NewReader(logStream) - return logReader, cleanUp, err -} - -func (impl *CiHandlerImpl) getLogsFromRepository(ciWorkflow *pipelineConfig.CiWorkflow, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { - impl.Logger.Debug("getting historic logs", "ciWorkflowId", ciWorkflow.Id) - ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() - ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion - logsFilePath := impl.config.GetDefaultBuildLogsKeyPrefix() + "/" + ciWorkflow.Name + "/main.log" // this is for backward compatibilty - if strings.Contains(ciWorkflow.LogLocation, "main.log") { - logsFilePath = ciWorkflow.LogLocation - } - ciLogRequest := types.BuildLogRequest{ - PipelineId: ciWorkflow.CiPipelineId, - WorkflowId: ciWorkflow.Id, - PodName: ciWorkflow.PodName, - LogsFilePath: logsFilePath, - CloudProvider: impl.config.CloudProvider, - AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ - Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, - AccountName: impl.config.AzureAccountName, - BlobContainerName: impl.config.AzureBlobContainerCiLog, - AccountKey: impl.config.AzureAccountKey, - }, - AwsS3BaseConfig: &blob_storage.AwsS3BaseConfig{ - AccessKey: impl.config.BlobStorageS3AccessKey, - Passkey: impl.config.BlobStorageS3SecretKey, - EndpointUrl: impl.config.BlobStorageS3Endpoint, - IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - BucketName: ciConfigLogsBucket, - Region: ciConfigCiCacheRegion, - VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, - }, - GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ - BucketName: ciConfigLogsBucket, - CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - }, - } - useExternalBlobStorage := isExternalBlobStorageEnabled(isExt, impl.config.UseBlobStorageConfigInCiWorkflow) - if useExternalBlobStorage { - // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - // from them else return. - cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) - if err != nil { - impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) - return nil, nil, err - } - rq := &ciLogRequest - rq.SetBuildLogRequest(cmConfig, secretConfig) - } - oldLogsStream, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, ciLogRequest) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return nil, nil, err - } - logReader := bufio.NewReader(oldLogsStream) - return logReader, cleanUp, err -} - -func (impl *CiHandlerImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) { - ciWorkflow, err := impl.ciWorkflowRepository.FindById(buildId) - if err != nil { - impl.Logger.Errorw("unable to fetch ciWorkflow", "err", err) - return nil, err - } - useExternalBlobStorage := isExternalBlobStorageEnabled(ciWorkflow.IsExternalRunInJobType(), impl.config.UseBlobStorageConfigInCiWorkflow) - if !ciWorkflow.BlobStorageEnabled { - return nil, errors.New("logs-not-stored-in-repository") - } - - if ciWorkflow.CiPipelineId != pipelineId { - impl.Logger.Error("invalid request, wf not in pipeline") - return nil, errors.New("invalid request, wf not in pipeline") - } - - ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() - item := strconv.Itoa(ciWorkflow.Id) - ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion - azureBlobConfig := &blob_storage.AzureBlobBaseConfig{ - Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, - AccountName: impl.config.AzureAccountName, - BlobContainerName: impl.config.AzureBlobContainerCiLog, - AccountKey: impl.config.AzureAccountKey, - } - awsS3BaseConfig := &blob_storage.AwsS3BaseConfig{ - AccessKey: impl.config.BlobStorageS3AccessKey, - Passkey: impl.config.BlobStorageS3SecretKey, - EndpointUrl: impl.config.BlobStorageS3Endpoint, - IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - BucketName: ciConfigLogsBucket, - Region: ciConfigCiCacheRegion, - VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, - } - gcpBlobBaseConfig := &blob_storage.GcpBlobBaseConfig{ - BucketName: ciConfigLogsBucket, - CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - } - - ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() - key := fmt.Sprintf(ciArtifactLocationFormat, ciWorkflow.Id, ciWorkflow.Id) - if len(ciWorkflow.CiArtifactLocation) != 0 && util3.IsValidUrlSubPath(ciWorkflow.CiArtifactLocation) { - key = ciWorkflow.CiArtifactLocation - } else if util3.IsValidUrlSubPath(key) { - impl.ciWorkflowRepository.MigrateCiArtifactLocation(ciWorkflow.Id, key) - } - baseLogLocationPathConfig := impl.config.BaseLogLocationPath - blobStorageService := blob_storage.NewBlobStorageServiceImpl(nil) - destinationKey := filepath.Clean(filepath.Join(baseLogLocationPathConfig, item)) - request := &blob_storage.BlobStorageRequest{ - StorageType: impl.config.CloudProvider, - SourceKey: key, - DestinationKey: destinationKey, - AzureBlobBaseConfig: azureBlobConfig, - AwsS3BaseConfig: awsS3BaseConfig, - GcpBlobBaseConfig: gcpBlobBaseConfig, - } - if useExternalBlobStorage { - envBean, err := impl.envService.FindById(ciWorkflow.EnvironmentId) - if err != nil { - impl.Logger.Errorw("error in getting envBean by envId", "err", err, "envId", ciWorkflow.EnvironmentId) - return nil, err - } - clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(envBean.ClusterId) - if err != nil { - impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig by clusterId", "err", err, "clusterId", envBean.ClusterId) - return nil, err - } - // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - // from them else return. - cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) - if err != nil { - impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) - return nil, err - } - request = updateRequestWithExtClusterCmAndSecret(request, cmConfig, secretConfig) - } - _, numBytes, err := blobStorageService.Get(request) - if err != nil { - impl.Logger.Errorw("error occurred while downloading file", "request", request, "error", err) - return nil, errors.New("failed to download resource") - } - - file, err := os.Open(destinationKey) - if err != nil { - impl.Logger.Errorw("unable to open file", "file", item, "err", err) - return nil, errors.New("unable to open file") - } - - impl.Logger.Infow("Downloaded ", "filename", file.Name(), "bytes", numBytes) - return file, nil -} - -func (impl *CiHandlerImpl) GetHistoricBuildLogs(workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) { - var err error - if ciWorkflow == nil { - ciWorkflow, err = impl.ciWorkflowRepository.FindById(workflowId) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return nil, err - } - } - ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() - ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion - ciLogRequest := types.BuildLogRequest{ - PipelineId: ciWorkflow.CiPipelineId, - WorkflowId: ciWorkflow.Id, - PodName: ciWorkflow.PodName, - LogsFilePath: ciWorkflow.LogLocation, - CloudProvider: impl.config.CloudProvider, - AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ - Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, - AccountName: impl.config.AzureAccountName, - BlobContainerName: impl.config.AzureBlobContainerCiLog, - AccountKey: impl.config.AzureAccountKey, - }, - AwsS3BaseConfig: &blob_storage.AwsS3BaseConfig{ - AccessKey: impl.config.BlobStorageS3AccessKey, - Passkey: impl.config.BlobStorageS3SecretKey, - EndpointUrl: impl.config.BlobStorageS3Endpoint, - IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - BucketName: ciConfigLogsBucket, - Region: ciConfigCiCacheRegion, - VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, - }, - GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ - BucketName: ciConfigLogsBucket, - CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - }, - } - useExternalBlobStorage := isExternalBlobStorageEnabled(ciWorkflow.IsExternalRunInJobType(), impl.config.UseBlobStorageConfigInCiWorkflow) - if useExternalBlobStorage { - envBean, err := impl.envService.FindById(ciWorkflow.EnvironmentId) - if err != nil { - impl.Logger.Errorw("error in getting envBean by envId", "err", err, "envId", ciWorkflow.EnvironmentId) - return nil, err - } - clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(envBean.ClusterId) - if err != nil { - impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig by clusterId", "err", err, "clusterId", envBean.ClusterId) - return nil, err - } - // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - // from them else return. - cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) - if err != nil { - impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) - return nil, err - } - rq := &ciLogRequest - rq.SetBuildLogRequest(cmConfig, secretConfig) - } - logsFile, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, ciLogRequest) - logs, err := ioutil.ReadFile(logsFile.Name()) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return map[string]string{}, err - } - logStr := string(logs) - resp := make(map[string]string) - resp["logs"] = logStr - defer cleanUp() - return resp, err -} func ExtractWorkflowStatus(workflowStatus v1alpha1.WorkflowStatus) (string, string, string, string, string, string) { workflowName := "" diff --git a/pkg/pipeline/PipelineStageService.go b/pkg/pipeline/PipelineStageService.go index 2ff4123f0f..2a538b7282 100644 --- a/pkg/pipeline/PipelineStageService.go +++ b/pkg/pipeline/PipelineStageService.go @@ -74,6 +74,11 @@ func NewPipelineStageService(logger *zap.SugaredLogger, } } +const ( + preCdStage = "preCD" + postCdStage = "postCD" +) + type PipelineStageServiceImpl struct { logger *zap.SugaredLogger pipelineStageRepository repository.PipelineStageRepository diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index 2a70ebe51e..affa658f1d 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -50,6 +50,7 @@ import ( triggerBean "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/userDeploymentRequest/service" eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" + "github.com/devtron-labs/devtron/pkg/executor" k8sPkg "github.com/devtron-labs/devtron/pkg/k8s" "github.com/devtron-labs/devtron/pkg/pipeline" constants2 "github.com/devtron-labs/devtron/pkg/pipeline/constants" @@ -148,7 +149,7 @@ type WorkflowDagExecutorImpl struct { K8sUtil *k8s.K8sServiceImpl envRepository repository5.EnvironmentRepository k8sCommonService k8sPkg.K8sCommonService - workflowService pipeline.WorkflowService + workflowService executor.WorkflowService ciTriggerService trigger.Service } @@ -181,7 +182,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi K8sUtil *k8s.K8sServiceImpl, envRepository repository5.EnvironmentRepository, k8sCommonService k8sPkg.K8sCommonService, - workflowService pipeline.WorkflowService, + workflowService executor.WorkflowService, ciTriggerService trigger.Service, ) *WorkflowDagExecutorImpl { wde := &WorkflowDagExecutorImpl{logger: Logger, diff --git a/wire_gen.go b/wire_gen.go index a1365d92a7..4f51c1ac3c 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -202,6 +202,7 @@ import ( "github.com/devtron-labs/devtron/pkg/eventProcessor/celEvaluator" "github.com/devtron-labs/devtron/pkg/eventProcessor/in" "github.com/devtron-labs/devtron/pkg/eventProcessor/out" + "github.com/devtron-labs/devtron/pkg/executor" "github.com/devtron-labs/devtron/pkg/externalLink" "github.com/devtron-labs/devtron/pkg/fluxApplication" "github.com/devtron-labs/devtron/pkg/generateManifest" @@ -625,7 +626,7 @@ func InitializeApp() (*App, error) { } ciInfraGetter := ci.NewCiInfraGetter(sugaredLogger, infraConfigServiceImpl, infraConfigAuditServiceImpl) infraProviderImpl := infraProviders.NewInfraProviderImpl(sugaredLogger, infraGetter, ciInfraGetter) - workflowServiceImpl, err := pipeline.NewWorkflowServiceImpl(sugaredLogger, environmentRepositoryImpl, ciCdConfig, configReadServiceImpl, globalCMCSServiceImpl, argoWorkflowExecutorImpl, k8sServiceImpl, systemWorkflowExecutorImpl, k8sCommonServiceImpl, infraProviderImpl) + workflowServiceImpl, err := executor.NewWorkflowServiceImpl(sugaredLogger, environmentRepositoryImpl, ciCdConfig, configReadServiceImpl, globalCMCSServiceImpl, argoWorkflowExecutorImpl, k8sServiceImpl, systemWorkflowExecutorImpl, k8sCommonServiceImpl, infraProviderImpl) if err != nil { return nil, err } @@ -673,7 +674,12 @@ func InitializeApp() (*App, error) { ciCdPipelineOrchestratorImpl := pipeline.NewCiCdPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, clientImpl, ciCdConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, pipelineStageServiceImpl, gitMaterialHistoryServiceImpl, ciPipelineHistoryServiceImpl, ciTemplateReadServiceImpl, ciTemplateServiceImpl, dockerArtifactStoreRepositoryImpl, ciArtifactRepositoryImpl, configMapServiceImpl, customTagServiceImpl, genericNoteServiceImpl, chartServiceImpl, transactionUtilImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl, deploymentConfigReadServiceImpl, chartReadServiceImpl) pluginInputVariableParserImpl := pipeline.NewPluginInputVariableParserImpl(sugaredLogger, dockerRegistryConfigImpl, customTagServiceImpl) ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workFlowStageStatusServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciWorkflowRepositoryImpl, transactionUtilImpl) - serviceImpl := trigger.NewServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciPipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateReadServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, ciCdPipelineOrchestratorImpl, attributesServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl, ciServiceImpl, ciWorkflowRepositoryImpl, clientImpl) + ciLogServiceImpl, err := pipeline.NewCiLogServiceImpl(sugaredLogger, k8sServiceImpl) + if err != nil { + return nil, err + } + blobStorageConfigServiceImpl := pipeline.NewBlobStorageConfigServiceImpl(sugaredLogger, k8sServiceImpl, ciCdConfig) + serviceImpl := trigger.NewServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciPipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateReadServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, ciCdPipelineOrchestratorImpl, attributesServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl, ciServiceImpl, ciWorkflowRepositoryImpl, clientImpl, ciLogServiceImpl, blobStorageConfigServiceImpl, clusterServiceImplExtended, environmentServiceImpl, k8sServiceImpl) gitWebhookServiceImpl := gitWebhook.NewGitWebhookServiceImpl(sugaredLogger, gitWebhookRepositoryImpl, serviceImpl) gitWebhookRestHandlerImpl := restHandler.NewGitWebhookRestHandlerImpl(sugaredLogger, gitWebhookServiceImpl) ecrConfig, err := pipeline.GetEcrConfig() @@ -726,13 +732,8 @@ func InitializeApp() (*App, error) { pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, gitMaterialReadServiceImpl, chartRepositoryImpl, ciPipelineConfigServiceImpl, ciMaterialConfigServiceImpl, appArtifactManagerImpl, devtronAppCMCSServiceImpl, devtronAppStrategyServiceImpl, appDeploymentTypeChangeManagerImpl, cdPipelineConfigServiceImpl, devtronAppConfigServiceImpl) deploymentTemplateValidationServiceImpl := deploymentTemplate.NewDeploymentTemplateValidationServiceImpl(sugaredLogger, chartRefServiceImpl, scopedVariableManagerImpl) devtronAppGitOpConfigServiceImpl := gitOpsConfig.NewDevtronAppGitOpConfigServiceImpl(sugaredLogger, chartRepositoryImpl, chartServiceImpl, gitOpsConfigReadServiceImpl, gitOpsValidationServiceImpl, argoClientWrapperServiceImpl, deploymentConfigServiceImpl, chartReadServiceImpl) - ciLogServiceImpl, err := pipeline.NewCiLogServiceImpl(sugaredLogger, k8sServiceImpl) - if err != nil { - return nil, err - } - blobStorageConfigServiceImpl := pipeline.NewBlobStorageConfigServiceImpl(sugaredLogger, k8sServiceImpl, ciCdConfig) - ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, workflowServiceImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, environmentServiceImpl, workFlowStageStatusServiceImpl, k8sServiceImpl) - cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, workflowServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, customTagServiceImpl, deploymentConfigServiceImpl, workFlowStageStatusServiceImpl, cdWorkflowRunnerServiceImpl) + ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, workFlowStageStatusServiceImpl) + cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, customTagServiceImpl, deploymentConfigServiceImpl, workFlowStageStatusServiceImpl, cdWorkflowRunnerServiceImpl) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl, chartServiceImpl, deploymentConfigServiceImpl, pipelineBuilderImpl) appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, attributesServiceImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, pipelineStageServiceImpl, ciTemplateReadServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigServiceImpl, gitOpsConfigReadServiceImpl, chartReadServiceImpl) deploymentTemplateRepositoryImpl := repository2.NewDeploymentTemplateRepositoryImpl(db, sugaredLogger) @@ -750,7 +751,6 @@ func InitializeApp() (*App, error) { cveStoreRepositoryImpl := repository24.NewCveStoreRepositoryImpl(db, sugaredLogger) policyServiceImpl := imageScanning.NewPolicyServiceImpl(environmentServiceImpl, sugaredLogger, appRepositoryImpl, pipelineOverrideRepositoryImpl, cvePolicyRepositoryImpl, clusterServiceImplExtended, pipelineRepositoryImpl, imageScanResultRepositoryImpl, imageScanDeployInfoRepositoryImpl, imageScanObjectMetaRepositoryImpl, httpClient, ciArtifactRepositoryImpl, ciCdConfig, imageScanHistoryReadServiceImpl, cveStoreRepositoryImpl, ciTemplateRepositoryImpl, clusterReadServiceImpl, transactionUtilImpl) imageScanResultReadServiceImpl := read18.NewImageScanResultReadServiceImpl(sugaredLogger, imageScanResultRepositoryImpl) - pipelineConfigRestHandlerImpl := configure.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, deploymentTemplateValidationServiceImpl, chartServiceImpl, devtronAppGitOpConfigServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, generateManifestDeploymentTemplateServiceImpl, appWorkflowServiceImpl, gitMaterialReadServiceImpl, policyServiceImpl, imageScanResultReadServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingReadServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, ciCdPipelineOrchestratorImpl, gitProviderReadServiceImpl, teamReadServiceImpl, environmentRepositoryImpl, chartReadServiceImpl, serviceImpl) gitOpsManifestPushServiceImpl := publish.NewGitOpsManifestPushServiceImpl(sugaredLogger, pipelineStatusTimelineServiceImpl, pipelineOverrideRepositoryImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, chartServiceImpl, gitOperationServiceImpl, argoClientWrapperServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, chartTemplateServiceImpl) manifestCreationServiceImpl := manifest.NewManifestCreationServiceImpl(sugaredLogger, dockerRegistryIpsConfigServiceImpl, chartRefServiceImpl, scopedVariableCMCSManagerImpl, k8sCommonServiceImpl, deployedAppMetricsServiceImpl, imageDigestPolicyServiceImpl, utilMergeUtil, appCrudOperationServiceImpl, deploymentTemplateServiceImpl, argoClientWrapperServiceImpl, configMapHistoryRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineOverrideRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, pipelineConfigRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, deploymentConfigServiceImpl, envConfigOverrideReadServiceImpl) configMapHistoryReadServiceImpl := read19.NewConfigMapHistoryReadService(sugaredLogger, configMapHistoryRepositoryImpl, scopedVariableCMCSManagerImpl) @@ -763,10 +763,11 @@ func InitializeApp() (*App, error) { scanToolExecutionHistoryMappingRepositoryImpl := repository24.NewScanToolExecutionHistoryMappingRepositoryImpl(db, sugaredLogger) cdWorkflowReadServiceImpl := read20.NewCdWorkflowReadServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) imageScanServiceImpl := imageScanning.NewImageScanServiceImpl(sugaredLogger, imageScanHistoryRepositoryImpl, imageScanResultRepositoryImpl, imageScanObjectMetaRepositoryImpl, cveStoreRepositoryImpl, imageScanDeployInfoRepositoryImpl, userServiceImpl, appRepositoryImpl, environmentServiceImpl, ciArtifactRepositoryImpl, policyServiceImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, scanToolMetadataRepositoryImpl, scanToolExecutionHistoryMappingRepositoryImpl, cvePolicyRepositoryImpl, cdWorkflowReadServiceImpl) - triggerServiceImpl, err := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, workflowServiceImpl, imageDigestPolicyServiceImpl, userServiceImpl, clientImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryReadServiceImpl, imageScanDeployInfoReadServiceImpl, imageScanDeployInfoServiceImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateReadServiceImpl, gitMaterialReadServiceImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, ciCdPipelineOrchestratorImpl, gitOperationServiceImpl, attributesServiceImpl, clusterRepositoryImpl, cdWorkflowRunnerServiceImpl) + triggerServiceImpl, err := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, imageDigestPolicyServiceImpl, userServiceImpl, clientImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryReadServiceImpl, imageScanDeployInfoReadServiceImpl, imageScanDeployInfoServiceImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateReadServiceImpl, gitMaterialReadServiceImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, ciCdPipelineOrchestratorImpl, gitOperationServiceImpl, attributesServiceImpl, clusterRepositoryImpl, cdWorkflowRunnerServiceImpl, clusterServiceImplExtended, ciLogServiceImpl, workflowServiceImpl, blobStorageConfigServiceImpl) if err != nil { return nil, err } + pipelineConfigRestHandlerImpl := configure.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, deploymentTemplateValidationServiceImpl, chartServiceImpl, devtronAppGitOpConfigServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, generateManifestDeploymentTemplateServiceImpl, appWorkflowServiceImpl, gitMaterialReadServiceImpl, policyServiceImpl, imageScanResultReadServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingReadServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, ciCdPipelineOrchestratorImpl, gitProviderReadServiceImpl, teamReadServiceImpl, environmentRepositoryImpl, chartReadServiceImpl, serviceImpl, triggerServiceImpl) commonArtifactServiceImpl := artifacts.NewCommonArtifactServiceImpl(sugaredLogger, ciArtifactRepositoryImpl) workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, cdWorkflowRunnerServiceImpl, ciServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl, runnable, imageScanHistoryRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, environmentRepositoryImpl, k8sCommonServiceImpl, workflowServiceImpl, serviceImpl) externalCiRestHandlerImpl := restHandler.NewExternalCiRestHandlerImpl(sugaredLogger, validate, userServiceImpl, enforcerImpl, workflowDagExecutorImpl) From 1a2de648a2b5566e3ed017506a4257b6ac1575ea Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Mon, 7 Apr 2025 10:07:26 +0530 Subject: [PATCH 24/37] workflow service ent change --- pkg/executor/WorkflowService.go | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/pkg/executor/WorkflowService.go b/pkg/executor/WorkflowService.go index 339c566113..3db6887c1f 100644 --- a/pkg/executor/WorkflowService.go +++ b/pkg/executor/WorkflowService.go @@ -70,9 +70,9 @@ type WorkflowServiceImpl struct { globalCMCSService pipeline.GlobalCMCSService argoWorkflowExecutor executors.ArgoWorkflowExecutor systemWorkflowExecutor executors.SystemWorkflowExecutor - k8sUtil *k8s.K8sServiceImpl k8sCommonService k8s2.K8sCommonService infraProvider infraProviders.InfraProvider + k8sUtil *k8s.K8sServiceImpl } // TODO: Move to bean @@ -83,10 +83,11 @@ func NewWorkflowServiceImpl(Logger *zap.SugaredLogger, configMapService read.ConfigReadService, globalCMCSService pipeline.GlobalCMCSService, argoWorkflowExecutor executors.ArgoWorkflowExecutor, - k8sUtil *k8s.K8sServiceImpl, systemWorkflowExecutor executors.SystemWorkflowExecutor, k8sCommonService k8s2.K8sCommonService, - infraProvider infraProviders.InfraProvider) (*WorkflowServiceImpl, error) { + infraProvider infraProviders.InfraProvider, + k8sUtil *k8s.K8sServiceImpl, +) (*WorkflowServiceImpl, error) { commonWorkflowService := &WorkflowServiceImpl{ Logger: Logger, ciCdConfig: ciCdConfig, @@ -110,9 +111,6 @@ func NewWorkflowServiceImpl(Logger *zap.SugaredLogger, const ( CI_NODE_SELECTOR_APP_LABEL_KEY = "devtron.ai/node-selector" - - preCdStage = "preCD" - postCdStage = "postCD" ) func (impl *WorkflowServiceImpl) SubmitWorkflow(workflowRequest *types.WorkflowRequest) (*unstructured.UnstructuredList, string, error) { @@ -120,13 +118,16 @@ func (impl *WorkflowServiceImpl) SubmitWorkflow(workflowRequest *types.WorkflowR if err != nil { return nil, "", err } - workflowExecutor := impl.getWorkflowExecutor(workflowRequest.WorkflowExecutor) - if workflowExecutor == nil { - return nil, "", errors.New("workflow executor not found") + var createdWf *unstructured.UnstructuredList + canExecuteWorkflow, jobHelmChartPath, err := impl.checkIfCanExecuteWorkflowAndHandleVirtualExec(workflowRequest, workflowTemplate) + if canExecuteWorkflow { + workflowExecutor := impl.getWorkflowExecutor(workflowRequest.WorkflowExecutor) + if workflowExecutor == nil { + return nil, "", errors.New("workflow executor not found") + } + createdWf, err = workflowExecutor.ExecuteWorkflow(workflowTemplate) } - createdWf, err := workflowExecutor.ExecuteWorkflow(workflowTemplate) - jobHelmPackagePath := "" // due to ENT - return createdWf, jobHelmPackagePath, err + return createdWf, jobHelmChartPath, err } func (impl *WorkflowServiceImpl) createWorkflowTemplate(workflowRequest *types.WorkflowRequest) (bean3.WorkflowTemplate, error) { @@ -142,6 +143,11 @@ func (impl *WorkflowServiceImpl) createWorkflowTemplate(workflowRequest *types.W return bean3.WorkflowTemplate{}, err } + workflowTemplate, err = impl.updateWorkflowTemplateWithLabels(workflowRequest, workflowTemplate) + if err != nil { + impl.Logger.Errorw("error occurred while updating workflow template with labels", "err", err) + return bean3.WorkflowTemplate{}, err + } workflowRequest.AddNodeConstraintsFromConfig(&workflowTemplate, impl.ciCdConfig) infraConfiguration := &v1.InfraConfig{} shouldAddExistingCmCsInWorkflow := impl.shouldAddExistingCmCsInWorkflow(workflowRequest) From 3120992fb41866bf72590b9fd0199f55d8f85d84 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Mon, 7 Apr 2025 10:07:36 +0530 Subject: [PATCH 25/37] workflow service ent change --- pkg/executor/WorkflowService_ent.go | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 pkg/executor/WorkflowService_ent.go diff --git a/pkg/executor/WorkflowService_ent.go b/pkg/executor/WorkflowService_ent.go new file mode 100644 index 0000000000..f01a801a90 --- /dev/null +++ b/pkg/executor/WorkflowService_ent.go @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2020-2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package executor + +import ( + bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/types" +) + +func (impl *WorkflowServiceImpl) checkIfCanExecuteWorkflowAndHandleVirtualExec(workflowRequest *types.WorkflowRequest, workflowTemplate bean3.WorkflowTemplate) (canExecuteWorkflow bool, jobHelmChartPath string, err error) { + return true, "", nil +} + +func (impl *WorkflowServiceImpl) updateWorkflowTemplateWithLabels(workflowRequest *types.WorkflowRequest, workflowTemplate bean3.WorkflowTemplate) (bean3.WorkflowTemplate, error) { + return workflowTemplate, nil +} From 7ae5b3d8eabefb0c7cbfc2929b0722d4110748d1 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Mon, 7 Apr 2025 10:53:37 +0530 Subject: [PATCH 26/37] Redundant import removed --- .../trigger/devtronApps/TriggerService.go | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/TriggerService.go index 942761fc47..bcef986aa7 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService.go @@ -31,7 +31,6 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer" bean7 "github.com/devtron-labs/devtron/client/argocdServer/bean" client "github.com/devtron-labs/devtron/client/events" - gitSensorClient "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/middleware" "github.com/devtron-labs/devtron/internal/sql/models" repository3 "github.com/devtron-labs/devtron/internal/sql/repository" @@ -146,7 +145,6 @@ type TriggerServiceImpl struct { scopedVariableManager variables.ScopedVariableCMCSManager imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService userService user.UserService - gitSensorClient gitSensorClient.Client config *types.CdConfig helmAppService client2.HelmAppService imageScanService security2.ImageScanService @@ -175,7 +173,6 @@ type TriggerServiceImpl struct { K8sUtil *util5.K8sServiceImpl transactionUtilImpl *sql.TransactionUtilImpl deploymentConfigService common.DeploymentConfigService - deploymentServiceTypeConfig *globalUtil.DeploymentServiceTypeConfig ciCdPipelineOrchestrator pipeline.CiCdPipelineOrchestrator gitOperationService git.GitOperationService attributeService attributes.AttributesService @@ -207,7 +204,6 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, scopedVariableManager variables.ScopedVariableCMCSManager, imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService, userService user.UserService, - gitSensorClient gitSensorClient.Client, helmAppService client2.HelmAppService, enforcerUtil rbac.EnforcerUtil, userDeploymentRequestService service.UserDeploymentRequestService, @@ -269,7 +265,6 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, scopedVariableManager: scopedVariableManager, imageDigestPolicyService: imageDigestPolicyService, userService: userService, - gitSensorClient: gitSensorClient, helmAppService: helmAppService, enforcerUtil: enforcerUtil, eventFactory: eventFactory, @@ -303,12 +298,11 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, transactionUtilImpl: transactionUtilImpl, - deploymentConfigService: deploymentConfigService, - deploymentServiceTypeConfig: envVariables.DeploymentServiceTypeConfig, - ciCdPipelineOrchestrator: ciCdPipelineOrchestrator, - gitOperationService: gitOperationService, - attributeService: attributeService, - cdWorkflowRunnerService: cdWorkflowRunnerService, + deploymentConfigService: deploymentConfigService, + ciCdPipelineOrchestrator: ciCdPipelineOrchestrator, + gitOperationService: gitOperationService, + attributeService: attributeService, + cdWorkflowRunnerService: cdWorkflowRunnerService, clusterRepository: clusterRepository, clusterService: clusterService, From 5ab4dfb183e15f6ad3ceefd11819056ea6f16883 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Mon, 7 Apr 2025 12:02:07 +0530 Subject: [PATCH 27/37] rename pkg/ --- api/argoApplication/ArgoApplicationRestHandler.go | 4 ++-- api/argoApplication/wire_argoApplication.go | 6 +++--- api/helm-app/HelmAppRestHandler.go | 4 ++-- api/k8s/application/k8sApplicationRestHandler.go | 4 ++-- cmd/external-app/wire_gen.go | 6 +++--- pkg/app/AppService.go | 2 +- .../InstalledAppDeploymentTypeChangeService.go | 4 ++-- .../service/FullMode/resource/ResourceTreeService.go | 4 ++-- .../argoApplication/ArgoApplicationService.go | 6 +++--- .../argoApplication/ArgoApplicationServiceExtended.go | 4 ++-- pkg/{nucleus => coreEntities}/argoApplication/bean/bean.go | 0 .../argoApplication/helper/deploymentStatusHelper.go | 0 .../argoApplication/helper/helper.go | 2 +- .../argoApplication/read/ArgoApplicationReadService.go | 4 ++-- .../read/config/ArgoApplicationConfigService.go | 4 ++-- .../deployedApp/status/resourceTree/ResourceTreeService.go | 4 ++-- pkg/deployment/trigger/devtronApps/TriggerService_ent1.go | 2 +- pkg/k8s/K8sCommonService.go | 2 +- pkg/k8s/application/k8sApplicationService.go | 2 +- pkg/terminal/terminalSesion.go | 2 +- wire_gen.go | 6 +++--- 21 files changed, 36 insertions(+), 36 deletions(-) rename pkg/{nucleus => coreEntities}/argoApplication/ArgoApplicationService.go (97%) rename pkg/{nucleus => coreEntities}/argoApplication/ArgoApplicationServiceExtended.go (99%) rename pkg/{nucleus => coreEntities}/argoApplication/bean/bean.go (100%) rename pkg/{nucleus => coreEntities}/argoApplication/helper/deploymentStatusHelper.go (100%) rename pkg/{nucleus => coreEntities}/argoApplication/helper/helper.go (98%) rename pkg/{nucleus => coreEntities}/argoApplication/read/ArgoApplicationReadService.go (98%) rename pkg/{nucleus => coreEntities}/argoApplication/read/config/ArgoApplicationConfigService.go (97%) diff --git a/api/argoApplication/ArgoApplicationRestHandler.go b/api/argoApplication/ArgoApplicationRestHandler.go index 27d9a8632e..2ac9e84e33 100644 --- a/api/argoApplication/ArgoApplicationRestHandler.go +++ b/api/argoApplication/ArgoApplicationRestHandler.go @@ -21,8 +21,8 @@ import ( "errors" "github.com/devtron-labs/devtron/api/restHandler/common" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read" "go.uber.org/zap" "net/http" "strconv" diff --git a/api/argoApplication/wire_argoApplication.go b/api/argoApplication/wire_argoApplication.go index 6373e274f8..95af6b89d1 100644 --- a/api/argoApplication/wire_argoApplication.go +++ b/api/argoApplication/wire_argoApplication.go @@ -17,9 +17,9 @@ package argoApplication import ( - argoApplication2 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read/config" + argoApplication2 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read/config" "github.com/google/wire" ) diff --git a/api/helm-app/HelmAppRestHandler.go b/api/helm-app/HelmAppRestHandler.go index feab53cc95..a8010b3503 100644 --- a/api/helm-app/HelmAppRestHandler.go +++ b/api/helm-app/HelmAppRestHandler.go @@ -25,11 +25,11 @@ import ( "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/EAMode" clusterBean "github.com/devtron-labs/devtron/pkg/cluster/bean" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" clientErrors "github.com/devtron-labs/devtron/pkg/errors" "github.com/devtron-labs/devtron/pkg/fluxApplication" bean2 "github.com/devtron-labs/devtron/pkg/k8s/application/bean" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/pipeline" "net/http" "strconv" diff --git a/api/k8s/application/k8sApplicationRestHandler.go b/api/k8s/application/k8sApplicationRestHandler.go index 55d42ed15c..190c79c05b 100644 --- a/api/k8s/application/k8sApplicationRestHandler.go +++ b/api/k8s/application/k8sApplicationRestHandler.go @@ -36,14 +36,14 @@ import ( "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/auth/user" bean4 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read" clientErrors "github.com/devtron-labs/devtron/pkg/errors" "github.com/devtron-labs/devtron/pkg/fluxApplication" "github.com/devtron-labs/devtron/pkg/k8s" application2 "github.com/devtron-labs/devtron/pkg/k8s/application" bean2 "github.com/devtron-labs/devtron/pkg/k8s/application/bean" bean3 "github.com/devtron-labs/devtron/pkg/k8s/bean" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read" "github.com/devtron-labs/devtron/pkg/terminal" "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/rbac" diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index 7e1c13fc04..01f77271cd 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -115,9 +115,9 @@ import ( read7 "github.com/devtron-labs/devtron/pkg/module/read" "github.com/devtron-labs/devtron/pkg/module/repo" "github.com/devtron-labs/devtron/pkg/module/store" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" - read9 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read" - config3 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read/config" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" + read9 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read" + config3 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool" repository11 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 5d44b6f4a4..217f6d0097 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -29,6 +29,7 @@ import ( internalUtil "github.com/devtron-labs/devtron/internal/util" bean3 "github.com/devtron-labs/devtron/pkg/app/bean" installedAppReader "github.com/devtron-labs/devtron/pkg/appStore/installedApp/read" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" bean2 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" @@ -38,7 +39,6 @@ import ( bean6 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/bean" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/read" bean4 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/workflow/cd" "net/url" "strconv" diff --git a/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go b/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go index 1895a3975b..a3c530dba2 100644 --- a/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go +++ b/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go @@ -42,13 +42,13 @@ import ( "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/cluster/read" repository5 "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" + bean4 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" "github.com/devtron-labs/devtron/pkg/deployment/common" bean3 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" bean2 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/k8s" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" - bean4 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" util3 "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" "go.uber.org/zap" diff --git a/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go b/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go index 082cb6856e..49c965f78b 100644 --- a/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go +++ b/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go @@ -37,12 +37,12 @@ import ( "github.com/devtron-labs/devtron/pkg/appStore/bean" appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/repository" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" + bean3 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" "github.com/devtron-labs/devtron/pkg/deployment/common" bean2 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/k8s" application3 "github.com/devtron-labs/devtron/pkg/k8s/application" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" - bean3 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" util3 "github.com/devtron-labs/devtron/pkg/util" util2 "github.com/devtron-labs/devtron/util" "github.com/tidwall/gjson" diff --git a/pkg/nucleus/argoApplication/ArgoApplicationService.go b/pkg/coreEntities/argoApplication/ArgoApplicationService.go similarity index 97% rename from pkg/nucleus/argoApplication/ArgoApplicationService.go rename to pkg/coreEntities/argoApplication/ArgoApplicationService.go index 4bd4f893bc..db8ae43dd3 100644 --- a/pkg/nucleus/argoApplication/ArgoApplicationService.go +++ b/pkg/coreEntities/argoApplication/ArgoApplicationService.go @@ -27,13 +27,13 @@ import ( util2 "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/cluster/adapter" clusterRepository "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/deployment/common" commonBean "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/k8s/application" k8s2 "github.com/devtron-labs/devtron/pkg/k8s/bean" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read/config" "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/sliceUtil" "go.uber.org/zap" diff --git a/pkg/nucleus/argoApplication/ArgoApplicationServiceExtended.go b/pkg/coreEntities/argoApplication/ArgoApplicationServiceExtended.go similarity index 99% rename from pkg/nucleus/argoApplication/ArgoApplicationServiceExtended.go rename to pkg/coreEntities/argoApplication/ArgoApplicationServiceExtended.go index 2643bbe432..40c239ea16 100644 --- a/pkg/nucleus/argoApplication/ArgoApplicationServiceExtended.go +++ b/pkg/coreEntities/argoApplication/ArgoApplicationServiceExtended.go @@ -29,8 +29,8 @@ import ( util2 "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app/appDetails/adapter" "github.com/devtron-labs/devtron/pkg/cluster" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read" util3 "github.com/devtron-labs/devtron/pkg/util" "github.com/devtron-labs/devtron/util" "google.golang.org/grpc" diff --git a/pkg/nucleus/argoApplication/bean/bean.go b/pkg/coreEntities/argoApplication/bean/bean.go similarity index 100% rename from pkg/nucleus/argoApplication/bean/bean.go rename to pkg/coreEntities/argoApplication/bean/bean.go diff --git a/pkg/nucleus/argoApplication/helper/deploymentStatusHelper.go b/pkg/coreEntities/argoApplication/helper/deploymentStatusHelper.go similarity index 100% rename from pkg/nucleus/argoApplication/helper/deploymentStatusHelper.go rename to pkg/coreEntities/argoApplication/helper/deploymentStatusHelper.go diff --git a/pkg/nucleus/argoApplication/helper/helper.go b/pkg/coreEntities/argoApplication/helper/helper.go similarity index 98% rename from pkg/nucleus/argoApplication/helper/helper.go rename to pkg/coreEntities/argoApplication/helper/helper.go index f68840f416..1d377555b2 100644 --- a/pkg/nucleus/argoApplication/helper/helper.go +++ b/pkg/coreEntities/argoApplication/helper/helper.go @@ -5,7 +5,7 @@ import ( "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/api/helm-app/gRPC" "github.com/devtron-labs/devtron/pkg/cluster/repository" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" "strconv" "strings" ) diff --git a/pkg/nucleus/argoApplication/read/ArgoApplicationReadService.go b/pkg/coreEntities/argoApplication/read/ArgoApplicationReadService.go similarity index 98% rename from pkg/nucleus/argoApplication/read/ArgoApplicationReadService.go rename to pkg/coreEntities/argoApplication/read/ArgoApplicationReadService.go index d95521ba17..10742756a9 100644 --- a/pkg/nucleus/argoApplication/read/ArgoApplicationReadService.go +++ b/pkg/coreEntities/argoApplication/read/ArgoApplicationReadService.go @@ -26,9 +26,9 @@ import ( "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/pkg/cluster/adapter" clusterRepository "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" clientErrors "github.com/devtron-labs/devtron/pkg/errors" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" "go.uber.org/zap" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/pkg/nucleus/argoApplication/read/config/ArgoApplicationConfigService.go b/pkg/coreEntities/argoApplication/read/config/ArgoApplicationConfigService.go similarity index 97% rename from pkg/nucleus/argoApplication/read/config/ArgoApplicationConfigService.go rename to pkg/coreEntities/argoApplication/read/config/ArgoApplicationConfigService.go index 95aee5d25c..dfcda06dbc 100644 --- a/pkg/nucleus/argoApplication/read/config/ArgoApplicationConfigService.go +++ b/pkg/coreEntities/argoApplication/read/config/ArgoApplicationConfigService.go @@ -8,8 +8,8 @@ import ( k8sCommonBean "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/pkg/cluster/adapter" clusterRepository "github.com/devtron-labs/devtron/pkg/cluster/repository" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" "go.uber.org/zap" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/pkg/deployment/deployedApp/status/resourceTree/ResourceTreeService.go b/pkg/deployment/deployedApp/status/resourceTree/ResourceTreeService.go index 4f5ea83ca5..1462c172a5 100644 --- a/pkg/deployment/deployedApp/status/resourceTree/ResourceTreeService.go +++ b/pkg/deployment/deployedApp/status/resourceTree/ResourceTreeService.go @@ -35,11 +35,11 @@ import ( "github.com/devtron-labs/devtron/pkg/app" "github.com/devtron-labs/devtron/pkg/appStatus" read2 "github.com/devtron-labs/devtron/pkg/cluster/environment/read" + argoApplication2 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" + bean2 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" commonBean "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/k8s" application2 "github.com/devtron-labs/devtron/pkg/k8s/application" - argoApplication2 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" - bean2 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" util2 "github.com/devtron-labs/devtron/util" "go.uber.org/zap" "strconv" diff --git a/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go b/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go index 6c54cf9c57..fb45e1ac27 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go @@ -10,9 +10,9 @@ import ( bean4 "github.com/devtron-labs/devtron/pkg/app/bean" bean2 "github.com/devtron-labs/devtron/pkg/bean" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/deployment/manifest/publish" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/pipeline/repository" "helm.sh/helm/v3/pkg/chart" ) diff --git a/pkg/k8s/K8sCommonService.go b/pkg/k8s/K8sCommonService.go index d448dd2311..6d58635abc 100644 --- a/pkg/k8s/K8sCommonService.go +++ b/pkg/k8s/K8sCommonService.go @@ -28,9 +28,9 @@ import ( bean2 "github.com/devtron-labs/devtron/pkg/cluster/bean" bean4 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" "github.com/devtron-labs/devtron/pkg/cluster/read" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read/config" bean3 "github.com/devtron-labs/devtron/pkg/k8s/application/bean" bean5 "github.com/devtron-labs/devtron/pkg/k8s/bean" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read/config" "github.com/devtron-labs/devtron/util" "go.opentelemetry.io/otel" "go.uber.org/zap" diff --git a/pkg/k8s/application/k8sApplicationService.go b/pkg/k8s/application/k8sApplicationService.go index 7ab5a6b6b7..d555018363 100644 --- a/pkg/k8s/application/k8sApplicationService.go +++ b/pkg/k8s/application/k8sApplicationService.go @@ -28,11 +28,11 @@ import ( "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" bean5 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" "github.com/devtron-labs/devtron/pkg/cluster/read" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" clientErrors "github.com/devtron-labs/devtron/pkg/errors" "github.com/devtron-labs/devtron/pkg/fluxApplication" bean2 "github.com/devtron-labs/devtron/pkg/fluxApplication/bean" bean4 "github.com/devtron-labs/devtron/pkg/k8s/bean" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" "io" v1 "k8s.io/client-go/kubernetes/typed/core/v1" "net/http" diff --git a/pkg/terminal/terminalSesion.go b/pkg/terminal/terminalSesion.go index 07549991f7..7f720b770d 100644 --- a/pkg/terminal/terminalSesion.go +++ b/pkg/terminal/terminalSesion.go @@ -32,7 +32,7 @@ import ( bean2 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" "github.com/devtron-labs/devtron/pkg/cluster/read" "github.com/devtron-labs/devtron/pkg/cluster/repository" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read/config" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read/config" errors1 "github.com/juju/errors" "go.uber.org/zap" "io" diff --git a/wire_gen.go b/wire_gen.go index 4f51c1ac3c..f0051f606c 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -227,9 +227,9 @@ import ( "github.com/devtron-labs/devtron/pkg/module/repo" "github.com/devtron-labs/devtron/pkg/module/store" "github.com/devtron-labs/devtron/pkg/notifier" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" - read22 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read" - config2 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read/config" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" + read22 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read" + config2 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/history" From 6bd6815c1cf22ed35f2b19522ab67204cc440b12 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Mon, 7 Apr 2025 12:06:53 +0530 Subject: [PATCH 28/37] reverted pkg categorisation --- .../ArgoApplicationRestHandler.go | 4 ++-- api/argoApplication/wire_argoApplication.go | 16 ++++++++-------- api/helm-app/HelmAppRestHandler.go | 4 ++-- api/k8s/application/k8sApplicationRestHandler.go | 4 ++-- cmd/external-app/wire_gen.go | 6 +++--- pkg/app/AppService.go | 2 +- .../InstalledAppDeploymentTypeChangeService.go | 4 ++-- .../FullMode/resource/ResourceTreeService.go | 4 ++-- .../argoApplication/ArgoApplicationService.go | 6 +++--- .../ArgoApplicationServiceExtended.go | 4 ++-- .../argoApplication/bean/bean.go | 0 .../helper/deploymentStatusHelper.go | 0 .../argoApplication/helper/helper.go | 2 +- .../read/ArgoApplicationReadService.go | 4 ++-- .../read/config/ArgoApplicationConfigService.go | 4 ++-- .../status/resourceTree/ResourceTreeService.go | 4 ++-- .../trigger/devtronApps/TriggerService_ent1.go | 2 +- pkg/k8s/K8sCommonService.go | 2 +- pkg/k8s/application/k8sApplicationService.go | 2 +- pkg/terminal/terminalSesion.go | 2 +- wire_gen.go | 10 +++++----- 21 files changed, 43 insertions(+), 43 deletions(-) rename pkg/{coreEntities => }/argoApplication/ArgoApplicationService.go (97%) rename pkg/{coreEntities => }/argoApplication/ArgoApplicationServiceExtended.go (99%) rename pkg/{coreEntities => }/argoApplication/bean/bean.go (100%) rename pkg/{coreEntities => }/argoApplication/helper/deploymentStatusHelper.go (100%) rename pkg/{coreEntities => }/argoApplication/helper/helper.go (98%) rename pkg/{coreEntities => }/argoApplication/read/ArgoApplicationReadService.go (98%) rename pkg/{coreEntities => }/argoApplication/read/config/ArgoApplicationConfigService.go (97%) diff --git a/api/argoApplication/ArgoApplicationRestHandler.go b/api/argoApplication/ArgoApplicationRestHandler.go index 2ac9e84e33..edcbabdca9 100644 --- a/api/argoApplication/ArgoApplicationRestHandler.go +++ b/api/argoApplication/ArgoApplicationRestHandler.go @@ -20,9 +20,9 @@ import ( "context" "errors" "github.com/devtron-labs/devtron/api/restHandler/common" + "github.com/devtron-labs/devtron/pkg/argoApplication" + "github.com/devtron-labs/devtron/pkg/argoApplication/read" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read" "go.uber.org/zap" "net/http" "strconv" diff --git a/api/argoApplication/wire_argoApplication.go b/api/argoApplication/wire_argoApplication.go index 95af6b89d1..45ef8139db 100644 --- a/api/argoApplication/wire_argoApplication.go +++ b/api/argoApplication/wire_argoApplication.go @@ -17,9 +17,9 @@ package argoApplication import ( - argoApplication2 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read/config" + argoApplication3 "github.com/devtron-labs/devtron/pkg/argoApplication" + "github.com/devtron-labs/devtron/pkg/argoApplication/read" + "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" "github.com/google/wire" ) @@ -30,9 +30,9 @@ var ArgoApplicationWireSetFull = wire.NewSet( config.NewArgoApplicationConfigServiceImpl, wire.Bind(new(config.ArgoApplicationConfigService), new(*config.ArgoApplicationConfigServiceImpl)), - argoApplication2.NewArgoApplicationServiceImpl, - argoApplication2.NewArgoApplicationServiceExtendedServiceImpl, - wire.Bind(new(argoApplication2.ArgoApplicationService), new(*argoApplication2.ArgoApplicationServiceExtendedImpl)), + argoApplication3.NewArgoApplicationServiceImpl, + argoApplication3.NewArgoApplicationServiceExtendedServiceImpl, + wire.Bind(new(argoApplication3.ArgoApplicationService), new(*argoApplication3.ArgoApplicationServiceExtendedImpl)), NewArgoApplicationRestHandlerImpl, wire.Bind(new(ArgoApplicationRestHandler), new(*ArgoApplicationRestHandlerImpl)), @@ -48,8 +48,8 @@ var ArgoApplicationWireSetEA = wire.NewSet( config.NewArgoApplicationConfigServiceImpl, wire.Bind(new(config.ArgoApplicationConfigService), new(*config.ArgoApplicationConfigServiceImpl)), - argoApplication2.NewArgoApplicationServiceImpl, - wire.Bind(new(argoApplication2.ArgoApplicationService), new(*argoApplication2.ArgoApplicationServiceImpl)), + argoApplication3.NewArgoApplicationServiceImpl, + wire.Bind(new(argoApplication3.ArgoApplicationService), new(*argoApplication3.ArgoApplicationServiceImpl)), NewArgoApplicationRestHandlerImpl, wire.Bind(new(ArgoApplicationRestHandler), new(*ArgoApplicationRestHandlerImpl)), diff --git a/api/helm-app/HelmAppRestHandler.go b/api/helm-app/HelmAppRestHandler.go index a8010b3503..a21cf730ff 100644 --- a/api/helm-app/HelmAppRestHandler.go +++ b/api/helm-app/HelmAppRestHandler.go @@ -24,9 +24,9 @@ import ( service2 "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/EAMode" + "github.com/devtron-labs/devtron/pkg/argoApplication" + "github.com/devtron-labs/devtron/pkg/argoApplication/helper" clusterBean "github.com/devtron-labs/devtron/pkg/cluster/bean" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" clientErrors "github.com/devtron-labs/devtron/pkg/errors" "github.com/devtron-labs/devtron/pkg/fluxApplication" bean2 "github.com/devtron-labs/devtron/pkg/k8s/application/bean" diff --git a/api/k8s/application/k8sApplicationRestHandler.go b/api/k8s/application/k8sApplicationRestHandler.go index 190c79c05b..dd68e18e15 100644 --- a/api/k8s/application/k8sApplicationRestHandler.go +++ b/api/k8s/application/k8sApplicationRestHandler.go @@ -33,11 +33,11 @@ import ( client "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/api/restHandler/common" util2 "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/argoApplication/helper" + "github.com/devtron-labs/devtron/pkg/argoApplication/read" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/auth/user" bean4 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read" clientErrors "github.com/devtron-labs/devtron/pkg/errors" "github.com/devtron-labs/devtron/pkg/fluxApplication" "github.com/devtron-labs/devtron/pkg/k8s" diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index 01f77271cd..e827637c0a 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -75,6 +75,9 @@ import ( "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/common" "github.com/devtron-labs/devtron/pkg/appStore/values/repository" service4 "github.com/devtron-labs/devtron/pkg/appStore/values/service" + "github.com/devtron-labs/devtron/pkg/argoApplication" + read9 "github.com/devtron-labs/devtron/pkg/argoApplication/read" + config3 "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/attributes" "github.com/devtron-labs/devtron/pkg/auth/authentication" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" @@ -115,9 +118,6 @@ import ( read7 "github.com/devtron-labs/devtron/pkg/module/read" "github.com/devtron-labs/devtron/pkg/module/repo" "github.com/devtron-labs/devtron/pkg/module/store" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" - read9 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read" - config3 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool" repository11 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 217f6d0097..5bb33db0c8 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -29,7 +29,7 @@ import ( internalUtil "github.com/devtron-labs/devtron/internal/util" bean3 "github.com/devtron-labs/devtron/pkg/app/bean" installedAppReader "github.com/devtron-labs/devtron/pkg/appStore/installedApp/read" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" + "github.com/devtron-labs/devtron/pkg/argoApplication/helper" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" bean2 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" diff --git a/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go b/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go index a3c530dba2..2d713500d6 100644 --- a/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go +++ b/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go @@ -37,13 +37,13 @@ import ( deployment2 "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/EAMode/deployment" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/FullMode/deployment" util2 "github.com/devtron-labs/devtron/pkg/appStore/util" + "github.com/devtron-labs/devtron/pkg/argoApplication" + bean4 "github.com/devtron-labs/devtron/pkg/argoApplication/bean" "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/cluster" "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/cluster/read" repository5 "github.com/devtron-labs/devtron/pkg/cluster/repository" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" - bean4 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" "github.com/devtron-labs/devtron/pkg/deployment/common" bean3 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" diff --git a/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go b/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go index 49c965f78b..5a4cd57313 100644 --- a/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go +++ b/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go @@ -37,8 +37,8 @@ import ( "github.com/devtron-labs/devtron/pkg/appStore/bean" appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/repository" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" - bean3 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/argoApplication" + bean3 "github.com/devtron-labs/devtron/pkg/argoApplication/bean" "github.com/devtron-labs/devtron/pkg/deployment/common" bean2 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/k8s" diff --git a/pkg/coreEntities/argoApplication/ArgoApplicationService.go b/pkg/argoApplication/ArgoApplicationService.go similarity index 97% rename from pkg/coreEntities/argoApplication/ArgoApplicationService.go rename to pkg/argoApplication/ArgoApplicationService.go index db8ae43dd3..48cd31c56d 100644 --- a/pkg/coreEntities/argoApplication/ArgoApplicationService.go +++ b/pkg/argoApplication/ArgoApplicationService.go @@ -25,11 +25,11 @@ import ( "github.com/devtron-labs/devtron/api/helm-app/service" argoApplication "github.com/devtron-labs/devtron/client/argocdServer/bean" util2 "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/argoApplication/helper" + "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/cluster/adapter" clusterRepository "github.com/devtron-labs/devtron/pkg/cluster/repository" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/deployment/common" commonBean "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/k8s/application" diff --git a/pkg/coreEntities/argoApplication/ArgoApplicationServiceExtended.go b/pkg/argoApplication/ArgoApplicationServiceExtended.go similarity index 99% rename from pkg/coreEntities/argoApplication/ArgoApplicationServiceExtended.go rename to pkg/argoApplication/ArgoApplicationServiceExtended.go index 40c239ea16..0cfb81b2d3 100644 --- a/pkg/coreEntities/argoApplication/ArgoApplicationServiceExtended.go +++ b/pkg/argoApplication/ArgoApplicationServiceExtended.go @@ -28,9 +28,9 @@ import ( argoApplication "github.com/devtron-labs/devtron/client/argocdServer/bean" util2 "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app/appDetails/adapter" + "github.com/devtron-labs/devtron/pkg/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/argoApplication/read" "github.com/devtron-labs/devtron/pkg/cluster" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read" util3 "github.com/devtron-labs/devtron/pkg/util" "github.com/devtron-labs/devtron/util" "google.golang.org/grpc" diff --git a/pkg/coreEntities/argoApplication/bean/bean.go b/pkg/argoApplication/bean/bean.go similarity index 100% rename from pkg/coreEntities/argoApplication/bean/bean.go rename to pkg/argoApplication/bean/bean.go diff --git a/pkg/coreEntities/argoApplication/helper/deploymentStatusHelper.go b/pkg/argoApplication/helper/deploymentStatusHelper.go similarity index 100% rename from pkg/coreEntities/argoApplication/helper/deploymentStatusHelper.go rename to pkg/argoApplication/helper/deploymentStatusHelper.go diff --git a/pkg/coreEntities/argoApplication/helper/helper.go b/pkg/argoApplication/helper/helper.go similarity index 98% rename from pkg/coreEntities/argoApplication/helper/helper.go rename to pkg/argoApplication/helper/helper.go index 1d377555b2..6197e150b4 100644 --- a/pkg/coreEntities/argoApplication/helper/helper.go +++ b/pkg/argoApplication/helper/helper.go @@ -4,8 +4,8 @@ import ( "fmt" "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/api/helm-app/gRPC" + "github.com/devtron-labs/devtron/pkg/argoApplication/bean" "github.com/devtron-labs/devtron/pkg/cluster/repository" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" "strconv" "strings" ) diff --git a/pkg/coreEntities/argoApplication/read/ArgoApplicationReadService.go b/pkg/argoApplication/read/ArgoApplicationReadService.go similarity index 98% rename from pkg/coreEntities/argoApplication/read/ArgoApplicationReadService.go rename to pkg/argoApplication/read/ArgoApplicationReadService.go index 10742756a9..b0f30ccb8f 100644 --- a/pkg/coreEntities/argoApplication/read/ArgoApplicationReadService.go +++ b/pkg/argoApplication/read/ArgoApplicationReadService.go @@ -24,10 +24,10 @@ import ( k8sCommonBean "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/api/helm-app/gRPC" "github.com/devtron-labs/devtron/api/helm-app/service" + "github.com/devtron-labs/devtron/pkg/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/cluster/adapter" clusterRepository "github.com/devtron-labs/devtron/pkg/cluster/repository" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" clientErrors "github.com/devtron-labs/devtron/pkg/errors" "go.uber.org/zap" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/coreEntities/argoApplication/read/config/ArgoApplicationConfigService.go b/pkg/argoApplication/read/config/ArgoApplicationConfigService.go similarity index 97% rename from pkg/coreEntities/argoApplication/read/config/ArgoApplicationConfigService.go rename to pkg/argoApplication/read/config/ArgoApplicationConfigService.go index dfcda06dbc..0c5a8bf819 100644 --- a/pkg/coreEntities/argoApplication/read/config/ArgoApplicationConfigService.go +++ b/pkg/argoApplication/read/config/ArgoApplicationConfigService.go @@ -6,10 +6,10 @@ import ( "fmt" "github.com/devtron-labs/common-lib/utils/k8s" k8sCommonBean "github.com/devtron-labs/common-lib/utils/k8s/commonBean" + "github.com/devtron-labs/devtron/pkg/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/cluster/adapter" clusterRepository "github.com/devtron-labs/devtron/pkg/cluster/repository" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" "go.uber.org/zap" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/pkg/deployment/deployedApp/status/resourceTree/ResourceTreeService.go b/pkg/deployment/deployedApp/status/resourceTree/ResourceTreeService.go index 1462c172a5..b6fd6b5f69 100644 --- a/pkg/deployment/deployedApp/status/resourceTree/ResourceTreeService.go +++ b/pkg/deployment/deployedApp/status/resourceTree/ResourceTreeService.go @@ -34,9 +34,9 @@ import ( "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" "github.com/devtron-labs/devtron/pkg/appStatus" + argoApplication2 "github.com/devtron-labs/devtron/pkg/argoApplication" + bean2 "github.com/devtron-labs/devtron/pkg/argoApplication/bean" read2 "github.com/devtron-labs/devtron/pkg/cluster/environment/read" - argoApplication2 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" - bean2 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" commonBean "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/k8s" application2 "github.com/devtron-labs/devtron/pkg/k8s/application" diff --git a/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go b/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go index fb45e1ac27..f008f72adc 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go @@ -8,9 +8,9 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/app" bean4 "github.com/devtron-labs/devtron/pkg/app/bean" + "github.com/devtron-labs/devtron/pkg/argoApplication/helper" bean2 "github.com/devtron-labs/devtron/pkg/bean" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/deployment/manifest/publish" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/pipeline/repository" diff --git a/pkg/k8s/K8sCommonService.go b/pkg/k8s/K8sCommonService.go index 6d58635abc..0c3a2a9401 100644 --- a/pkg/k8s/K8sCommonService.go +++ b/pkg/k8s/K8sCommonService.go @@ -25,10 +25,10 @@ import ( "github.com/devtron-labs/devtron/api/bean/AppView" helmBean "github.com/devtron-labs/devtron/api/helm-app/service/bean" internalUtil "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" bean2 "github.com/devtron-labs/devtron/pkg/cluster/bean" bean4 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" "github.com/devtron-labs/devtron/pkg/cluster/read" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read/config" bean3 "github.com/devtron-labs/devtron/pkg/k8s/application/bean" bean5 "github.com/devtron-labs/devtron/pkg/k8s/bean" "github.com/devtron-labs/devtron/util" diff --git a/pkg/k8s/application/k8sApplicationService.go b/pkg/k8s/application/k8sApplicationService.go index d555018363..39588ef15e 100644 --- a/pkg/k8s/application/k8sApplicationService.go +++ b/pkg/k8s/application/k8sApplicationService.go @@ -25,10 +25,10 @@ import ( "github.com/devtron-labs/devtron/api/helm-app/gRPC" client "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/api/helm-app/service/bean" + "github.com/devtron-labs/devtron/pkg/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" bean5 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" "github.com/devtron-labs/devtron/pkg/cluster/read" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" clientErrors "github.com/devtron-labs/devtron/pkg/errors" "github.com/devtron-labs/devtron/pkg/fluxApplication" bean2 "github.com/devtron-labs/devtron/pkg/fluxApplication/bean" diff --git a/pkg/terminal/terminalSesion.go b/pkg/terminal/terminalSesion.go index 7f720b770d..b2b5f2841a 100644 --- a/pkg/terminal/terminalSesion.go +++ b/pkg/terminal/terminalSesion.go @@ -26,13 +26,13 @@ import ( "github.com/caarlos0/env" "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/internal/middleware" + "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/cluster" "github.com/devtron-labs/devtron/pkg/cluster/bean" "github.com/devtron-labs/devtron/pkg/cluster/environment" bean2 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" "github.com/devtron-labs/devtron/pkg/cluster/read" "github.com/devtron-labs/devtron/pkg/cluster/repository" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read/config" errors1 "github.com/juju/errors" "go.uber.org/zap" "io" diff --git a/wire_gen.go b/wire_gen.go index f0051f606c..e397a3a3a3 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -130,6 +130,9 @@ import ( "github.com/devtron-labs/devtron/pkg/appStore/values/repository" service4 "github.com/devtron-labs/devtron/pkg/appStore/values/service" appWorkflow2 "github.com/devtron-labs/devtron/pkg/appWorkflow" + argoApplication3 "github.com/devtron-labs/devtron/pkg/argoApplication" + read22 "github.com/devtron-labs/devtron/pkg/argoApplication/read" + config2 "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/asyncProvider" "github.com/devtron-labs/devtron/pkg/attributes" "github.com/devtron-labs/devtron/pkg/auth/authentication" @@ -227,9 +230,6 @@ import ( "github.com/devtron-labs/devtron/pkg/module/repo" "github.com/devtron-labs/devtron/pkg/module/store" "github.com/devtron-labs/devtron/pkg/notifier" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" - read22 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read" - config2 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/history" @@ -824,9 +824,9 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - argoApplicationServiceImpl := argoApplication.NewArgoApplicationServiceImpl(sugaredLogger, clusterRepositoryImpl, k8sServiceImpl, helmAppClientImpl, helmAppServiceImpl, k8sApplicationServiceImpl, argoApplicationConfigServiceImpl, deploymentConfigServiceImpl) + argoApplicationServiceImpl := argoApplication3.NewArgoApplicationServiceImpl(sugaredLogger, clusterRepositoryImpl, k8sServiceImpl, helmAppClientImpl, helmAppServiceImpl, k8sApplicationServiceImpl, argoApplicationConfigServiceImpl, deploymentConfigServiceImpl) argoApplicationReadServiceImpl := read22.NewArgoApplicationReadServiceImpl(sugaredLogger, clusterRepositoryImpl, k8sServiceImpl, helmAppClientImpl, helmAppServiceImpl) - argoApplicationServiceExtendedImpl := argoApplication.NewArgoApplicationServiceExtendedServiceImpl(acdAuthConfig, argoApplicationServiceImpl, argoClientWrapperServiceImpl, argoApplicationReadServiceImpl, clusterServiceImplExtended) + argoApplicationServiceExtendedImpl := argoApplication3.NewArgoApplicationServiceExtendedServiceImpl(acdAuthConfig, argoApplicationServiceImpl, argoClientWrapperServiceImpl, argoApplicationReadServiceImpl, clusterServiceImplExtended) installedAppResourceServiceImpl := resource.NewInstalledAppResourceServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, argoClientWrapperServiceImpl, acdAuthConfig, installedAppVersionHistoryRepositoryImpl, helmAppServiceImpl, helmAppReadServiceImpl, appStatusServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, k8sServiceImpl, deploymentConfigServiceImpl, ociRegistryConfigRepositoryImpl, argoApplicationServiceExtendedImpl) chartGroupEntriesRepositoryImpl := repository28.NewChartGroupEntriesRepositoryImpl(db, sugaredLogger) chartGroupReposotoryImpl := repository28.NewChartGroupReposotoryImpl(db, sugaredLogger) From b54bb9efc677cf5be493a9adab904071a1497871 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Tue, 8 Apr 2025 00:40:25 +0530 Subject: [PATCH 29/37] chore: posthug refactoring and const renamed --- App.go | 6 +- Wire.go | 8 +- client/telemetry/TelemetryEventClient.go | 26 ++--- .../telemetry/TelemetryEventClientExtended.go | 10 +- cmd/external-app/externalApp.go | 6 +- cmd/external-app/wire.go | 4 +- cmd/external-app/wire_gen.go | 2 +- go.mod | 4 +- go.sum | 8 +- .../pipelineConfig/CdWorfkflowRepository.go | 1 - pkg/pipeline/CdHandler.go | 4 - pkg/pipeline/bean/WorkflowTemplate.go | 6 +- pkg/pipeline/executors/WorkflowUtils.go | 10 +- .../sql/32703300_workflow_message.down.sql | 1 + scripts/sql/32703300_workflow_message.up.sql | 11 +++ .../devtron-labs/common-lib/informer/bean.go | 2 +- .../common-lib/telemetry/PosthogClient.go | 99 +++++++++++++++++++ .../devtron-labs/common-lib/telemetry/bean.go | 34 +++++++ .../common-lib/utils/http/HttpUtil.go | 12 ++- vendor/modules.txt | 9 +- wire_gen.go | 8 +- 21 files changed, 208 insertions(+), 63 deletions(-) create mode 100644 scripts/sql/32703300_workflow_message.down.sql create mode 100644 scripts/sql/32703300_workflow_message.up.sql create mode 100644 vendor/github.com/devtron-labs/common-lib/telemetry/PosthogClient.go create mode 100644 vendor/github.com/devtron-labs/common-lib/telemetry/bean.go diff --git a/App.go b/App.go index 37c38ed9d2..649c319298 100644 --- a/App.go +++ b/App.go @@ -23,7 +23,7 @@ import ( "fmt" "github.com/devtron-labs/common-lib/middlewares" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" - "github.com/devtron-labs/devtron/client/telemetry/posthog" + posthogTelemetry "github.com/devtron-labs/common-lib/telemetry" "github.com/devtron-labs/devtron/pkg/eventProcessor" "github.com/devtron-labs/devtron/pkg/eventProcessor/in" "log" @@ -55,7 +55,7 @@ type App struct { EnforcerV2 *casbinv2.SyncedEnforcer server *http.Server db *pg.DB - posthogClient *posthog.PosthogClient + posthogClient *posthogTelemetry.PosthogClient // eventProcessor.CentralEventProcessor is used to register event processors centralEventProcessor *eventProcessor.CentralEventProcessor // do not remove this. // used for local dev only @@ -73,7 +73,7 @@ func NewApp(router *router.MuxRouter, enforcer *casbin.SyncedEnforcer, db *pg.DB, sessionManager2 *authMiddleware.SessionManager, - posthogClient *posthog.PosthogClient, + posthogClient *posthogTelemetry.PosthogClient, loggingMiddleware util.LoggingMiddleware, centralEventProcessor *eventProcessor.CentralEventProcessor, pubSubClient *pubsub.PubSubClientServiceImpl, diff --git a/Wire.go b/Wire.go index f7446f1573..d58d997949 100644 --- a/Wire.go +++ b/Wire.go @@ -22,7 +22,8 @@ package main import ( "github.com/devtron-labs/authenticator/middleware" cloudProviderIdentifier "github.com/devtron-labs/common-lib/cloud-provider-identifier" - pubsub1 "github.com/devtron-labs/common-lib/pubsub-lib" + pubSub "github.com/devtron-labs/common-lib/pubsub-lib" + posthogTelemetry "github.com/devtron-labs/common-lib/telemetry" util4 "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/api/apiToken" appStoreRestHandler "github.com/devtron-labs/devtron/api/appStore" @@ -97,7 +98,6 @@ import ( "github.com/devtron-labs/devtron/client/lens" "github.com/devtron-labs/devtron/client/proxy" "github.com/devtron-labs/devtron/client/telemetry" - "github.com/devtron-labs/devtron/client/telemetry/posthog" "github.com/devtron-labs/devtron/internal/sql/repository" app2 "github.com/devtron-labs/devtron/internal/sql/repository/app" appStatusRepo "github.com/devtron-labs/devtron/internal/sql/repository/appStatus" @@ -502,7 +502,7 @@ func InitializeApp() (*App, error) { pipeline.NewCiLogServiceImpl, wire.Bind(new(pipeline.CiLogService), new(*pipeline.CiLogServiceImpl)), - pubsub1.NewPubSubClientServiceImpl, + pubSub.NewPubSubClientServiceImpl, rbac.NewEnforcerUtilImpl, wire.Bind(new(rbac.EnforcerUtil), new(*rbac.EnforcerUtilImpl)), @@ -694,7 +694,7 @@ func InitializeApp() (*App, error) { wire.Bind(new(router.TelemetryRouter), new(*router.TelemetryRouterImpl)), restHandler.NewTelemetryRestHandlerImpl, wire.Bind(new(restHandler.TelemetryRestHandler), new(*restHandler.TelemetryRestHandlerImpl)), - posthog.NewPosthogClient, + posthogTelemetry.NewPosthogClient, ucid.WireSet, cloudProviderIdentifier.NewProviderIdentifierServiceImpl, diff --git a/client/telemetry/TelemetryEventClient.go b/client/telemetry/TelemetryEventClient.go index 7a4a697094..4a8f673ab7 100644 --- a/client/telemetry/TelemetryEventClient.go +++ b/client/telemetry/TelemetryEventClient.go @@ -22,9 +22,9 @@ import ( "encoding/json" "fmt" cloudProviderIdentifier "github.com/devtron-labs/common-lib/cloud-provider-identifier" + posthogTelemetry "github.com/devtron-labs/common-lib/telemetry" "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/api/helm-app/gRPC" - posthogClient "github.com/devtron-labs/devtron/client/telemetry/posthog" installedAppReader "github.com/devtron-labs/devtron/pkg/appStore/installedApp/read" bean2 "github.com/devtron-labs/devtron/pkg/attributes/bean" "github.com/devtron-labs/devtron/pkg/auth/user/bean" @@ -63,7 +63,7 @@ type TelemetryEventClientImpl struct { userService user2.UserService attributeRepo repository.AttributesRepository ssoLoginService sso.SSOLoginService - posthogClient *posthogClient.PosthogClient + posthogClient *posthogTelemetry.PosthogClient ucid ucidService.Service moduleRepository moduleRepo.ModuleRepository serverDataStore *serverDataStore.ServerDataStore @@ -88,7 +88,7 @@ type TelemetryEventClient interface { func NewTelemetryEventClientImpl(logger *zap.SugaredLogger, client *http.Client, clusterService cluster.ClusterService, K8sUtil *k8s.K8sServiceImpl, aCDAuthConfig *util3.ACDAuthConfig, userService user2.UserService, attributeRepo repository.AttributesRepository, ssoLoginService sso.SSOLoginService, - posthog *posthogClient.PosthogClient, ucid ucidService.Service, + posthog *posthogTelemetry.PosthogClient, ucid ucidService.Service, moduleRepository moduleRepo.ModuleRepository, serverDataStore *serverDataStore.ServerDataStore, userAuditService user2.UserAuditService, helmAppClient gRPC.HelmAppClient, cloudProviderIdentifierService cloudProviderIdentifier.ProviderIdentifierService, cronLogger *cron3.CronLoggerImpl, @@ -120,13 +120,13 @@ func NewTelemetryEventClientImpl(logger *zap.SugaredLogger, client *http.Client, } watcher.HeartbeatEventForTelemetry() - _, err := cron.AddFunc(posthogClient.SummaryCronExpr, watcher.SummaryEventForTelemetryEA) + _, err := cron.AddFunc(posthogTelemetry.SummaryCronExpr, watcher.SummaryEventForTelemetryEA) if err != nil { logger.Errorw("error in starting summery event", "err", err) return nil, err } - _, err = cron.AddFunc(posthogClient.HeartbeatCronExpr, watcher.HeartbeatEventForTelemetry) + _, err = cron.AddFunc(posthogTelemetry.HeartbeatCronExpr, watcher.HeartbeatEventForTelemetry) if err != nil { logger.Errorw("error in starting heartbeat event", "err", err) return nil, err @@ -275,7 +275,7 @@ func (impl *TelemetryEventClientImpl) SendSummaryEvent(eventType string) error { return err } - if posthogClient.IsOptOut { + if posthogTelemetry.IsOptOut { impl.logger.Warnw("client is opt-out for telemetry, there will be no events capture", "ucid", ucid) return err } @@ -353,7 +353,7 @@ func (impl *TelemetryEventClientImpl) SendGenericTelemetryEvent(eventType string return nil } - if posthogClient.IsOptOut { + if posthogTelemetry.IsOptOut { impl.logger.Warnw("client is opt-out for telemetry, there will be no events capture", "ucid", ucid) return nil } @@ -364,7 +364,7 @@ func (impl *TelemetryEventClientImpl) SendGenericTelemetryEvent(eventType string func (impl *TelemetryEventClientImpl) EnqueueGenericPostHogEvent(ucid string, eventType string, prop map[string]interface{}) error { if impl.posthogClient.Client == nil { impl.logger.Warn("no posthog client found, creating new") - client, err := impl.retryPosthogClient(posthogClient.PosthogApiKey, posthogClient.PosthogEndpoint) + client, err := impl.retryPosthogClient(posthogTelemetry.PosthogApiKey, posthogTelemetry.PosthogEndpoint) if err == nil { impl.posthogClient.Client = client } @@ -389,7 +389,7 @@ func (impl *TelemetryEventClientImpl) HeartbeatEventForTelemetry() { impl.logger.Errorw("exception caught inside telemetry heartbeat event", "err", err) return } - if posthogClient.IsOptOut { + if posthogTelemetry.IsOptOut { impl.logger.Warnw("client is opt-out for telemetry, there will be no events capture", "ucid", ucid) return } @@ -437,9 +437,9 @@ func (impl *TelemetryEventClientImpl) GetTelemetryMetaInfo() (*TelemetryMetaInfo return nil, err } data := &TelemetryMetaInfo{ - Url: posthogClient.PosthogEndpoint, + Url: posthogTelemetry.PosthogEndpoint, UCID: ucid, - ApiKey: posthogClient.PosthogEncodedApiKey, + ApiKey: posthogTelemetry.PosthogEncodedApiKey, } return data, err } @@ -668,7 +668,7 @@ func (impl *TelemetryEventClientImpl) getUCIDAndCheckIsOptedOut(ctx context.Cont impl.logger.Errorw("error sending event to posthog, failed check for opt-out", "err", err) return "", err } - posthogClient.IsOptOut = flag + posthogTelemetry.IsOptOut = flag } return ucid, nil } @@ -676,7 +676,7 @@ func (impl *TelemetryEventClientImpl) getUCIDAndCheckIsOptedOut(ctx context.Cont func (impl *TelemetryEventClientImpl) checkForOptOut(ctx context.Context, UCID string) (bool, error) { newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TelemetryEventClientImpl.checkForOptOut") defer span.End() - decodedUrl, err := base64.StdEncoding.DecodeString(posthogClient.TelemetryOptOutApiBaseUrl) + decodedUrl, err := base64.StdEncoding.DecodeString(posthogTelemetry.TelemetryOptOutApiBaseUrl) if err != nil { impl.logger.Errorw("check opt-out list failed, decode error", "err", err) return false, err diff --git a/client/telemetry/TelemetryEventClientExtended.go b/client/telemetry/TelemetryEventClientExtended.go index 1670a6b4db..d5b309b6f6 100644 --- a/client/telemetry/TelemetryEventClientExtended.go +++ b/client/telemetry/TelemetryEventClientExtended.go @@ -20,9 +20,9 @@ import ( "context" "encoding/json" cloudProviderIdentifier "github.com/devtron-labs/common-lib/cloud-provider-identifier" + posthogTelemetry "github.com/devtron-labs/common-lib/telemetry" util2 "github.com/devtron-labs/common-lib/utils/k8s" client "github.com/devtron-labs/devtron/api/helm-app/gRPC" - posthogClient "github.com/devtron-labs/devtron/client/telemetry/posthog" installedAppReader "github.com/devtron-labs/devtron/pkg/appStore/installedApp/read" "github.com/devtron-labs/devtron/pkg/auth/sso" user2 "github.com/devtron-labs/devtron/pkg/auth/user" @@ -74,7 +74,7 @@ type TelemetryEventClientImplExtended struct { func NewTelemetryEventClientImplExtended(logger *zap.SugaredLogger, client *http.Client, clusterService cluster.ClusterService, K8sUtil *util2.K8sServiceImpl, aCDAuthConfig *util3.ACDAuthConfig, environmentService environment.EnvironmentService, userService user2.UserService, - appListingRepository repository.AppListingRepository, posthog *posthogClient.PosthogClient, ucid ucidService.Service, + appListingRepository repository.AppListingRepository, posthog *posthogTelemetry.PosthogClient, ucid ucidService.Service, ciPipelineConfigReadService ciConfig.CiPipelineConfigReadService, pipelineRepository pipelineConfig.PipelineRepository, gitProviderRepository repository3.GitProviderRepository, attributeRepo repository.AttributesRepository, ssoLoginService sso.SSOLoginService, appRepository app.AppRepository, @@ -129,13 +129,13 @@ func NewTelemetryEventClientImplExtended(logger *zap.SugaredLogger, client *http } watcher.HeartbeatEventForTelemetry() - _, err := cron.AddFunc(posthogClient.SummaryCronExpr, watcher.SummaryEventForTelemetry) + _, err := cron.AddFunc(posthogTelemetry.SummaryCronExpr, watcher.SummaryEventForTelemetry) if err != nil { logger.Errorw("error in starting summery event", "err", err) return nil, err } - _, err = cron.AddFunc(posthogClient.HeartbeatCronExpr, watcher.HeartbeatEventForTelemetry) + _, err = cron.AddFunc(posthogTelemetry.HeartbeatCronExpr, watcher.HeartbeatEventForTelemetry) if err != nil { logger.Errorw("error in starting heartbeat event", "err", err) return nil, err @@ -158,7 +158,7 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string) return err } - if posthogClient.IsOptOut { + if posthogTelemetry.IsOptOut { impl.logger.Warnw("client is opt-out for telemetry, there will be no events capture", "ucid", ucid) return err } diff --git a/cmd/external-app/externalApp.go b/cmd/external-app/externalApp.go index 8d5ee62cb7..dba06f4eea 100644 --- a/cmd/external-app/externalApp.go +++ b/cmd/external-app/externalApp.go @@ -20,7 +20,7 @@ import ( "context" "errors" "fmt" - "github.com/devtron-labs/devtron/client/telemetry/posthog" + posthogTelemetry "github.com/devtron-labs/common-lib/telemetry" "net/http" "os" "time" @@ -41,14 +41,14 @@ type App struct { Logger *zap.SugaredLogger server *http.Server telemetry telemetry.TelemetryEventClient - posthogClient *posthog.PosthogClient + posthogClient *posthogTelemetry.PosthogClient } func NewApp(db *pg.DB, sessionManager *authMiddleware.SessionManager, MuxRouter *MuxRouter, telemetry telemetry.TelemetryEventClient, - posthogClient *posthog.PosthogClient, + posthogClient *posthogTelemetry.PosthogClient, Logger *zap.SugaredLogger) *App { return &App{ db: db, diff --git a/cmd/external-app/wire.go b/cmd/external-app/wire.go index 4dc491cd7f..31a15cbf8a 100644 --- a/cmd/external-app/wire.go +++ b/cmd/external-app/wire.go @@ -22,6 +22,7 @@ package main import ( "github.com/devtron-labs/authenticator/middleware" cloudProviderIdentifier "github.com/devtron-labs/common-lib/cloud-provider-identifier" + posthogTelemetry "github.com/devtron-labs/common-lib/telemetry" util4 "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/api/apiToken" "github.com/devtron-labs/devtron/api/appStore" @@ -60,7 +61,6 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer/session" "github.com/devtron-labs/devtron/client/dashboard" "github.com/devtron-labs/devtron/client/telemetry" - "github.com/devtron-labs/devtron/client/telemetry/posthog" "github.com/devtron-labs/devtron/internal/sql/repository" app2 "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/appStatus" @@ -133,7 +133,7 @@ func InitializeApp() (*App, error) { util.NewSugardLogger, util.IntValidator, util2.GetACDAuthConfig, - posthog.NewPosthogClient, + posthogTelemetry.NewPosthogClient, ucid.WireSet, delete2.NewDeleteServiceImpl, gitMaterial.GitMaterialWireSet, diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index f1d6763cc8..0190d93983 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -52,7 +52,7 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer/repoCredsK8sClient" "github.com/devtron-labs/devtron/client/dashboard" "github.com/devtron-labs/devtron/client/telemetry" - "github.com/devtron-labs/devtron/client/telemetry/posthog" + posthog "github.com/devtron-labs/common-lib/telemetry" repository5 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/appStatus" diff --git a/go.mod b/go.mod index aa0d1cdcd6..7631bd384e 100644 --- a/go.mod +++ b/go.mod @@ -307,8 +307,8 @@ require ( replace ( github.com/argoproj/argo-workflows/v3 => github.com/devtron-labs/argo-workflows/v3 v3.5.13 - github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250403171407-721f466f49ce - github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250403171407-721f466f49ce + github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250407184841-36bb51a7d2fa + github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250407184841-36bb51a7d2fa github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127 github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5 k8s.io/api => k8s.io/api v0.29.7 diff --git a/go.sum b/go.sum index 2a9b9bb3e6..22e91ce7b6 100644 --- a/go.sum +++ b/go.sum @@ -829,10 +829,10 @@ github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzq github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/devtron-labs/argo-workflows/v3 v3.5.13 h1:3pINq0gXOSeTw2z/vYe+j80lRpSN5Rp/8mfQORh8SmU= github.com/devtron-labs/argo-workflows/v3 v3.5.13/go.mod h1:/vqxcovDPT4zqr4DjR5v7CF8ggpY1l3TSa2CIG3jmjA= -github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250403171407-721f466f49ce h1:A4+SH98TFhHo/cUWi9CE3rbw0TjU6uH7cfOFOsmQrqk= -github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250403171407-721f466f49ce/go.mod h1:FfaLDXN1ZXxyRpnskBqVIYkpkWDCzBmDgIO9xqLnxdQ= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250403171407-721f466f49ce h1:hp5p/mPLsoCyqM51b88Uu8AgCTn0K2bwKEsB5Fy2C24= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250403171407-721f466f49ce/go.mod h1:zkNShlkcHxsmnL0gKNbs0uyRL8lZonGKr5Km63uTLI0= +github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250407184841-36bb51a7d2fa h1:NsMmPjQXq+Adr4mS22rTikyS3RhP8ARNgNVR9zFWD74= +github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250407184841-36bb51a7d2fa/go.mod h1:FfaLDXN1ZXxyRpnskBqVIYkpkWDCzBmDgIO9xqLnxdQ= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250407184841-36bb51a7d2fa h1:vrNxj/SMaypXKcIQqoN1tOXZdIeYi1mz6Htp5MPnASw= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250407184841-36bb51a7d2fa/go.mod h1:zkNShlkcHxsmnL0gKNbs0uyRL8lZonGKr5Km63uTLI0= github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU= github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y= github.com/devtron-labs/protos v0.0.3-0.20250323220609-ecf8a0f7305e h1:U6UdYbW8a7xn5IzFPd8cywjVVPfutGJCudjePAfL/Hs= diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index 9aa3eeaa28..6d7a9455ba 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -458,7 +458,6 @@ func (impl *CdWorkflowRepositoryImpl) SaveWorkFlowRunnerWithTx(wfr *CdWorkflowRu } func (impl *CdWorkflowRepositoryImpl) UpdateWorkFlowRunnerWithTx(wfr *CdWorkflowRunner, tx *pg.Tx) error { - wfr.Message = util.GetTruncatedMessage(wfr.Message, 1000) err := tx.Update(wfr) return err } diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 4889e361b6..f277f3de72 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -291,10 +291,6 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus eventProcessorBean.CiCd if impl.stateChanged(status, podStatus, message, workflowStatus.FinishedAt.Time, savedWorkflow) { if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.PodStatus) { savedWorkflow.Message = message - // NOTE: we are doing this to fix where a pre-cd / post-cd workflow message becomes larger than 1000, and in db we had set the charter limit to 1000 - if len(message) > 1000 { - savedWorkflow.Message = message[:1000] - } if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.Status) { savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time } diff --git a/pkg/pipeline/bean/WorkflowTemplate.go b/pkg/pipeline/bean/WorkflowTemplate.go index abc8be4588..dcab419b5e 100644 --- a/pkg/pipeline/bean/WorkflowTemplate.go +++ b/pkg/pipeline/bean/WorkflowTemplate.go @@ -81,9 +81,9 @@ func (workflowTemplate *WorkflowTemplate) SetActiveDeadlineSeconds(timeout int64 func (workflowTemplate *WorkflowTemplate) CreateObjectMetadata() *v12.ObjectMeta { workflowLabels := map[string]string{ - WorkflowGenerateNamePrefix: workflowTemplate.WorkflowNamePrefix, - informerBean.WorkflowTypeLabelKey: workflowTemplate.WorkflowType, - informerBean.DevtronAdministratorInstanceLabelKey: workflowTemplate.DevtronInstanceUID, + WorkflowGenerateNamePrefix: workflowTemplate.WorkflowNamePrefix, + informerBean.WorkflowTypeLabelKey: workflowTemplate.WorkflowType, + informerBean.DevtronOwnerInstanceLabelKey: workflowTemplate.DevtronInstanceUID, } switch workflowTemplate.WorkflowType { case CI_WORKFLOW_NAME: diff --git a/pkg/pipeline/executors/WorkflowUtils.go b/pkg/pipeline/executors/WorkflowUtils.go index c08bfd8efd..bbf7439660 100644 --- a/pkg/pipeline/executors/WorkflowUtils.go +++ b/pkg/pipeline/executors/WorkflowUtils.go @@ -298,10 +298,10 @@ func CheckIfReTriggerRequired(status, message, workflowRunnerStatus string) bool func GetWorkflowLabelsForSystemExecutor(workflowTemplate bean.WorkflowTemplate) map[string]string { return map[string]string{ - DEVTRON_WORKFLOW_LABEL_KEY: DEVTRON_WORKFLOW_LABEL_VALUE, - bean.DevtronLabelPurposeKey: bean.DevtronLabelPurposeWorkflow, - informerBean.WorkflowTypeLabelKey: workflowTemplate.WorkflowType, - bean.WorkflowGenerateNamePrefix: workflowTemplate.WorkflowNamePrefix, - informerBean.DevtronAdministratorInstanceLabelKey: workflowTemplate.DevtronInstanceUID, + DEVTRON_WORKFLOW_LABEL_KEY: DEVTRON_WORKFLOW_LABEL_VALUE, + bean.DevtronLabelPurposeKey: bean.DevtronLabelPurposeWorkflow, + informerBean.WorkflowTypeLabelKey: workflowTemplate.WorkflowType, + bean.WorkflowGenerateNamePrefix: workflowTemplate.WorkflowNamePrefix, + informerBean.DevtronOwnerInstanceLabelKey: workflowTemplate.DevtronInstanceUID, } } diff --git a/scripts/sql/32703300_workflow_message.down.sql b/scripts/sql/32703300_workflow_message.down.sql new file mode 100644 index 0000000000..7b794d76d6 --- /dev/null +++ b/scripts/sql/32703300_workflow_message.down.sql @@ -0,0 +1 @@ +-- Migration down is not applicable here as cd_workflow_runner.message has moved to higher DataType \ No newline at end of file diff --git a/scripts/sql/32703300_workflow_message.up.sql b/scripts/sql/32703300_workflow_message.up.sql new file mode 100644 index 0000000000..bc3083ee77 --- /dev/null +++ b/scripts/sql/32703300_workflow_message.up.sql @@ -0,0 +1,11 @@ +BEGIN; + +-- cd_workflow_runner.message has a limit of 1000 characters. migrating to +ALTER TABLE "public"."cd_workflow_runner" + ALTER COLUMN "message" TYPE TEXT; + +-- ci_workflow.message has a limit of 250 characters. migrating to +ALTER TABLE "public"."ci_workflow" + ALTER COLUMN "message" TYPE TEXT; + +COMMIT; \ No newline at end of file diff --git a/vendor/github.com/devtron-labs/common-lib/informer/bean.go b/vendor/github.com/devtron-labs/common-lib/informer/bean.go index e3b0304328..38aed85afc 100644 --- a/vendor/github.com/devtron-labs/common-lib/informer/bean.go +++ b/vendor/github.com/devtron-labs/common-lib/informer/bean.go @@ -32,7 +32,7 @@ const ( ) const ( - DevtronAdministratorInstanceLabelKey = "devtron.ai/administrator-instance" + DevtronOwnerInstanceLabelKey = "devtron.ai/owner-instance" ) const ( diff --git a/vendor/github.com/devtron-labs/common-lib/telemetry/PosthogClient.go b/vendor/github.com/devtron-labs/common-lib/telemetry/PosthogClient.go new file mode 100644 index 0000000000..79a8c72f88 --- /dev/null +++ b/vendor/github.com/devtron-labs/common-lib/telemetry/PosthogClient.go @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2020-2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package telemetry + +import ( + "context" + "encoding/base64" + "encoding/json" + "go.opentelemetry.io/otel" + "time" + + "github.com/devtron-labs/common-lib/utils/http" + "github.com/patrickmn/go-cache" + "github.com/posthog/posthog-go" + "go.uber.org/zap" +) + +type PosthogClient struct { + Client posthog.Client + cache *cache.Cache +} + +func (p *PosthogClient) GetCache() *cache.Cache { + return p.cache +} + +func NewPosthogClient(logger *zap.SugaredLogger) (*PosthogClient, error) { + if PosthogApiKey == "" { + encodedApiKey, apiKey, posthogUrl, err := getPosthogApiKey(context.Background(), TelemetryApiKeyEndpoint, logger) + if err != nil { + logger.Errorw("exception caught while getting api key", "err", err) + } else { + PosthogApiKey = apiKey + PosthogEncodedApiKey = encodedApiKey + PosthogEndpoint = posthogUrl + } + } + + client, err := posthog.NewWithConfig(PosthogApiKey, posthog.Config{Endpoint: PosthogEndpoint}) + //defer client.Close() + if err != nil { + logger.Errorw("exception caught while creating posthog client", "err", err) + } + d := time.Duration(CacheExpiry) + c := cache.New(d*time.Minute, d*time.Minute) + pgClient := &PosthogClient{ + Client: client, + cache: c, + } + return pgClient, nil +} + +func getPosthogApiKey(ctx context.Context, encodedPosthogApiKeyUrl string, logger *zap.SugaredLogger) (string, string, string, error) { + newCtx, span := otel.Tracer("common").Start(ctx, "telemetry.getPosthogApiKey") + defer span.End() + decodedPosthogApiKeyUrl, err := base64.StdEncoding.DecodeString(encodedPosthogApiKeyUrl) + if err != nil { + logger.Errorw("error fetching posthog api key, decode error", "err", err) + return "", "", "", err + } + apiKeyUrl := string(decodedPosthogApiKeyUrl) + response, err := http.HttpRequest(newCtx, apiKeyUrl) + if err != nil { + logger.Errorw("error fetching posthog api key, http call", "err", err) + return "", "", "", err + } + posthogInfo := response["result"] + posthogInfoByte, err := json.Marshal(posthogInfo) + if err != nil { + logger.Errorw("error in fetched posthog info, http call", "err", err) + return "", "", "", err + } + var datamap map[string]string + if err = json.Unmarshal(posthogInfoByte, &datamap); err != nil { + logger.Errorw("error while unmarshal data", "err", err) + return "", "", "", err + } + encodedApiKey := datamap[ResponseApiKey] + posthogUrl := datamap[ResponseUrlKey] + apiKey, err := base64.StdEncoding.DecodeString(encodedApiKey) + if err != nil { + return "", "", "", err + } + return encodedApiKey, string(apiKey), posthogUrl, err +} diff --git a/vendor/github.com/devtron-labs/common-lib/telemetry/bean.go b/vendor/github.com/devtron-labs/common-lib/telemetry/bean.go new file mode 100644 index 0000000000..b85621d0f3 --- /dev/null +++ b/vendor/github.com/devtron-labs/common-lib/telemetry/bean.go @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package telemetry + +var ( + PosthogApiKey string = "" + PosthogEndpoint string = "https://app.posthog.com" + SummaryCronExpr string = "0 0 * * *" // Run once a day, midnight + HeartbeatCronExpr string = "0 0/6 * * *" + CacheExpiry int = 1440 + PosthogEncodedApiKey string = "" + IsOptOut bool = false +) + +const ( + TelemetryApiKeyEndpoint string = "aHR0cHM6Ly90ZWxlbWV0cnkuZGV2dHJvbi5haS9kZXZ0cm9uL3RlbGVtZXRyeS9wb3N0aG9nSW5mbw==" + TelemetryOptOutApiBaseUrl string = "aHR0cHM6Ly90ZWxlbWV0cnkuZGV2dHJvbi5haS9kZXZ0cm9uL3RlbGVtZXRyeS9vcHQtb3V0" + ResponseApiKey string = "PosthogApiKey" + ResponseUrlKey string = "PosthogEndpoint" +) diff --git a/vendor/github.com/devtron-labs/common-lib/utils/http/HttpUtil.go b/vendor/github.com/devtron-labs/common-lib/utils/http/HttpUtil.go index 933b97f4f9..9f8524e489 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/http/HttpUtil.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/http/HttpUtil.go @@ -17,11 +17,13 @@ package http import ( + "context" "crypto/tls" "crypto/x509" "encoding/json" "github.com/pkg/errors" - "io/ioutil" + "go.opentelemetry.io/otel" + "io" "net/http" "os" ) @@ -83,8 +85,10 @@ func CertPoolFromFile(filename string) (*x509.CertPool, error) { return cp, nil } -func HttpRequest(url string) (map[string]interface{}, error) { - req, err := http.NewRequest(http.MethodGet, url, nil) +func HttpRequest(ctx context.Context, url string) (map[string]interface{}, error) { + newCtx, span := otel.Tracer("common").Start(ctx, "http.HttpRequest") + defer span.End() + req, err := http.NewRequestWithContext(newCtx, http.MethodGet, url, nil) if err != nil { return nil, err } @@ -95,7 +99,7 @@ func HttpRequest(url string) (map[string]interface{}, error) { return nil, err } if res.StatusCode >= 200 && res.StatusCode <= 299 { - resBody, err := ioutil.ReadAll(res.Body) + resBody, err := io.ReadAll(res.Body) if err != nil { return nil, err } diff --git a/vendor/modules.txt b/vendor/modules.txt index ae3dc39f9b..2e26f2c11c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -472,7 +472,7 @@ github.com/davecgh/go-spew/spew # github.com/deckarep/golang-set v1.8.0 ## explicit; go 1.17 github.com/deckarep/golang-set -# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250403171407-721f466f49ce +# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250407184841-36bb51a7d2fa ## explicit; go 1.21 github.com/devtron-labs/authenticator/apiToken github.com/devtron-labs/authenticator/client @@ -480,7 +480,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250403171407-721f466f49ce +# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250407184841-36bb51a7d2fa ## explicit; go 1.21 github.com/devtron-labs/common-lib/async github.com/devtron-labs/common-lib/blob-storage @@ -497,6 +497,7 @@ github.com/devtron-labs/common-lib/middlewares github.com/devtron-labs/common-lib/pubsub-lib github.com/devtron-labs/common-lib/pubsub-lib/metrics github.com/devtron-labs/common-lib/pubsub-lib/model +github.com/devtron-labs/common-lib/telemetry github.com/devtron-labs/common-lib/utils github.com/devtron-labs/common-lib/utils/bean github.com/devtron-labs/common-lib/utils/grpc @@ -2352,8 +2353,8 @@ xorm.io/xorm/names xorm.io/xorm/schemas xorm.io/xorm/tags # github.com/argoproj/argo-workflows/v3 => github.com/devtron-labs/argo-workflows/v3 v3.5.13 -# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250403171407-721f466f49ce -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250403171407-721f466f49ce +# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250407184841-36bb51a7d2fa +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250407184841-36bb51a7d2fa # github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127 # github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5 # k8s.io/api => k8s.io/api v0.29.7 diff --git a/wire_gen.go b/wire_gen.go index b9702bc9da..d5261faa8e 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -12,6 +12,7 @@ import ( "github.com/devtron-labs/authenticator/middleware" "github.com/devtron-labs/common-lib/cloud-provider-identifier" "github.com/devtron-labs/common-lib/pubsub-lib" + "github.com/devtron-labs/common-lib/telemetry" "github.com/devtron-labs/common-lib/utils/grpc" "github.com/devtron-labs/common-lib/utils/k8s" apiToken2 "github.com/devtron-labs/devtron/api/apiToken" @@ -90,8 +91,7 @@ import ( "github.com/devtron-labs/devtron/client/grafana" "github.com/devtron-labs/devtron/client/lens" "github.com/devtron-labs/devtron/client/proxy" - "github.com/devtron-labs/devtron/client/telemetry" - "github.com/devtron-labs/devtron/client/telemetry/posthog" + telemetry2 "github.com/devtron-labs/devtron/client/telemetry" repository2 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/appStatus" @@ -929,12 +929,12 @@ func InitializeApp() (*App, error) { ssoLoginServiceImpl := sso.NewSSOLoginServiceImpl(sugaredLogger, ssoLoginRepositoryImpl, k8sServiceImpl, environmentVariables, userAuthOidcHelperImpl) ssoLoginRestHandlerImpl := sso2.NewSsoLoginRestHandlerImpl(validate, sugaredLogger, enforcerImpl, userServiceImpl, ssoLoginServiceImpl) ssoLoginRouterImpl := sso2.NewSsoLoginRouterImpl(ssoLoginRestHandlerImpl) - posthogClient, err := posthog.NewPosthogClient(sugaredLogger) + posthogClient, err := telemetry.NewPosthogClient(sugaredLogger) if err != nil { return nil, err } providerIdentifierServiceImpl := providerIdentifier.NewProviderIdentifierServiceImpl(sugaredLogger) - telemetryEventClientImplExtended, err := telemetry.NewTelemetryEventClientImplExtended(sugaredLogger, httpClient, clusterServiceImplExtended, k8sServiceImpl, acdAuthConfig, environmentServiceImpl, userServiceImpl, appListingRepositoryImpl, posthogClient, serviceImpl, ciPipelineConfigReadServiceImpl, pipelineRepositoryImpl, gitProviderRepositoryImpl, attributesRepositoryImpl, ssoLoginServiceImpl, appRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, gitMaterialReadServiceImpl, ciTemplateRepositoryImpl, chartRepositoryImpl, userAuditServiceImpl, ciBuildConfigServiceImpl, moduleRepositoryImpl, serverDataStoreServerDataStore, helmAppClientImpl, installedAppReadServiceImpl, userAttributesRepositoryImpl, providerIdentifierServiceImpl, cronLoggerImpl, gitOpsConfigReadServiceImpl) + telemetryEventClientImplExtended, err := telemetry2.NewTelemetryEventClientImplExtended(sugaredLogger, httpClient, clusterServiceImplExtended, k8sServiceImpl, acdAuthConfig, environmentServiceImpl, userServiceImpl, appListingRepositoryImpl, posthogClient, serviceImpl, ciPipelineConfigReadServiceImpl, pipelineRepositoryImpl, gitProviderRepositoryImpl, attributesRepositoryImpl, ssoLoginServiceImpl, appRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, gitMaterialReadServiceImpl, ciTemplateRepositoryImpl, chartRepositoryImpl, userAuditServiceImpl, ciBuildConfigServiceImpl, moduleRepositoryImpl, serverDataStoreServerDataStore, helmAppClientImpl, installedAppReadServiceImpl, userAttributesRepositoryImpl, providerIdentifierServiceImpl, cronLoggerImpl, gitOpsConfigReadServiceImpl) if err != nil { return nil, err } From ae11bb21674b1da0b90a5daf5c1332b5d1f51deb Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Tue, 8 Apr 2025 00:45:56 +0530 Subject: [PATCH 30/37] chore: kubewatch const renamed --- pkg/eventProcessor/bean/workflowEventBean.go | 2 +- .../in/WorkflowEventProcessorService.go | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/eventProcessor/bean/workflowEventBean.go b/pkg/eventProcessor/bean/workflowEventBean.go index 3f401845c4..ad150af539 100644 --- a/pkg/eventProcessor/bean/workflowEventBean.go +++ b/pkg/eventProcessor/bean/workflowEventBean.go @@ -117,7 +117,7 @@ type DevtronAppReleaseContextType struct { } type CiCdStatus struct { - DevtronAdministratorInstance string `json:"devtronAdministratorInstance"` + DevtronOwnerInstance string `json:"devtronOwnerInstance"` *v1alpha1.WorkflowStatus } diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index 9816c31920..fd90d10631 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -398,14 +398,14 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCIWorkflowStatusUpdate() error impl.logger.Errorw("error while unmarshalling wf status update", "err", err, "msg", msg.Data) return } - if len(wfStatus.DevtronAdministratorInstance) != 0 { + if len(wfStatus.DevtronOwnerInstance) != 0 { devtronUCID, _, err := impl.ucid.GetUCIDWithOutCache() if err != nil { impl.logger.Errorw("error in getting UCID", "err", err) return } - if wfStatus.DevtronAdministratorInstance != devtronUCID { - impl.logger.Warnw("mis match in UCID. skipping...", "devtronAdministratorInstance", wfStatus.DevtronAdministratorInstance, "devtronUCID", devtronUCID) + if wfStatus.DevtronOwnerInstance != devtronUCID { + impl.logger.Warnw("mis match in UCID. skipping...", "devtronAdministratorInstance", wfStatus.DevtronOwnerInstance, "devtronUCID", devtronUCID) return } } @@ -457,14 +457,14 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error impl.logger.Error("Error while unmarshalling wfStatus json object", "error", err) return } - if len(wfStatus.DevtronAdministratorInstance) != 0 { + if len(wfStatus.DevtronOwnerInstance) != 0 { devtronUCID, _, err := impl.ucid.GetUCIDWithOutCache() if err != nil { impl.logger.Errorw("error in getting UCID", "err", err) return } - if wfStatus.DevtronAdministratorInstance != devtronUCID { - impl.logger.Warnw("mis match in UCID. skipping...", "devtronAdministratorInstance", wfStatus.DevtronAdministratorInstance, "devtronUCID", devtronUCID) + if wfStatus.DevtronOwnerInstance != devtronUCID { + impl.logger.Warnw("mis match in UCID. skipping...", "devtronAdministratorInstance", wfStatus.DevtronOwnerInstance, "devtronUCID", devtronUCID) return } } From 13a26e15dc973dc72222a533b01cd5ce21e30dd6 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Tue, 8 Apr 2025 01:07:26 +0530 Subject: [PATCH 31/37] chore: removed posthug module --- client/telemetry/posthog/PosthogClient.go | 99 ----------------------- client/telemetry/posthog/bean.go | 34 -------- cmd/external-app/wire_gen.go | 10 +-- 3 files changed, 5 insertions(+), 138 deletions(-) delete mode 100644 client/telemetry/posthog/PosthogClient.go delete mode 100644 client/telemetry/posthog/bean.go diff --git a/client/telemetry/posthog/PosthogClient.go b/client/telemetry/posthog/PosthogClient.go deleted file mode 100644 index 7e6213864f..0000000000 --- a/client/telemetry/posthog/PosthogClient.go +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (c) 2020-2024. Devtron Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package posthog - -import ( - "context" - "encoding/base64" - "encoding/json" - "go.opentelemetry.io/otel" - "time" - - "github.com/devtron-labs/devtron/util" - "github.com/patrickmn/go-cache" - "github.com/posthog/posthog-go" - "go.uber.org/zap" -) - -type PosthogClient struct { - Client posthog.Client - cache *cache.Cache -} - -func (p *PosthogClient) GetCache() *cache.Cache { - return p.cache -} - -func NewPosthogClient(logger *zap.SugaredLogger) (*PosthogClient, error) { - if PosthogApiKey == "" { - encodedApiKey, apiKey, posthogUrl, err := getPosthogApiKey(context.Background(), TelemetryApiKeyEndpoint, logger) - if err != nil { - logger.Errorw("exception caught while getting api key", "err", err) - } else { - PosthogApiKey = apiKey - PosthogEncodedApiKey = encodedApiKey - PosthogEndpoint = posthogUrl - } - } - - client, err := posthog.NewWithConfig(PosthogApiKey, posthog.Config{Endpoint: PosthogEndpoint}) - //defer client.Close() - if err != nil { - logger.Errorw("exception caught while creating posthog client", "err", err) - } - d := time.Duration(CacheExpiry) - c := cache.New(d*time.Minute, d*time.Minute) - pgClient := &PosthogClient{ - Client: client, - cache: c, - } - return pgClient, nil -} - -func getPosthogApiKey(ctx context.Context, encodedPosthogApiKeyUrl string, logger *zap.SugaredLogger) (string, string, string, error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "telemetry.getPosthogApiKey") - defer span.End() - decodedPosthogApiKeyUrl, err := base64.StdEncoding.DecodeString(encodedPosthogApiKeyUrl) - if err != nil { - logger.Errorw("error fetching posthog api key, decode error", "err", err) - return "", "", "", err - } - apiKeyUrl := string(decodedPosthogApiKeyUrl) - response, err := util.HttpRequest(newCtx, apiKeyUrl) - if err != nil { - logger.Errorw("error fetching posthog api key, http call", "err", err) - return "", "", "", err - } - posthogInfo := response["result"] - posthogInfoByte, err := json.Marshal(posthogInfo) - if err != nil { - logger.Errorw("error in fetched posthog info, http call", "err", err) - return "", "", "", err - } - var datamap map[string]string - if err = json.Unmarshal(posthogInfoByte, &datamap); err != nil { - logger.Errorw("error while unmarshal data", "err", err) - return "", "", "", err - } - encodedApiKey := datamap[ResponseApiKey] - posthogUrl := datamap[ResponseUrlKey] - apiKey, err := base64.StdEncoding.DecodeString(encodedApiKey) - if err != nil { - return "", "", "", err - } - return encodedApiKey, string(apiKey), posthogUrl, err -} diff --git a/client/telemetry/posthog/bean.go b/client/telemetry/posthog/bean.go deleted file mode 100644 index 8910b7d8fe..0000000000 --- a/client/telemetry/posthog/bean.go +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2024. Devtron Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package posthog - -var ( - PosthogApiKey string = "" - PosthogEndpoint string = "https://app.posthog.com" - SummaryCronExpr string = "0 0 * * *" // Run once a day, midnight - HeartbeatCronExpr string = "0 0/6 * * *" - CacheExpiry int = 1440 - PosthogEncodedApiKey string = "" - IsOptOut bool = false -) - -const ( - TelemetryApiKeyEndpoint string = "aHR0cHM6Ly90ZWxlbWV0cnkuZGV2dHJvbi5haS9kZXZ0cm9uL3RlbGVtZXRyeS9wb3N0aG9nSW5mbw==" - TelemetryOptOutApiBaseUrl string = "aHR0cHM6Ly90ZWxlbWV0cnkuZGV2dHJvbi5haS9kZXZ0cm9uL3RlbGVtZXRyeS9vcHQtb3V0" - ResponseApiKey string = "PosthogApiKey" - ResponseUrlKey string = "PosthogEndpoint" -) diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index 0190d93983..14934dea14 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -11,6 +11,7 @@ import ( "github.com/devtron-labs/authenticator/client" "github.com/devtron-labs/authenticator/middleware" "github.com/devtron-labs/common-lib/cloud-provider-identifier" + "github.com/devtron-labs/common-lib/telemetry" "github.com/devtron-labs/common-lib/utils/grpc" "github.com/devtron-labs/common-lib/utils/k8s" apiToken2 "github.com/devtron-labs/devtron/api/apiToken" @@ -51,8 +52,7 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer/config" "github.com/devtron-labs/devtron/client/argocdServer/repoCredsK8sClient" "github.com/devtron-labs/devtron/client/dashboard" - "github.com/devtron-labs/devtron/client/telemetry" - posthog "github.com/devtron-labs/common-lib/telemetry" + telemetry2 "github.com/devtron-labs/devtron/client/telemetry" repository5 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/appStatus" @@ -386,13 +386,14 @@ func InitializeApp() (*App, error) { chartProviderRouterImpl := chartProvider2.NewChartProviderRouterImpl(chartProviderRestHandlerImpl) dockerRegRestHandlerImpl := restHandler.NewDockerRegRestHandlerImpl(dockerRegistryConfigImpl, sugaredLogger, chartProviderServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, deleteServiceImpl) dockerRegRouterImpl := router.NewDockerRegRouterImpl(dockerRegRestHandlerImpl) - posthogClient, err := posthog.NewPosthogClient(sugaredLogger) + posthogClient, err := telemetry.NewPosthogClient(sugaredLogger) if err != nil { return nil, err } serviceImpl := ucid.NewServiceImpl(sugaredLogger, k8sServiceImpl, acdAuthConfig) providerIdentifierServiceImpl := providerIdentifier.NewProviderIdentifierServiceImpl(sugaredLogger) - telemetryEventClientImpl, err := telemetry.NewTelemetryEventClientImpl(sugaredLogger, httpClient, clusterServiceImpl, k8sServiceImpl, acdAuthConfig, userServiceImpl, attributesRepositoryImpl, ssoLoginServiceImpl, posthogClient, serviceImpl, moduleRepositoryImpl, serverDataStoreServerDataStore, userAuditServiceImpl, helmAppClientImpl, providerIdentifierServiceImpl, cronLoggerImpl, installedAppReadServiceEAImpl, environmentVariables) + userAttributesRepositoryImpl := repository5.NewUserAttributesRepositoryImpl(db) + telemetryEventClientImpl, err := telemetry2.NewTelemetryEventClientImpl(sugaredLogger, httpClient, clusterServiceImpl, k8sServiceImpl, acdAuthConfig, userServiceImpl, attributesRepositoryImpl, ssoLoginServiceImpl, posthogClient, serviceImpl, moduleRepositoryImpl, serverDataStoreServerDataStore, userAuditServiceImpl, helmAppClientImpl, providerIdentifierServiceImpl, cronLoggerImpl, installedAppReadServiceEAImpl, environmentVariables, userAttributesRepositoryImpl) if err != nil { return nil, err } @@ -449,7 +450,6 @@ func InitializeApp() (*App, error) { webhookHelmServiceImpl := webhookHelm.NewWebhookHelmServiceImpl(sugaredLogger, helmAppServiceImpl, clusterServiceImpl, chartRepositoryServiceImpl, attributesServiceImpl) webhookHelmRestHandlerImpl := webhookHelm2.NewWebhookHelmRestHandlerImpl(sugaredLogger, webhookHelmServiceImpl, userServiceImpl, enforcerImpl, validate) webhookHelmRouterImpl := webhookHelm2.NewWebhookHelmRouterImpl(webhookHelmRestHandlerImpl) - userAttributesRepositoryImpl := repository5.NewUserAttributesRepositoryImpl(db) userAttributesServiceImpl := attributes.NewUserAttributesServiceImpl(sugaredLogger, userAttributesRepositoryImpl) userAttributesRestHandlerImpl := restHandler.NewUserAttributesRestHandlerImpl(sugaredLogger, enforcerImpl, userServiceImpl, userAttributesServiceImpl) userAttributesRouterImpl := router.NewUserAttributesRouterImpl(userAttributesRestHandlerImpl) From 0204c134f82d3a0cea5c7ae87ca8abbbf6f4d6c6 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Tue, 8 Apr 2025 01:11:27 +0530 Subject: [PATCH 32/37] chore: ci/cd workflow message datatype updated --- pkg/pipeline/CiHandler.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 8fb4a4b8da..043cd0aeeb 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -1202,10 +1202,6 @@ func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus eventProcessorBean.CiCd if impl.stateChanged(status, podStatus, message, workflowStatus.FinishedAt.Time, savedWorkflow) { if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.PodStatus) { savedWorkflow.Message = message - // NOTE: we are doing this for a quick fix where ci pending message become larger than 250 and in db we had set the charter limit to 250 - if len(message) > 250 { - savedWorkflow.Message = message[:250] - } if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.Status) { savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time } From a5bf2612c4fbdea916a2c4f73d2174ce6999a313 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 8 Apr 2025 12:10:19 +0530 Subject: [PATCH 33/37] wip --- .../configure/BuildPipelineRestHandler.go | 10 +- .../DeploymentPipelineRestHandler.go | 6 +- .../configure/PipelineConfigRestHandler.go | 12 +- .../trigger/PipelineTriggerRestHandler.go | 8 +- client/cron/CiTriggerCron.go | 8 +- pkg/build/git/gitWebhook/GitWebhookService.go | 8 +- .../trigger/{Service.go => HandlerService.go} | 91 ++++++++------- .../{Service_ent.go => HandlerService_ent.go} | 30 ++--- pkg/build/trigger/wire_trigger.go | 4 +- pkg/bulkAction/service/BulkUpdateService.go | 8 +- .../deployedApp/DeployedAppService.go | 8 +- .../{TriggerService.go => HandlerService.go} | 104 +++++++++--------- ...Service_ent1.go => HandlerService_ent1.go} | 22 ++-- .../devtronApps/PostStageTriggerService.go | 4 +- .../PostStageTriggerService_ent.go | 4 +- .../devtronApps/PrePostCommonService.go | 14 +-- .../devtronApps/PreStageTriggerService.go | 30 ++--- .../devtronApps/PreStageTriggerService_ent.go | 12 +- .../trigger/devtronApps/feasibilityChecker.go | 2 +- .../wire_devtronAppsDeployTrigger.go | 4 +- .../in/CDPipelineEventProcessorService.go | 8 +- .../in/WorkflowEventProcessorService.go | 16 +-- pkg/workflow/dag/WorkflowDagExecutor.go | 28 ++--- wire_gen.go | 34 +++--- 24 files changed, 237 insertions(+), 238 deletions(-) rename pkg/build/trigger/{Service.go => HandlerService.go} (92%) rename pkg/build/trigger/{Service_ent.go => HandlerService_ent.go} (60%) rename pkg/deployment/trigger/devtronApps/{TriggerService.go => HandlerService.go} (96%) rename pkg/deployment/trigger/devtronApps/{TriggerService_ent1.go => HandlerService_ent1.go} (79%) diff --git a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go index 78ddc57454..95cc31f80b 100644 --- a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go @@ -740,7 +740,7 @@ func (handler *PipelineConfigRestHandlerImpl) TriggerCiPipeline(w http.ResponseW ciTriggerRequest.TriggeredBy = userId handler.Logger.Infow("request payload, TriggerCiPipeline", "payload", ciTriggerRequest) response := make(map[string]string) - resp, err := handler.ciTriggerService.HandleCIManual(ciTriggerRequest) + resp, err := handler.ciHandlerService.HandleCIManual(ciTriggerRequest) if errors.Is(err, bean1.ErrImagePathInUse) { handler.Logger.Errorw("service err duplicate image tag, TriggerCiPipeline", "err", err, "payload", ciTriggerRequest) common.WriteJsonResp(w, err, err, http.StatusConflict) @@ -974,7 +974,7 @@ func (handler *PipelineConfigRestHandlerImpl) DownloadCiWorkflowArtifacts(w http } //RBAC - file, err := handler.ciTriggerService.DownloadCiWorkflowArtifacts(pipelineId, buildId) + file, err := handler.ciHandlerService.DownloadCiWorkflowArtifacts(pipelineId, buildId) defer file.Close() if err != nil { handler.Logger.Errorw("service err, DownloadCiWorkflowArtifacts", "err", err, "pipelineId", pipelineId, "buildId", buildId) @@ -1029,7 +1029,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetHistoricBuildLogs(w http.Respon return } //RBAC - resp, err := handler.ciTriggerService.GetHistoricBuildLogs(workflowId, nil) + resp, err := handler.ciHandlerService.GetHistoricBuildLogs(workflowId, nil) if err != nil { handler.Logger.Errorw("service err, GetHistoricBuildLogs", "err", err, "pipelineId", pipelineId, "workflowId", workflowId) common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) @@ -1168,7 +1168,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetBuildLogs(w http.ResponseWriter return } } - logsReader, cleanUp, err := handler.ciTriggerService.GetRunningWorkflowLogs(workflowId) + logsReader, cleanUp, err := handler.ciHandlerService.GetRunningWorkflowLogs(workflowId) if err != nil { handler.Logger.Errorw("service err, GetBuildLogs", "err", err, "pipelineId", pipelineId, "workflowId", workflowId, "lastEventId", lastEventId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) @@ -1611,7 +1611,7 @@ func (handler *PipelineConfigRestHandlerImpl) CancelWorkflow(w http.ResponseWrit //RBAC - resp, err := handler.ciTriggerService.CancelBuild(workflowId, forceAbort) + resp, err := handler.ciHandlerService.CancelBuild(workflowId, forceAbort) if err != nil { handler.Logger.Errorw("service err, CancelWorkflow", "err", err, "workflowId", workflowId, "pipelineId", pipelineId) if util.IsErrNoRows(err) { diff --git a/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go index a856e40c56..79b8a1b980 100644 --- a/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go @@ -1643,7 +1643,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetPrePostDeploymentLogs(w http.Re } //RBAC CHECK - logsReader, cleanUp, err := handler.cdTriggerService.GetRunningWorkflowLogs(environmentId, pipelineId, workflowId) + logsReader, cleanUp, err := handler.cdHandlerService.GetRunningWorkflowLogs(environmentId, pipelineId, workflowId) if err != nil { handler.Logger.Errorw("service err, GetPrePostDeploymentLogs", "err", err, "appId", appId, "environmentId", environmentId, "pipelineId", pipelineId, "workflowId", workflowId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) @@ -1768,7 +1768,7 @@ func (handler *PipelineConfigRestHandlerImpl) DownloadArtifacts(w http.ResponseW } //RBAC CHECK - file, err := handler.cdTriggerService.DownloadCdWorkflowArtifacts(buildId) + file, err := handler.cdHandlerService.DownloadCdWorkflowArtifacts(buildId) defer file.Close() if err != nil { @@ -1952,7 +1952,7 @@ func (handler *PipelineConfigRestHandlerImpl) CancelStage(w http.ResponseWriter, } //RBAC - resp, err := handler.cdTriggerService.CancelStage(workflowRunnerId, forceAbort, userId) + resp, err := handler.cdHandlerService.CancelStage(workflowRunnerId, forceAbort, userId) if err != nil { handler.Logger.Errorw("service err, CancelStage", "err", err, "pipelineId", pipelineId, "workflowRunnerId", workflowRunnerId) if util.IsErrNoRows(err) { diff --git a/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go b/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go index 11de4523aa..7781bbc508 100644 --- a/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go +++ b/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go @@ -140,8 +140,8 @@ type PipelineConfigRestHandlerImpl struct { teamReadService read3.TeamReadService environmentRepository repository2.EnvironmentRepository chartReadService read5.ChartReadService - ciTriggerService trigger.Service - cdTriggerService devtronApps.TriggerService + ciHandlerService trigger.HandlerService + cdHandlerService devtronApps.HandlerService } func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger *zap.SugaredLogger, @@ -176,8 +176,8 @@ func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger teamReadService read3.TeamReadService, EnvironmentRepository repository2.EnvironmentRepository, chartReadService read5.ChartReadService, - ciTriggerService trigger.Service, - cdTriggerService devtronApps.TriggerService, + ciHandlerService trigger.HandlerService, + cdHandlerService devtronApps.HandlerService, ) *PipelineConfigRestHandlerImpl { envConfig := &PipelineRestHandlerEnvConfig{} err := env.Parse(envConfig) @@ -220,8 +220,8 @@ func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger teamReadService: teamReadService, environmentRepository: EnvironmentRepository, chartReadService: chartReadService, - ciTriggerService: ciTriggerService, - cdTriggerService: cdTriggerService, + ciHandlerService: ciHandlerService, + cdHandlerService: cdHandlerService, } } diff --git a/api/restHandler/app/pipeline/trigger/PipelineTriggerRestHandler.go b/api/restHandler/app/pipeline/trigger/PipelineTriggerRestHandler.go index 08b43625b1..8fab862852 100644 --- a/api/restHandler/app/pipeline/trigger/PipelineTriggerRestHandler.go +++ b/api/restHandler/app/pipeline/trigger/PipelineTriggerRestHandler.go @@ -66,7 +66,7 @@ type PipelineTriggerRestHandlerImpl struct { deploymentGroupService deploymentGroup.DeploymentGroupService deploymentConfigService pipeline.PipelineDeploymentConfigService deployedAppService deployedApp.DeployedAppService - cdTriggerService devtronApps.TriggerService + cdHandlerService devtronApps.HandlerService workflowEventPublishService out.WorkflowEventPublishService } @@ -75,7 +75,7 @@ func NewPipelineRestHandler(appService app.AppService, userAuthService user.User deploymentGroupService deploymentGroup.DeploymentGroupService, deploymentConfigService pipeline.PipelineDeploymentConfigService, deployedAppService deployedApp.DeployedAppService, - cdTriggerService devtronApps.TriggerService, + cdHandlerService devtronApps.HandlerService, workflowEventPublishService out.WorkflowEventPublishService) *PipelineTriggerRestHandlerImpl { pipelineHandler := &PipelineTriggerRestHandlerImpl{ appService: appService, @@ -88,7 +88,7 @@ func NewPipelineRestHandler(appService app.AppService, userAuthService user.User deploymentGroupService: deploymentGroupService, deploymentConfigService: deploymentConfigService, deployedAppService: deployedAppService, - cdTriggerService: cdTriggerService, + cdHandlerService: cdHandlerService, workflowEventPublishService: workflowEventPublishService, } return pipelineHandler @@ -140,7 +140,7 @@ func (handler PipelineTriggerRestHandlerImpl) OverrideConfig(w http.ResponseWrit triggerContext := bean3.TriggerContext{ Context: ctx, } - mergeResp, helmPackageName, _, err := handler.cdTriggerService.ManualCdTrigger(triggerContext, &overrideRequest) + mergeResp, helmPackageName, _, err := handler.cdHandlerService.ManualCdTrigger(triggerContext, &overrideRequest) span.End() if err != nil { handler.logger.Errorw("request err, OverrideConfig", "err", err, "payload", overrideRequest) diff --git a/client/cron/CiTriggerCron.go b/client/cron/CiTriggerCron.go index b5b9305afa..5eb7020d36 100644 --- a/client/cron/CiTriggerCron.go +++ b/client/cron/CiTriggerCron.go @@ -42,12 +42,12 @@ type CiTriggerCronImpl struct { pipelineStageRepository repository.PipelineStageRepository ciArtifactRepository repository2.CiArtifactRepository globalPluginRepository repository3.GlobalPluginRepository - ciTriggerService trigger.Service + ciHandlerService trigger.HandlerService } func NewCiTriggerCronImpl(logger *zap.SugaredLogger, cfg *CiTriggerCronConfig, pipelineStageRepository repository.PipelineStageRepository, ciArtifactRepository repository2.CiArtifactRepository, globalPluginRepository repository3.GlobalPluginRepository, cronLogger *cron2.CronLoggerImpl, - ciTriggerService trigger.Service) *CiTriggerCronImpl { + ciHandlerService trigger.HandlerService) *CiTriggerCronImpl { cron := cron.New( cron.WithChain(cron.Recover(cronLogger))) cron.Start() @@ -58,7 +58,7 @@ func NewCiTriggerCronImpl(logger *zap.SugaredLogger, cfg *CiTriggerCronConfig, p cfg: cfg, ciArtifactRepository: ciArtifactRepository, globalPluginRepository: globalPluginRepository, - ciTriggerService: ciTriggerService, + ciHandlerService: ciHandlerService, } _, err := cron.AddFunc(fmt.Sprintf("@every %dm", cfg.SourceControllerCronTime), impl.TriggerCiCron) @@ -104,7 +104,7 @@ func (impl *CiTriggerCronImpl) TriggerCiCron() { InvalidateCache: false, PipelineType: string(common.CI_JOB), } - _, err = impl.ciTriggerService.HandleCIManual(ciTriggerRequest) + _, err = impl.ciHandlerService.HandleCIManual(ciTriggerRequest) if err != nil { return } diff --git a/pkg/build/git/gitWebhook/GitWebhookService.go b/pkg/build/git/gitWebhook/GitWebhookService.go index 8b4405a68b..7ef9e0edc6 100644 --- a/pkg/build/git/gitWebhook/GitWebhookService.go +++ b/pkg/build/git/gitWebhook/GitWebhookService.go @@ -34,15 +34,15 @@ type GitWebhookService interface { type GitWebhookServiceImpl struct { logger *zap.SugaredLogger gitWebhookRepository repository.GitWebhookRepository - ciTriggerService trigger.Service + ciHandlerService trigger.HandlerService } func NewGitWebhookServiceImpl(Logger *zap.SugaredLogger, gitWebhookRepository repository.GitWebhookRepository, - ciTriggerService trigger.Service) *GitWebhookServiceImpl { + ciHandlerService trigger.HandlerService) *GitWebhookServiceImpl { return &GitWebhookServiceImpl{ logger: Logger, gitWebhookRepository: gitWebhookRepository, - ciTriggerService: ciTriggerService, + ciHandlerService: ciHandlerService, } } @@ -71,7 +71,7 @@ func (impl *GitWebhookServiceImpl) HandleGitWebhook(gitWebhookRequest gitSensor. } } - resp, err := impl.ciTriggerService.HandleCIWebhook(bean.GitCiTriggerRequest{ + resp, err := impl.ciHandlerService.HandleCIWebhook(bean.GitCiTriggerRequest{ CiPipelineMaterial: ciPipelineMaterial, TriggeredBy: bean2.SYSTEM_USER_ID, // Automatic trigger, system user ExtraEnvironmentVariables: gitWebhookRequest.ExtraEnvironmentVariables, diff --git a/pkg/build/trigger/Service.go b/pkg/build/trigger/HandlerService.go similarity index 92% rename from pkg/build/trigger/Service.go rename to pkg/build/trigger/HandlerService.go index 0812c485ba..8a4b521666 100644 --- a/pkg/build/trigger/Service.go +++ b/pkg/build/trigger/HandlerService.go @@ -68,12 +68,11 @@ import ( "time" ) -type Service interface { +type HandlerService interface { HandlePodDeleted(ciWorkflow *pipelineConfig.CiWorkflow) CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) - //TriggerCiPipeline(trigger types.Trigger) (int, error) StartCiWorkflowAndPrepareWfRequest(trigger types.Trigger) (*pipelineConfig.CiPipeline, map[string]string, *pipelineConfig.CiWorkflow, *types.WorkflowRequest, error) @@ -88,7 +87,7 @@ type BuildxCacheFlags struct { AsyncBuildxCacheExport bool `env:"ASYNC_BUILDX_CACHE_EXPORT" envDefault:"false"` } -type ServiceImpl struct { +type HandlerServiceImpl struct { Logger *zap.SugaredLogger workflowService executor.WorkflowService ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository @@ -118,7 +117,7 @@ type ServiceImpl struct { K8sUtil *k8s.K8sServiceImpl } -func NewServiceImpl(Logger *zap.SugaredLogger, workflowService executor.WorkflowService, +func NewHandlerServiceImpl(Logger *zap.SugaredLogger, workflowService executor.WorkflowService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, ciPipelineRepository pipelineConfig.CiPipelineRepository, ciArtifactRepository repository5.CiArtifactRepository, @@ -141,13 +140,13 @@ func NewServiceImpl(Logger *zap.SugaredLogger, workflowService executor.Workflow clusterService cluster.ClusterService, envService environment.EnvironmentService, K8sUtil *k8s.K8sServiceImpl, -) *ServiceImpl { +) *HandlerServiceImpl { buildxCacheFlags := &BuildxCacheFlags{} err := env.Parse(buildxCacheFlags) if err != nil { Logger.Infow("error occurred while parsing BuildxCacheFlags env,so setting BuildxCacheModeMin and AsyncBuildxCacheExport to default value", "err", err) } - cis := &ServiceImpl{ + cis := &HandlerServiceImpl{ Logger: Logger, workflowService: workflowService, ciPipelineMaterialRepository: ciPipelineMaterialRepository, @@ -183,7 +182,7 @@ func NewServiceImpl(Logger *zap.SugaredLogger, workflowService executor.Workflow return cis } -func (impl *ServiceImpl) HandlePodDeleted(ciWorkflow *pipelineConfig.CiWorkflow) { +func (impl *HandlerServiceImpl) HandlePodDeleted(ciWorkflow *pipelineConfig.CiWorkflow) { if !impl.config.WorkflowRetriesEnabled() { impl.Logger.Debug("ci workflow retry feature disabled") return @@ -199,7 +198,7 @@ func (impl *ServiceImpl) HandlePodDeleted(ciWorkflow *pipelineConfig.CiWorkflow) } } -func (impl *ServiceImpl) CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error { +func (impl *HandlerServiceImpl) CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error { // return if re-trigger feature is disabled if !impl.config.WorkflowRetriesEnabled() { @@ -233,7 +232,7 @@ func (impl *ServiceImpl) CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowSta return err } -func (impl *ServiceImpl) reTriggerCi(retryCount int, refCiWorkflow *pipelineConfig.CiWorkflow) error { +func (impl *HandlerServiceImpl) reTriggerCi(retryCount int, refCiWorkflow *pipelineConfig.CiWorkflow) error { if retryCount >= impl.config.MaxCiWorkflowRetries { impl.Logger.Infow("maximum retries exhausted for this ciWorkflow", "ciWorkflowId", refCiWorkflow.Id, "retries", retryCount, "configuredRetries", impl.config.MaxCiWorkflowRetries) return nil @@ -268,7 +267,7 @@ func (impl *ServiceImpl) reTriggerCi(retryCount int, refCiWorkflow *pipelineConf return nil } -func (impl *ServiceImpl) HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) { +func (impl *HandlerServiceImpl) HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) { impl.Logger.Debugw("HandleCIManual for pipeline ", "PipelineId", ciTriggerRequest.PipelineId) commitHashes, runtimeParams, err := impl.buildManualTriggerCommitHashes(ciTriggerRequest) if err != nil { @@ -305,7 +304,7 @@ func (impl *ServiceImpl) HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) return id, nil } -func (impl *ServiceImpl) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) { +func (impl *HandlerServiceImpl) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) { impl.Logger.Debugw("HandleCIWebhook for material ", "material", gitCiTriggerRequest.CiPipelineMaterial) ciPipeline, err := impl.GetCiPipeline(gitCiTriggerRequest.CiPipelineMaterial.Id) if err != nil { @@ -357,7 +356,7 @@ func (impl *ServiceImpl) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRe return id, nil } -func (impl *ServiceImpl) extractPodStatusAndWorkflow(workflowStatus v1alpha1.WorkflowStatus) (string, string, *pipelineConfig.CiWorkflow, error) { +func (impl *HandlerServiceImpl) extractPodStatusAndWorkflow(workflowStatus v1alpha1.WorkflowStatus) (string, string, *pipelineConfig.CiWorkflow, error) { workflowName, status, _, message, _, _ := pipeline2.ExtractWorkflowStatus(workflowStatus) if workflowName == "" { impl.Logger.Errorw("extract workflow status, invalid wf name", "workflowName", workflowName, "status", status, "message", message) @@ -379,7 +378,7 @@ func (impl *ServiceImpl) extractPodStatusAndWorkflow(workflowStatus v1alpha1.Wor } -func (impl *ServiceImpl) getRefWorkflowAndCiRetryCount(savedWorkflow *pipelineConfig.CiWorkflow) (int, *pipelineConfig.CiWorkflow, error) { +func (impl *HandlerServiceImpl) getRefWorkflowAndCiRetryCount(savedWorkflow *pipelineConfig.CiWorkflow) (int, *pipelineConfig.CiWorkflow, error) { var err error if savedWorkflow.ReferenceCiWorkflowId != 0 { @@ -393,7 +392,7 @@ func (impl *ServiceImpl) getRefWorkflowAndCiRetryCount(savedWorkflow *pipelineCo return retryCount, savedWorkflow, err } -func (impl *ServiceImpl) validateBuildSequence(gitCiTriggerRequest bean.GitCiTriggerRequest, pipelineId int) (bool, error) { +func (impl *HandlerServiceImpl) validateBuildSequence(gitCiTriggerRequest bean.GitCiTriggerRequest, pipelineId int) (bool, error) { isValid := true lastTriggeredBuild, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflow(pipelineId) if !(lastTriggeredBuild.Status == string(v1alpha1.NodePending) || lastTriggeredBuild.Status == string(v1alpha1.NodeRunning)) { @@ -416,7 +415,7 @@ func (impl *ServiceImpl) validateBuildSequence(gitCiTriggerRequest bean.GitCiTri return isValid, nil } -func (impl *ServiceImpl) buildAutomaticTriggerCommitHashes(ciMaterials []*pipelineConfig.CiPipelineMaterial, request bean.GitCiTriggerRequest) (map[int]pipelineConfig.GitCommit, error) { +func (impl *HandlerServiceImpl) buildAutomaticTriggerCommitHashes(ciMaterials []*pipelineConfig.CiPipelineMaterial, request bean.GitCiTriggerRequest) (map[int]pipelineConfig.GitCommit, error) { commitHashes := map[int]pipelineConfig.GitCommit{} for _, ciMaterial := range ciMaterials { if ciMaterial.Id == request.CiPipelineMaterial.Id || len(ciMaterials) == 1 { @@ -435,7 +434,7 @@ func (impl *ServiceImpl) buildAutomaticTriggerCommitHashes(ciMaterials []*pipeli return commitHashes, nil } -func (impl *ServiceImpl) GetCiPipeline(ciMaterialId int) (*pipelineConfig.CiPipeline, error) { +func (impl *HandlerServiceImpl) GetCiPipeline(ciMaterialId int) (*pipelineConfig.CiPipeline, error) { ciMaterial, err := impl.ciPipelineMaterialRepository.GetById(ciMaterialId) if err != nil { return nil, err @@ -444,7 +443,7 @@ func (impl *ServiceImpl) GetCiPipeline(ciMaterialId int) (*pipelineConfig.CiPipe return ciPipeline, nil } -func (impl *ServiceImpl) buildManualTriggerCommitHashes(ciTriggerRequest bean.CiTriggerRequest) (map[int]pipelineConfig.GitCommit, *common.RuntimeParameters, error) { +func (impl *HandlerServiceImpl) buildManualTriggerCommitHashes(ciTriggerRequest bean.CiTriggerRequest) (map[int]pipelineConfig.GitCommit, *common.RuntimeParameters, error) { commitHashes := map[int]pipelineConfig.GitCommit{} runtimeParams := impl.getRuntimeParamsForBuildingManualTriggerHashes(ciTriggerRequest) for _, ciPipelineMaterial := range ciTriggerRequest.CiPipelineMaterial { @@ -479,7 +478,7 @@ func (impl *ServiceImpl) buildManualTriggerCommitHashes(ciTriggerRequest bean.Ci return commitHashes, runtimeParams, nil } -func (impl *ServiceImpl) BuildManualTriggerCommitHashesForSourceTypeBranchFix(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (pipelineConfig.GitCommit, error) { +func (impl *HandlerServiceImpl) BuildManualTriggerCommitHashesForSourceTypeBranchFix(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (pipelineConfig.GitCommit, error) { commitMetadataRequest := &gitSensor.CommitMetadataRequest{ PipelineMaterialId: ciPipelineMaterial.Id, GitHash: ciPipelineMaterial.GitCommit.Commit, @@ -509,7 +508,7 @@ func (impl *ServiceImpl) BuildManualTriggerCommitHashesForSourceTypeBranchFix(ci return gitCommit, nil } -func (impl *ServiceImpl) BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (pipelineConfig.GitCommit, map[string]string, error) { +func (impl *HandlerServiceImpl) BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (pipelineConfig.GitCommit, map[string]string, error) { webhookDataInput := ciPipelineMaterial.GitCommit.WebhookData // fetch webhook data on the basis of Id @@ -569,7 +568,7 @@ func (impl *ServiceImpl) BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPi return gitCommit, webhookAndCiData.ExtraEnvironmentVariables, nil } -func (impl *ServiceImpl) getLastSeenCommit(ciMaterialId int) (pipelineConfig.GitCommit, error) { +func (impl *HandlerServiceImpl) getLastSeenCommit(ciMaterialId int) (pipelineConfig.GitCommit, error) { var materialIds []int materialIds = append(materialIds, ciMaterialId) headReq := &gitSensor.HeadRequest{ @@ -601,7 +600,7 @@ func SetGitCommitValuesForBuildingCommitHash(ciMaterial *pipelineConfig.CiPipeli return newGitCommit } -func (impl *ServiceImpl) triggerCiPipeline(trigger types.Trigger) (int, error) { +func (impl *HandlerServiceImpl) triggerCiPipeline(trigger types.Trigger) (int, error) { pipeline, variableSnapshot, savedCiWf, workflowRequest, err := impl.StartCiWorkflowAndPrepareWfRequest(trigger) if err != nil { return 0, err @@ -632,7 +631,7 @@ func (impl *ServiceImpl) triggerCiPipeline(trigger types.Trigger) (int, error) { return savedCiWf.Id, err } -func (impl *ServiceImpl) GetCiMaterials(pipelineId int, ciMaterials []*pipelineConfig.CiPipelineMaterial) ([]*pipelineConfig.CiPipelineMaterial, error) { +func (impl *HandlerServiceImpl) GetCiMaterials(pipelineId int, ciMaterials []*pipelineConfig.CiPipelineMaterial) ([]*pipelineConfig.CiPipelineMaterial, error) { if !(len(ciMaterials) == 0) { return ciMaterials, nil } else { @@ -646,7 +645,7 @@ func (impl *ServiceImpl) GetCiMaterials(pipelineId int, ciMaterials []*pipelineC } } -func (impl *ServiceImpl) StartCiWorkflowAndPrepareWfRequest(trigger types.Trigger) (*pipelineConfig.CiPipeline, map[string]string, *pipelineConfig.CiWorkflow, *types.WorkflowRequest, error) { +func (impl *HandlerServiceImpl) StartCiWorkflowAndPrepareWfRequest(trigger types.Trigger) (*pipelineConfig.CiPipeline, map[string]string, *pipelineConfig.CiWorkflow, *types.WorkflowRequest, error) { impl.Logger.Debugw("ci pipeline manual trigger", "request", trigger) ciMaterials, err := impl.GetCiMaterials(trigger.PipelineId, trigger.CiMaterials) if err != nil { @@ -783,7 +782,7 @@ func (impl *ServiceImpl) StartCiWorkflowAndPrepareWfRequest(trigger types.Trigge return pipeline, variableSnapshot, savedCiWf, workflowRequest, nil } -func (impl *ServiceImpl) setBuildxK8sDriverData(workflowRequest *types.WorkflowRequest) error { +func (impl *HandlerServiceImpl) setBuildxK8sDriverData(workflowRequest *types.WorkflowRequest) error { dockerBuildConfig := workflowRequest.CiBuildConfig.DockerBuildConfig k8sDriverOptions, err := impl.getK8sDriverOptions(workflowRequest, dockerBuildConfig.TargetPlatform) if err != nil { @@ -793,7 +792,7 @@ func (impl *ServiceImpl) setBuildxK8sDriverData(workflowRequest *types.WorkflowR return nil } -func (impl *ServiceImpl) getEnvironmentForJob(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger) (*repository6.Environment, bool, error) { +func (impl *HandlerServiceImpl) getEnvironmentForJob(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger) (*repository6.Environment, bool, error) { app, err := impl.appRepository.FindById(pipeline.AppId) if err != nil { impl.Logger.Errorw("could not find app", "err", err) @@ -817,14 +816,14 @@ func (impl *ServiceImpl) getEnvironmentForJob(pipeline *pipelineConfig.CiPipelin } // TODO: Send all trigger data -func (impl *ServiceImpl) BuildPayload(trigger types.Trigger, pipeline *pipelineConfig.CiPipeline) *client.Payload { +func (impl *HandlerServiceImpl) BuildPayload(trigger types.Trigger, pipeline *pipelineConfig.CiPipeline) *client.Payload { payload := &client.Payload{} payload.AppName = pipeline.App.AppName payload.PipelineName = pipeline.Name return payload } -func (impl *ServiceImpl) saveNewWorkflowForCITrigger(pipeline *pipelineConfig.CiPipeline, ciWorkflowConfigNamespace string, +func (impl *HandlerServiceImpl) saveNewWorkflowForCITrigger(pipeline *pipelineConfig.CiPipeline, ciWorkflowConfigNamespace string, commitHashes map[int]pipelineConfig.GitCommit, userId int32, ciMaterials []*pipelineConfig.CiPipelineMaterial, EnvironmentId int, isJob bool, refCiWorkflowId int) (*pipelineConfig.CiWorkflow, error) { isCiTriggerBlocked, err := impl.checkIfCITriggerIsBlocked(pipeline, ciMaterials, isJob) @@ -862,7 +861,7 @@ func (impl *ServiceImpl) saveNewWorkflowForCITrigger(pipeline *pipelineConfig.Ci return ciWorkflow, nil } -func (impl *ServiceImpl) executeCiPipeline(workflowRequest *types.WorkflowRequest) error { +func (impl *HandlerServiceImpl) executeCiPipeline(workflowRequest *types.WorkflowRequest) error { _, _, err := impl.workflowService.SubmitWorkflow(workflowRequest) if err != nil { impl.Logger.Errorw("workflow error", "err", err) @@ -871,20 +870,20 @@ func (impl *ServiceImpl) executeCiPipeline(workflowRequest *types.WorkflowReques return nil } -func (impl *ServiceImpl) buildS3ArtifactLocation(ciWorkflowConfigLogsBucket string, savedWf *pipelineConfig.CiWorkflow) (string, string, string) { +func (impl *HandlerServiceImpl) buildS3ArtifactLocation(ciWorkflowConfigLogsBucket string, savedWf *pipelineConfig.CiWorkflow) (string, string, string) { ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() ArtifactLocation := fmt.Sprintf("s3://"+path.Join(ciWorkflowConfigLogsBucket, ciArtifactLocationFormat), savedWf.Id, savedWf.Id) artifactFileName := fmt.Sprintf(ciArtifactLocationFormat, savedWf.Id, savedWf.Id) return ArtifactLocation, ciWorkflowConfigLogsBucket, artifactFileName } -func (impl *ServiceImpl) buildDefaultArtifactLocation(savedWf *pipelineConfig.CiWorkflow) string { +func (impl *HandlerServiceImpl) buildDefaultArtifactLocation(savedWf *pipelineConfig.CiWorkflow) string { ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() ArtifactLocation := fmt.Sprintf(ciArtifactLocationFormat, savedWf.Id, savedWf.Id) return ArtifactLocation } -func (impl *ServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, savedWf *pipelineConfig.CiWorkflow, ciWorkflowConfigNamespace string, ciPipelineScripts []*pipelineConfig.CiPipelineScript, preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, refPluginsData []*pipelineConfigBean.RefPluginObject, isJob bool) (*types.WorkflowRequest, error) { +func (impl *HandlerServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, savedWf *pipelineConfig.CiWorkflow, ciWorkflowConfigNamespace string, ciPipelineScripts []*pipelineConfig.CiPipelineScript, preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, refPluginsData []*pipelineConfigBean.RefPluginObject, isJob bool) (*types.WorkflowRequest, error) { var ciProjectDetails []pipelineConfigBean.CiProjectDetails commitHashes := trigger.CommitHashes for _, ciMaterial := range ciMaterials { @@ -1263,7 +1262,7 @@ func (impl *ServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.Ci return workflowRequest, nil } -func (impl *ServiceImpl) GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, customTag string, customTagId int, buildImagePath string, buildImagedockerRegistryId string) (map[string][]string, map[string]bean2.RegistryCredentials, string, []int, error) { +func (impl *HandlerServiceImpl) GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, customTag string, customTagId int, buildImagePath string, buildImagedockerRegistryId string) (map[string][]string, map[string]bean2.RegistryCredentials, string, []int, error) { copyContainerImagePluginDetail, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(buildCommonBean.COPY_CONTAINER_IMAGE) if err != nil && err != pg.ErrNoRows { @@ -1334,7 +1333,7 @@ func (impl *ServiceImpl) GetWorkflowRequestVariablesForCopyContainerImagePlugin( return registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imagePathReservationIds, nil } -func (impl *ServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, destinationImages []string) ([]int, error) { +func (impl *HandlerServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, destinationImages []string) ([]int, error) { var imagePathReservationIds []int for _, image := range destinationImages { imagePathReservationData, err := impl.customTagService.ReserveImagePath(image, customTagId) @@ -1392,7 +1391,7 @@ func buildCiStepsDataFromDockerBuildScripts(dockerBuildScripts []*bean.CiScript) return ciSteps } -func (impl *ServiceImpl) buildImageTag(commitHashes map[int]pipelineConfig.GitCommit, id int, wfId int) string { +func (impl *HandlerServiceImpl) buildImageTag(commitHashes map[int]pipelineConfig.GitCommit, id int, wfId int) string { dockerImageTag := "" toAppendDevtronParamInTag := true for _, v := range commitHashes { @@ -1441,14 +1440,14 @@ func getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, commitOrCh return dockerImageTag } -func (impl *ServiceImpl) updateCiWorkflow(request *types.WorkflowRequest, savedWf *pipelineConfig.CiWorkflow) error { +func (impl *HandlerServiceImpl) updateCiWorkflow(request *types.WorkflowRequest, savedWf *pipelineConfig.CiWorkflow) error { ciBuildConfig := request.CiBuildConfig ciBuildType := string(ciBuildConfig.CiBuildType) savedWf.CiBuildType = ciBuildType return impl.ciService.UpdateCiWorkflowWithStage(savedWf) } -func (impl *ServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, workflowRequest *types.WorkflowRequest) error { +func (impl *HandlerServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, workflowRequest *types.WorkflowRequest) error { // externalCi artifact is meant only for CI_JOB if trigger.PipelineType != string(buildCommonBean.CI_JOB) { return nil @@ -1534,7 +1533,7 @@ func _getTruncatedImageTag(imageTag string) string { } } -func (impl *ServiceImpl) markCurrentCiWorkflowFailed(savedCiWf *pipelineConfig.CiWorkflow, validationErr error) error { +func (impl *HandlerServiceImpl) markCurrentCiWorkflowFailed(savedCiWf *pipelineConfig.CiWorkflow, validationErr error) error { // currently such requirement is not there if savedCiWf == nil { return nil @@ -1562,7 +1561,7 @@ func (impl *ServiceImpl) markCurrentCiWorkflowFailed(savedCiWf *pipelineConfig.C return nil } -func (impl *ServiceImpl) CancelBuild(workflowId int, forceAbort bool) (int, error) { +func (impl *HandlerServiceImpl) CancelBuild(workflowId int, forceAbort bool) (int, error) { workflow, err := impl.ciWorkflowRepository.FindById(workflowId) if err != nil { impl.Logger.Errorw("error in finding ci-workflow by workflow id", "ciWorkflowId", workflowId, "err", err) @@ -1639,7 +1638,7 @@ func (impl *ServiceImpl) CancelBuild(workflowId int, forceAbort bool) (int, erro return workflow.Id, nil } -func (impl *ServiceImpl) handleForceAbortCaseForCi(workflow *pipelineConfig.CiWorkflow, forceAbort bool) error { +func (impl *HandlerServiceImpl) handleForceAbortCaseForCi(workflow *pipelineConfig.CiWorkflow, forceAbort bool) error { isWorkflowInNonTerminalStage := workflow.Status == string(v1alpha1.NodePending) || workflow.Status == string(v1alpha1.NodeRunning) if !isWorkflowInNonTerminalStage { if forceAbort { @@ -1655,7 +1654,7 @@ func (impl *ServiceImpl) handleForceAbortCaseForCi(workflow *pipelineConfig.CiWo return nil } -func (impl *ServiceImpl) updateWorkflowForForceAbort(workflow *pipelineConfig.CiWorkflow) error { +func (impl *HandlerServiceImpl) updateWorkflowForForceAbort(workflow *pipelineConfig.CiWorkflow) error { workflow.Status = cdWorkflow.WorkflowCancel workflow.PodStatus = string(bean.Failed) workflow.Message = constants2.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE @@ -1667,7 +1666,7 @@ func (impl *ServiceImpl) updateWorkflowForForceAbort(workflow *pipelineConfig.Ci return nil } -func (impl *ServiceImpl) getRestConfig(workflow *pipelineConfig.CiWorkflow) (*rest.Config, error) { +func (impl *HandlerServiceImpl) getRestConfig(workflow *pipelineConfig.CiWorkflow) (*rest.Config, error) { env, err := impl.envRepository.FindById(workflow.EnvironmentId) if err != nil { impl.Logger.Errorw("could not fetch stage env", "err", err) @@ -1685,7 +1684,7 @@ func (impl *ServiceImpl) getRestConfig(workflow *pipelineConfig.CiWorkflow) (*re return restConfig, nil } -func (impl *ServiceImpl) GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, func() error, error) { +func (impl *HandlerServiceImpl) GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, func() error, error) { ciWorkflow, err := impl.ciWorkflowRepository.FindById(workflowId) if err != nil { impl.Logger.Errorw("err", "err", err) @@ -1694,7 +1693,7 @@ func (impl *ServiceImpl) GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, return impl.getWorkflowLogs(ciWorkflow) } -func (impl *ServiceImpl) getWorkflowLogs(ciWorkflow *pipelineConfig.CiWorkflow) (*bufio.Reader, func() error, error) { +func (impl *HandlerServiceImpl) getWorkflowLogs(ciWorkflow *pipelineConfig.CiWorkflow) (*bufio.Reader, func() error, error) { if string(v1alpha1.NodePending) == ciWorkflow.PodStatus { return bufio.NewReader(strings.NewReader("")), func() error { return nil }, nil } @@ -1736,7 +1735,7 @@ func (impl *ServiceImpl) getWorkflowLogs(ciWorkflow *pipelineConfig.CiWorkflow) return logReader, cleanUp, err } -func (impl *ServiceImpl) getLogsFromRepository(ciWorkflow *pipelineConfig.CiWorkflow, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { +func (impl *HandlerServiceImpl) getLogsFromRepository(ciWorkflow *pipelineConfig.CiWorkflow, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { impl.Logger.Debug("getting historic logs", "ciWorkflowId", ciWorkflow.Id) ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion @@ -1791,7 +1790,7 @@ func (impl *ServiceImpl) getLogsFromRepository(ciWorkflow *pipelineConfig.CiWork return logReader, cleanUp, err } -func (impl *ServiceImpl) GetHistoricBuildLogs(workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) { +func (impl *HandlerServiceImpl) GetHistoricBuildLogs(workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) { var err error if ciWorkflow == nil { ciWorkflow, err = impl.ciWorkflowRepository.FindById(workflowId) @@ -1863,7 +1862,7 @@ func (impl *ServiceImpl) GetHistoricBuildLogs(workflowId int, ciWorkflow *pipeli return resp, err } -func (impl *ServiceImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) { +func (impl *HandlerServiceImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) { ciWorkflow, err := impl.ciWorkflowRepository.FindById(buildId) if err != nil { impl.Logger.Errorw("unable to fetch ciWorkflow", "err", err) diff --git a/pkg/build/trigger/Service_ent.go b/pkg/build/trigger/HandlerService_ent.go similarity index 60% rename from pkg/build/trigger/Service_ent.go rename to pkg/build/trigger/HandlerService_ent.go index 0844a5986b..3938288091 100644 --- a/pkg/build/trigger/Service_ent.go +++ b/pkg/build/trigger/HandlerService_ent.go @@ -17,67 +17,67 @@ import ( "net/http" ) -func (impl *ServiceImpl) updateRuntimeParamsForAutoCI(ciPipelineId int, runtimeParameters *common.RuntimeParameters) (*common.RuntimeParameters, error) { +func (impl *HandlerServiceImpl) updateRuntimeParamsForAutoCI(ciPipelineId int, runtimeParameters *common.RuntimeParameters) (*common.RuntimeParameters, error) { return runtimeParameters, nil } -func (impl *ServiceImpl) getRuntimeParamsForBuildingManualTriggerHashes(ciTriggerRequest bean3.CiTriggerRequest) *common.RuntimeParameters { +func (impl *HandlerServiceImpl) getRuntimeParamsForBuildingManualTriggerHashes(ciTriggerRequest bean3.CiTriggerRequest) *common.RuntimeParameters { return common.NewRuntimeParameters() } -func (impl *ServiceImpl) fetchImageScanExecutionMedium() (*repository.ScanToolMetadata, bean.ScanExecutionMedium, error) { +func (impl *HandlerServiceImpl) fetchImageScanExecutionMedium() (*repository.ScanToolMetadata, bean.ScanExecutionMedium, error) { return &repository.ScanToolMetadata{}, "", nil } -func (impl *ServiceImpl) fetchImageScanExecutionStepsForWfRequest(scanToolMetadata *repository.ScanToolMetadata) ([]*types.ImageScanningSteps, []*pipelineConfigBean.RefPluginObject, error) { +func (impl *HandlerServiceImpl) fetchImageScanExecutionStepsForWfRequest(scanToolMetadata *repository.ScanToolMetadata) ([]*types.ImageScanningSteps, []*pipelineConfigBean.RefPluginObject, error) { return nil, nil, nil } -func (impl *ServiceImpl) checkIfCITriggerIsBlocked(pipeline *pipelineConfig.CiPipeline, +func (impl *HandlerServiceImpl) checkIfCITriggerIsBlocked(pipeline *pipelineConfig.CiPipeline, ciMaterials []*pipelineConfig.CiPipelineMaterial, isJob bool) (bool, error) { return false, nil } -func (impl *ServiceImpl) handleWFIfCITriggerIsBlocked(ciWorkflow *pipelineConfig.CiWorkflow) (*pipelineConfig.CiWorkflow, error) { +func (impl *HandlerServiceImpl) handleWFIfCITriggerIsBlocked(ciWorkflow *pipelineConfig.CiWorkflow) (*pipelineConfig.CiWorkflow, error) { impl.Logger.Errorw("cannot trigger pipeline, blocked by mandatory plugin policy", "ciPipelineId", ciWorkflow.CiPipelineId) return &pipelineConfig.CiWorkflow{}, util.GetApiErrorAdapter(http.StatusInternalServerError, "500", "Invalid flow access, corrupt data possibility", "Invalid flow access, corrupt data possibility") } -func (impl *ServiceImpl) checkArgoSetupRequirement(envModal *repository2.Environment) error { +func (impl *HandlerServiceImpl) checkArgoSetupRequirement(envModal *repository2.Environment) error { return nil } -func (impl *ServiceImpl) updateWorkflowRequestForDigestPull(pipelineId int, workflowRequest *types.WorkflowRequest) (*types.WorkflowRequest, error) { +func (impl *HandlerServiceImpl) updateWorkflowRequestForDigestPull(pipelineId int, workflowRequest *types.WorkflowRequest) (*types.WorkflowRequest, error) { return workflowRequest, nil } -func (impl *ServiceImpl) updateCIProjectDetailWithCloningMode(appId int, ciMaterial *pipelineConfig.CiPipelineMaterial, +func (impl *HandlerServiceImpl) updateCIProjectDetailWithCloningMode(appId int, ciMaterial *pipelineConfig.CiPipelineMaterial, ciProjectDetail pipelineConfigBean.CiProjectDetails) (pipelineConfigBean.CiProjectDetails, error) { return ciProjectDetail, nil } -func (impl *ServiceImpl) updateWorkflowRequestWithRemoteConnConf(dockerRegistry *repository3.DockerArtifactStore, +func (impl *HandlerServiceImpl) updateWorkflowRequestWithRemoteConnConf(dockerRegistry *repository3.DockerArtifactStore, workflowRequest *types.WorkflowRequest) (*types.WorkflowRequest, error) { return workflowRequest, nil } -func (impl *ServiceImpl) updateWorkflowRequestWithEntSupportData(workflowRequest *types.WorkflowRequest) *types.WorkflowRequest { +func (impl *HandlerServiceImpl) updateWorkflowRequestWithEntSupportData(workflowRequest *types.WorkflowRequest) *types.WorkflowRequest { return workflowRequest } -func (impl *ServiceImpl) updateWorkflowRequestWithBuildCacheData(workflowRequest *types.WorkflowRequest, +func (impl *HandlerServiceImpl) updateWorkflowRequestWithBuildCacheData(workflowRequest *types.WorkflowRequest, scope resourceQualifiers.Scope) (*types.WorkflowRequest, error) { workflowRequest.BuildxCacheModeMin = impl.buildxCacheFlags.BuildxCacheModeMin workflowRequest.AsyncBuildxCacheExport = impl.buildxCacheFlags.AsyncBuildxCacheExport return workflowRequest, nil } -func (impl *ServiceImpl) canSetK8sDriverData(workflowRequest *types.WorkflowRequest) bool { +func (impl *HandlerServiceImpl) canSetK8sDriverData(workflowRequest *types.WorkflowRequest) bool { return impl.config != nil && impl.config.BuildxK8sDriverOptions != "" && workflowRequest.CiBuildConfig != nil && workflowRequest.CiBuildConfig.DockerBuildConfig != nil } -func (impl *ServiceImpl) getK8sDriverOptions(workflowRequest *types.WorkflowRequest, targetPlatforms string) ([]map[string]string, error) { +func (impl *HandlerServiceImpl) getK8sDriverOptions(workflowRequest *types.WorkflowRequest, targetPlatforms string) ([]map[string]string, error) { buildxK8sDriverOptions := make([]map[string]string, 0) err := json.Unmarshal([]byte(impl.config.BuildxK8sDriverOptions), &buildxK8sDriverOptions) if err != nil { @@ -86,7 +86,7 @@ func (impl *ServiceImpl) getK8sDriverOptions(workflowRequest *types.WorkflowRequ return buildxK8sDriverOptions, nil } -func (impl *ServiceImpl) updateCIBuildConfig(ciBuildConfigBean *bean2.CiBuildConfigBean) *bean2.CiBuildConfigBean { +func (impl *HandlerServiceImpl) updateCIBuildConfig(ciBuildConfigBean *bean2.CiBuildConfigBean) *bean2.CiBuildConfigBean { defaultTargetPlatform := impl.config.DefaultTargetPlatform useBuildx := impl.config.UseBuildx if ciBuildConfigBean.DockerBuildConfig != nil { diff --git a/pkg/build/trigger/wire_trigger.go b/pkg/build/trigger/wire_trigger.go index dc462bcd88..aca718e3cd 100644 --- a/pkg/build/trigger/wire_trigger.go +++ b/pkg/build/trigger/wire_trigger.go @@ -5,6 +5,6 @@ import ( ) var WireSet = wire.NewSet( - NewServiceImpl, - wire.Bind(new(Service), new(*ServiceImpl)), + NewHandlerServiceImpl, + wire.Bind(new(HandlerService), new(*HandlerServiceImpl)), ) diff --git a/pkg/bulkAction/service/BulkUpdateService.go b/pkg/bulkAction/service/BulkUpdateService.go index afd8efeb41..3e6c0f7f78 100644 --- a/pkg/bulkAction/service/BulkUpdateService.go +++ b/pkg/bulkAction/service/BulkUpdateService.go @@ -103,7 +103,7 @@ type BulkUpdateServiceImpl struct { chartRefService chartRef.ChartRefService deployedAppService deployedApp.DeployedAppService cdPipelineEventPublishService out.CDPipelineEventPublishService - ciTriggerService trigger.Service + ciHandlerService trigger.HandlerService } func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateRepository, @@ -124,7 +124,7 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateReposito chartRefService chartRef.ChartRefService, deployedAppService deployedApp.DeployedAppService, cdPipelineEventPublishService out.CDPipelineEventPublishService, - ciTriggerService trigger.Service) *BulkUpdateServiceImpl { + ciHandlerService trigger.HandlerService) *BulkUpdateServiceImpl { return &BulkUpdateServiceImpl{ bulkUpdateRepository: bulkUpdateRepository, logger: logger, @@ -144,7 +144,7 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateReposito chartRefService: chartRefService, deployedAppService: deployedAppService, cdPipelineEventPublishService: cdPipelineEventPublishService, - ciTriggerService: ciTriggerService, + ciHandlerService: ciHandlerService, } } @@ -1471,7 +1471,7 @@ func (impl BulkUpdateServiceImpl) BulkBuildTrigger(request *bean4.BulkApplicatio } ciTriggerRequest := latestCommitsMap[pipeline.CiPipelineId] - _, err = impl.ciTriggerService.HandleCIManual(ciTriggerRequest) + _, err = impl.ciHandlerService.HandleCIManual(ciTriggerRequest) if err != nil { impl.logger.Errorw("service err, HandleCIManual", "err", err, "ciTriggerRequest", ciTriggerRequest) //return nil, err diff --git a/pkg/deployment/deployedApp/DeployedAppService.go b/pkg/deployment/deployedApp/DeployedAppService.go index f8c65d2373..83107b22fa 100644 --- a/pkg/deployment/deployedApp/DeployedAppService.go +++ b/pkg/deployment/deployedApp/DeployedAppService.go @@ -46,7 +46,7 @@ type DeployedAppService interface { type DeployedAppServiceImpl struct { logger *zap.SugaredLogger k8sCommonService k8s.K8sCommonService - cdTriggerService devtronApps.TriggerService + cdHandlerService devtronApps.HandlerService envRepository repository.EnvironmentRepository pipelineRepository pipelineConfig.PipelineRepository cdWorkflowRepository pipelineConfig.CdWorkflowRepository @@ -54,14 +54,14 @@ type DeployedAppServiceImpl struct { func NewDeployedAppServiceImpl(logger *zap.SugaredLogger, k8sCommonService k8s.K8sCommonService, - cdTriggerService devtronApps.TriggerService, + cdHandlerService devtronApps.HandlerService, envRepository repository.EnvironmentRepository, pipelineRepository pipelineConfig.PipelineRepository, cdWorkflowRepository pipelineConfig.CdWorkflowRepository) *DeployedAppServiceImpl { return &DeployedAppServiceImpl{ logger: logger, k8sCommonService: k8sCommonService, - cdTriggerService: cdTriggerService, + cdHandlerService: cdHandlerService, envRepository: envRepository, pipelineRepository: pipelineRepository, cdWorkflowRepository: cdWorkflowRepository, @@ -127,7 +127,7 @@ func (impl *DeployedAppServiceImpl) stopStartApp(ctx context.Context, stopReques Context: ctx, ReferenceId: stopRequest.ReferenceId, } - id, _, _, err := impl.cdTriggerService.ManualCdTrigger(triggerContext, overrideRequest) + id, _, _, err := impl.cdHandlerService.ManualCdTrigger(triggerContext, overrideRequest) if err != nil { impl.logger.Errorw("error in stopping app", "err", err, "appId", stopRequest.AppId, "envId", stopRequest.EnvironmentId) return 0, err diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/HandlerService.go similarity index 96% rename from pkg/deployment/trigger/devtronApps/TriggerService.go rename to pkg/deployment/trigger/devtronApps/HandlerService.go index bcef986aa7..7c915a3195 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService.go +++ b/pkg/deployment/trigger/devtronApps/HandlerService.go @@ -103,7 +103,7 @@ import ( "time" ) -type TriggerService interface { +type HandlerService interface { TriggerPostStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) TriggerPreStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) @@ -121,7 +121,7 @@ type TriggerService interface { GetRunningWorkflowLogs(environmentId int, pipelineId int, workflowId int) (*bufio.Reader, func() error, error) } -type TriggerServiceImpl struct { +type HandlerServiceImpl struct { logger *zap.SugaredLogger cdWorkflowCommonService cd.CdWorkflowCommonService gitOpsManifestPushService publish.GitOpsPushService @@ -184,7 +184,7 @@ type TriggerServiceImpl struct { blobConfigStorageService pipeline.BlobStorageConfigService } -func NewTriggerServiceImpl(logger *zap.SugaredLogger, +func NewHandlerServiceImpl(logger *zap.SugaredLogger, cdWorkflowCommonService cd.CdWorkflowCommonService, gitOpsManifestPushService publish.GitOpsPushService, gitOpsConfigReadService config.GitOpsConfigReadService, @@ -243,8 +243,8 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, ciLogService pipeline.CiLogService, workflowService executor.WorkflowService, blobConfigStorageService pipeline.BlobStorageConfigService, -) (*TriggerServiceImpl, error) { - impl := &TriggerServiceImpl{ +) (*HandlerServiceImpl, error) { + impl := &HandlerServiceImpl{ logger: logger, cdWorkflowCommonService: cdWorkflowCommonService, gitOpsManifestPushService: gitOpsManifestPushService, @@ -318,7 +318,7 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, return impl, nil } -func (impl *TriggerServiceImpl) TriggerStageForBulk(triggerRequest bean.TriggerRequest) error { +func (impl *HandlerServiceImpl) TriggerStageForBulk(triggerRequest bean.TriggerRequest) error { preStage, err := impl.pipelineStageService.GetCdStageByCdPipelineIdAndStageType(triggerRequest.Pipeline.Id, repository.PIPELINE_STAGE_TYPE_PRE_CD, false) if err != nil && err != pg.ErrNoRows { @@ -352,8 +352,8 @@ func (impl *TriggerServiceImpl) TriggerStageForBulk(triggerRequest bean.TriggerR } } -func (impl *TriggerServiceImpl) getCdPipelineForManualCdTrigger(ctx context.Context, pipelineId int) (*pipelineConfig.Pipeline, error) { - _, span := otel.Tracer("TriggerService").Start(ctx, "getCdPipelineForManualCdTrigger") +func (impl *HandlerServiceImpl) getCdPipelineForManualCdTrigger(ctx context.Context, pipelineId int) (*pipelineConfig.Pipeline, error) { + _, span := otel.Tracer("HandlerService").Start(ctx, "getCdPipelineForManualCdTrigger") defer span.End() cdPipeline, err := impl.pipelineRepository.FindById(pipelineId) if err != nil { @@ -373,8 +373,8 @@ func (impl *TriggerServiceImpl) getCdPipelineForManualCdTrigger(ctx context.Cont return cdPipeline, nil } -func (impl *TriggerServiceImpl) validateDeploymentTriggerRequest(ctx context.Context, validateDeploymentTriggerObj *bean.ValidateDeploymentTriggerObj) error { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.validateDeploymentTriggerRequest") +func (impl *HandlerServiceImpl) validateDeploymentTriggerRequest(ctx context.Context, validateDeploymentTriggerObj *bean.ValidateDeploymentTriggerObj) error { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.validateDeploymentTriggerRequest") defer span.End() // custom GitOps repo url validation --> Start err := impl.handleCustomGitOpsRepoValidation(validateDeploymentTriggerObj.Runner, validateDeploymentTriggerObj.CdPipeline, validateDeploymentTriggerObj.DeploymentConfig, validateDeploymentTriggerObj.TriggeredBy) @@ -406,7 +406,7 @@ func (impl *TriggerServiceImpl) validateDeploymentTriggerRequest(ctx context.Con } // TODO: write a wrapper to handle auto and manual trigger -func (impl *TriggerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerContext, overrideRequest *bean3.ValuesOverrideRequest) (int, string, *bean4.ManifestPushTemplate, error) { +func (impl *HandlerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerContext, overrideRequest *bean3.ValuesOverrideRequest) (int, string, *bean4.ManifestPushTemplate, error) { triggerContext.TriggerType = bean.Manual // setting triggeredAt variable to have consistent data for various audit log places in db for deployment time @@ -628,7 +628,7 @@ func isNotHibernateRequest(deploymentType models.DeploymentType) bool { } // TODO: write a wrapper to handle auto and manual trigger -func (impl *TriggerServiceImpl) TriggerAutomaticDeployment(request bean.TriggerRequest) error { +func (impl *HandlerServiceImpl) TriggerAutomaticDeployment(request bean.TriggerRequest) error { // in case of manual trigger auth is already applied and for auto triggers there is no need for auth check here triggeredBy := request.TriggeredBy pipeline := request.Pipeline @@ -705,7 +705,7 @@ func (impl *TriggerServiceImpl) TriggerAutomaticDeployment(request bean.TriggerR return nil } -func (impl *TriggerServiceImpl) TriggerCD(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) error { +func (impl *HandlerServiceImpl) TriggerCD(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) error { impl.logger.Debugw("automatic pipeline trigger attempt async", "artifactId", artifact.Id) err := impl.triggerReleaseAsync(ctx, artifact, cdWorkflowId, wfrId, pipeline, envDeploymentConfig, triggeredAt, triggeredBy) if err != nil { @@ -715,7 +715,7 @@ func (impl *TriggerServiceImpl) TriggerCD(ctx context.Context, artifact *reposit return err } -func (impl *TriggerServiceImpl) triggerReleaseAsync(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) error { +func (impl *HandlerServiceImpl) triggerReleaseAsync(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) error { err := impl.validateAndTrigger(ctx, pipeline, envDeploymentConfig, artifact, cdWorkflowId, wfrId, triggeredAt, triggeredBy) if err != nil { impl.logger.Errorw("error in trigger for pipeline", "pipelineId", strconv.Itoa(pipeline.Id)) @@ -724,7 +724,7 @@ func (impl *TriggerServiceImpl) triggerReleaseAsync(ctx context.Context, artifac return err } -func (impl *TriggerServiceImpl) validateAndTrigger(ctx context.Context, p *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time, triggeredBy int32) error { +func (impl *HandlerServiceImpl) validateAndTrigger(ctx context.Context, p *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time, triggeredBy int32) error { //TODO: verify this logic object := impl.enforcerUtil.GetAppRBACNameByAppId(p.AppId) envApp := strings.Split(object, "/") @@ -736,7 +736,7 @@ func (impl *TriggerServiceImpl) validateAndTrigger(ctx context.Context, p *pipel return err } -func (impl *TriggerServiceImpl) releasePipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time, triggeredBy int32) error { +func (impl *HandlerServiceImpl) releasePipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time, triggeredBy int32) error { startTime := time.Now() defer func() { impl.logger.Debugw("auto trigger release process completed", "timeTaken", time.Since(startTime), "cdPipelineId", pipeline.Id, "artifactId", artifact.Id, "wfrId", wfrId) @@ -770,9 +770,9 @@ func (impl *TriggerServiceImpl) releasePipeline(ctx context.Context, pipeline *p return err } -func (impl *TriggerServiceImpl) triggerAsyncRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, +func (impl *HandlerServiceImpl) triggerAsyncRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, userDeploymentRequestId int, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.triggerAsyncRelease") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.triggerAsyncRelease") defer span.End() // build merged values and save PCO history for the release valuesOverrideResponse, err := impl.manifestCreationService.GetValuesOverrideForTrigger(newCtx, overrideRequest, envDeploymentConfig, triggeredAt) @@ -791,8 +791,8 @@ func (impl *TriggerServiceImpl) triggerAsyncRelease(ctx context.Context, overrid return impl.workflowEventPublishService.TriggerAsyncRelease(userDeploymentRequestId, overrideRequest, valuesOverrideResponse, newCtx, deployedBy) } -func (impl *TriggerServiceImpl) handleCDTriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.handleCDTriggerRelease") +func (impl *HandlerServiceImpl) handleCDTriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.handleCDTriggerRelease") defer span.End() // Handling for auto trigger if overrideRequest.UserId == 0 { @@ -840,7 +840,7 @@ func (impl *TriggerServiceImpl) handleCDTriggerRelease(ctx context.Context, over return impl.TriggerRelease(newCtx, overrideRequest, envDeploymentConfig, triggeredAt, deployedBy) } -func (impl *TriggerServiceImpl) auditDeploymentTriggerHistory(cdWfrId int, valuesOverrideResponse *app.ValuesOverrideResponse, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (err error) { +func (impl *HandlerServiceImpl) auditDeploymentTriggerHistory(cdWfrId int, valuesOverrideResponse *app.ValuesOverrideResponse, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (err error) { if valuesOverrideResponse.Pipeline == nil || valuesOverrideResponse.EnvOverride == nil { impl.logger.Warnw("unable to save histories for deployment trigger, invalid valuesOverrideResponse received", "cdWfrId", cdWfrId) return nil @@ -854,9 +854,9 @@ func (impl *TriggerServiceImpl) auditDeploymentTriggerHistory(cdWfrId int, value } // TriggerRelease will trigger Install/Upgrade request for Devtron App releases synchronously -func (impl *TriggerServiceImpl) TriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, +func (impl *HandlerServiceImpl) TriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.TriggerRelease") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.TriggerRelease") defer span.End() triggerEvent, skipRequest, err := impl.buildTriggerEventForOverrideRequest(overrideRequest, triggeredAt) if err != nil { @@ -908,10 +908,10 @@ func (impl *TriggerServiceImpl) TriggerRelease(ctx context.Context, overrideRequ return releaseNo, valuesOverrideResponse.ManifestPushTemplate, nil } -func (impl *TriggerServiceImpl) performGitOps(ctx context.Context, +func (impl *HandlerServiceImpl) performGitOps(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, triggerEvent bean.TriggerEvent) error { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.performGitOps") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.performGitOps") defer span.End() // update workflow runner status, used in app workflow view err := impl.cdWorkflowCommonService.UpdateNonTerminalStatusInRunner(newCtx, overrideRequest.WfrId, overrideRequest.UserId, cdWorkflow.WorkflowInProgress) @@ -938,7 +938,7 @@ func (impl *TriggerServiceImpl) performGitOps(ctx context.Context, return nil } -func (impl *TriggerServiceImpl) buildTriggerEventForOverrideRequest(overrideRequest *bean3.ValuesOverrideRequest, triggeredAt time.Time) (triggerEvent bean.TriggerEvent, skipRequest bool, err error) { +func (impl *HandlerServiceImpl) buildTriggerEventForOverrideRequest(overrideRequest *bean3.ValuesOverrideRequest, triggeredAt time.Time) (triggerEvent bean.TriggerEvent, skipRequest bool, err error) { triggerEvent = helper.NewTriggerEvent(overrideRequest.DeploymentAppType, triggeredAt, overrideRequest.UserId) request := statusBean.NewTimelineGetRequest(). WithCdWfrId(overrideRequest.WfrId). @@ -973,8 +973,8 @@ func (impl *TriggerServiceImpl) buildTriggerEventForOverrideRequest(overrideRequ return triggerEvent, skipRequest, nil } -func (impl *TriggerServiceImpl) triggerPipeline(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, triggerEvent bean.TriggerEvent, ctx context.Context) (releaseNo int, err error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.triggerPipeline") +func (impl *HandlerServiceImpl) triggerPipeline(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, triggerEvent bean.TriggerEvent, ctx context.Context) (releaseNo int, err error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.triggerPipeline") defer span.End() if triggerEvent.PerformChartPush { impl.logger.Debugw("performing chart push operation in triggerPipeline", "cdWfrId", overrideRequest.WfrId) @@ -1010,7 +1010,7 @@ func (impl *TriggerServiceImpl) triggerPipeline(overrideRequest *bean3.ValuesOve return valuesOverrideResponse.PipelineOverride.PipelineReleaseCounter, nil } -func (impl *TriggerServiceImpl) buildManifestPushTemplate(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string) (*bean4.ManifestPushTemplate, error) { +func (impl *HandlerServiceImpl) buildManifestPushTemplate(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string) (*bean4.ManifestPushTemplate, error) { manifestPushTemplate := &bean4.ManifestPushTemplate{ WorkflowRunnerId: overrideRequest.WfrId, @@ -1057,8 +1057,8 @@ func (impl *TriggerServiceImpl) buildManifestPushTemplate(overrideRequest *bean3 return manifestPushTemplate, nil } -func (impl *TriggerServiceImpl) deployApp(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, triggerEvent bean.TriggerEvent) error { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.deployApp") +func (impl *HandlerServiceImpl) deployApp(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, triggerEvent bean.TriggerEvent) error { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.deployApp") defer span.End() var referenceChartByte []byte var err error @@ -1080,8 +1080,8 @@ func (impl *TriggerServiceImpl) deployApp(ctx context.Context, overrideRequest * return nil } -func (impl *TriggerServiceImpl) createHelmAppForCdPipeline(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse) (bool, []byte, error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.createHelmAppForCdPipeline") +func (impl *HandlerServiceImpl) createHelmAppForCdPipeline(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse) (bool, []byte, error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.createHelmAppForCdPipeline") defer span.End() pipelineModel := valuesOverrideResponse.Pipeline envOverride := valuesOverrideResponse.EnvOverride @@ -1186,7 +1186,7 @@ func (impl *TriggerServiceImpl) createHelmAppForCdPipeline(ctx context.Context, return true, referenceChartByte, nil } -func (impl *TriggerServiceImpl) getHelmHistoryLimitAndChartMetadataForHelmAppCreation(ctx context.Context, +func (impl *HandlerServiceImpl) getHelmHistoryLimitAndChartMetadataForHelmAppCreation(ctx context.Context, valuesOverrideResponse *app.ValuesOverrideResponse) (*chart.Metadata, int32, *gRPC.ReleaseIdentifier, error) { pipelineModel := valuesOverrideResponse.Pipeline envOverride := valuesOverrideResponse.EnvOverride @@ -1235,9 +1235,9 @@ func (impl *TriggerServiceImpl) getHelmHistoryLimitAndChartMetadataForHelmAppCre return chartMetaData, helmRevisionHistory, releaseIdentifier, nil } -func (impl *TriggerServiceImpl) deployArgoCdApp(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, +func (impl *HandlerServiceImpl) deployArgoCdApp(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse) error { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.deployArgoCdApp") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.deployArgoCdApp") defer span.End() name, err := impl.createArgoApplicationIfRequired(newCtx, valuesOverrideResponse.EnvOverride, valuesOverrideResponse.Pipeline, valuesOverrideResponse.DeploymentConfig, overrideRequest.UserId) if err != nil { @@ -1281,7 +1281,7 @@ func (impl *TriggerServiceImpl) deployArgoCdApp(ctx context.Context, overrideReq } // update repoUrl, revision and argo app sync mode (auto/manual) if needed -func (impl *TriggerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envOverride *bean10.EnvConfigOverride, deploymentConfig *bean9.DeploymentConfig) (bool, error) { +func (impl *HandlerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envOverride *bean10.EnvConfigOverride, deploymentConfig *bean9.DeploymentConfig) (bool, error) { if !deploymentConfig.IsArgoAppPatchSupported() { impl.logger.Infow("argo app patch not supported", "pipelineId", pipeline.Id, "pipelineName", pipeline.Name) return false, nil @@ -1290,7 +1290,7 @@ func (impl *TriggerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline impl.logger.Errorw("err in syncing ACD, ctx is NULL", "pipelineId", pipeline.Id, "pipelineName", pipeline.Name) return false, nil } - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.updateArgoPipeline") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.updateArgoPipeline") defer span.End() argoAppName := pipeline.DeploymentAppName impl.logger.Infow("received payload, updateArgoPipeline", "appId", pipeline.AppId, "pipelineName", pipeline.Name, "envId", envOverride.TargetEnvironment, "argoAppName", argoAppName) @@ -1346,9 +1346,9 @@ func (impl *TriggerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline } } -func (impl *TriggerServiceImpl) createArgoApplicationIfRequired(ctx context.Context, envConfigOverride *bean10.EnvConfigOverride, +func (impl *HandlerServiceImpl) createArgoApplicationIfRequired(ctx context.Context, envConfigOverride *bean10.EnvConfigOverride, pipeline *pipelineConfig.Pipeline, deploymentConfig *bean9.DeploymentConfig, userId int32) (string, error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.createArgoApplicationIfRequired") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.createArgoApplicationIfRequired") defer span.End() envModel, err := impl.envRepository.FindById(envConfigOverride.TargetEnvironment) if err != nil { @@ -1395,7 +1395,7 @@ func (impl *TriggerServiceImpl) createArgoApplicationIfRequired(ctx context.Cont } } -func (impl *TriggerServiceImpl) updatePipeline(pipeline *pipelineConfig.Pipeline, userId int32) (bool, error) { +func (impl *HandlerServiceImpl) updatePipeline(pipeline *pipelineConfig.Pipeline, userId int32) (bool, error) { err := impl.pipelineRepository.SetDeploymentAppCreatedInPipeline(true, pipeline.Id, userId) if err != nil { impl.logger.Errorw("error on updating cd pipeline for setting deployment app created", "err", err) @@ -1405,9 +1405,9 @@ func (impl *TriggerServiceImpl) updatePipeline(pipeline *pipelineConfig.Pipeline } // helmInstallReleaseWithCustomChart performs helm install with custom chart -func (impl *TriggerServiceImpl) helmInstallReleaseWithCustomChart(ctx context.Context, releaseIdentifier *gRPC.ReleaseIdentifier, +func (impl *HandlerServiceImpl) helmInstallReleaseWithCustomChart(ctx context.Context, releaseIdentifier *gRPC.ReleaseIdentifier, referenceChartByte []byte, valuesYaml, k8sServerVersion string, forceSync bool) (*gRPC.HelmInstallCustomResponse, error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.helmInstallReleaseWithCustomChart") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.helmInstallReleaseWithCustomChart") defer span.End() helmInstallRequest := gRPC.HelmInstallCustomRequest{ ValuesYaml: valuesYaml, @@ -1424,7 +1424,7 @@ func (impl *TriggerServiceImpl) helmInstallReleaseWithCustomChart(ctx context.Co return impl.helmAppClient.InstallReleaseWithCustomChart(newCtx, &helmInstallRequest) } -func (impl *TriggerServiceImpl) writeCDTriggerEvent(overrideRequest *bean3.ValuesOverrideRequest, artifact *repository3.CiArtifact, releaseId, pipelineOverrideId, wfrId int) { +func (impl *HandlerServiceImpl) writeCDTriggerEvent(overrideRequest *bean3.ValuesOverrideRequest, artifact *repository3.CiArtifact, releaseId, pipelineOverrideId, wfrId int) { event, err := impl.eventFactory.Build(util2.Trigger, &overrideRequest.PipelineId, overrideRequest.AppId, &overrideRequest.EnvId, util2.CD) if err != nil { @@ -1467,9 +1467,9 @@ func (impl *TriggerServiceImpl) writeCDTriggerEvent(overrideRequest *bean3.Value } } -func (impl *TriggerServiceImpl) markImageScanDeployed(ctx context.Context, appId, envId, clusterId int, +func (impl *HandlerServiceImpl) markImageScanDeployed(ctx context.Context, appId, envId, clusterId int, imageDigest string, isScanEnabled bool, image string) error { - _, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.markImageScanDeployed") + _, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.markImageScanDeployed") defer span.End() // TODO KB: send NATS event for self consumption impl.logger.Debugw("mark image scan deployed for devtron app, from cd auto or manual trigger", "imageDigest", imageDigest) @@ -1537,11 +1537,11 @@ func (impl *TriggerServiceImpl) markImageScanDeployed(ctx context.Context, appId return err } -func (impl *TriggerServiceImpl) isDevtronAsyncHelmInstallModeEnabled(forceSync bool) bool { +func (impl *HandlerServiceImpl) isDevtronAsyncHelmInstallModeEnabled(forceSync bool) bool { return impl.globalEnvVariables.EnableAsyncHelmInstallDevtronChart && !forceSync } -func (impl *TriggerServiceImpl) isDevtronAsyncInstallModeEnabled(overrideRequest *bean3.ValuesOverrideRequest) (bool, error) { +func (impl *HandlerServiceImpl) isDevtronAsyncInstallModeEnabled(overrideRequest *bean3.ValuesOverrideRequest) (bool, error) { if util.IsHelmApp(overrideRequest.DeploymentAppType) { return impl.isDevtronAsyncHelmInstallModeEnabled(overrideRequest.ForceSyncDeployment), nil } else if util.IsAcdApp(overrideRequest.DeploymentAppType) { @@ -1551,7 +1551,7 @@ func (impl *TriggerServiceImpl) isDevtronAsyncInstallModeEnabled(overrideRequest return false, nil } -func (impl *TriggerServiceImpl) deleteCorruptedPipelineStage(pipelineStage *repository.PipelineStage, triggeredBy int32) (error, bool) { +func (impl *HandlerServiceImpl) deleteCorruptedPipelineStage(pipelineStage *repository.PipelineStage, triggeredBy int32) (error, bool) { if pipelineStage != nil { stageReq := &bean8.PipelineStageDto{ Id: pipelineStage.Id, @@ -1567,7 +1567,7 @@ func (impl *TriggerServiceImpl) deleteCorruptedPipelineStage(pipelineStage *repo return nil, false } -func (impl *TriggerServiceImpl) handleCustomGitOpsRepoValidation(runner *pipelineConfig.CdWorkflowRunner, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredBy int32) error { +func (impl *HandlerServiceImpl) handleCustomGitOpsRepoValidation(runner *pipelineConfig.CdWorkflowRunner, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredBy int32) error { if !util.IsAcdApp(pipeline.DeploymentAppName) { return nil } @@ -1600,7 +1600,7 @@ func (impl *TriggerServiceImpl) handleCustomGitOpsRepoValidation(runner *pipelin return nil } -func (impl *TriggerServiceImpl) getSanitizedK8sVersion(referenceTemplate string) (string, error) { +func (impl *HandlerServiceImpl) getSanitizedK8sVersion(referenceTemplate string) (string, error) { var sanitizedK8sVersion string //handle specific case for all cronjob charts from cronjob-chart_1-2-0 to cronjob-chart_1-5-0 where semverCompare //comparison func has wrong api version mentioned, so for already installed charts via these charts that comparison @@ -1617,7 +1617,7 @@ func (impl *TriggerServiceImpl) getSanitizedK8sVersion(referenceTemplate string) return sanitizedK8sVersion, nil } -func (impl *TriggerServiceImpl) getReferenceChartByteForHelmTypeApp(envOverride *bean10.EnvConfigOverride, +func (impl *HandlerServiceImpl) getReferenceChartByteForHelmTypeApp(envOverride *bean10.EnvConfigOverride, chartMetaData *chart.Metadata, referenceTemplatePath string, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse) ([]byte, error) { referenceChartByte := envOverride.Chart.ReferenceChart diff --git a/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go b/pkg/deployment/trigger/devtronApps/HandlerService_ent1.go similarity index 79% rename from pkg/deployment/trigger/devtronApps/TriggerService_ent1.go rename to pkg/deployment/trigger/devtronApps/HandlerService_ent1.go index f008f72adc..24b7dab264 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go +++ b/pkg/deployment/trigger/devtronApps/HandlerService_ent1.go @@ -17,29 +17,29 @@ import ( "helm.sh/helm/v3/pkg/chart" ) -func (impl *TriggerServiceImpl) getEnrichedWorkflowRunner(overrideRequest *bean3.ValuesOverrideRequest, artifact *repository3.CiArtifact, wfrId int) *pipelineConfig.CdWorkflowRunner { +func (impl *HandlerServiceImpl) getEnrichedWorkflowRunner(overrideRequest *bean3.ValuesOverrideRequest, artifact *repository3.CiArtifact, wfrId int) *pipelineConfig.CdWorkflowRunner { return nil } -func (impl *TriggerServiceImpl) postDeployHook(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, referenceChartByte []byte, err error) { +func (impl *HandlerServiceImpl) postDeployHook(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, referenceChartByte []byte, err error) { impl.logger.Debugw("no post deploy hook registered") } -func (impl *TriggerServiceImpl) isDevtronAsyncArgoCdInstallModeEnabledForApp(appId, envId int, forceSync bool) (bool, error) { +func (impl *HandlerServiceImpl) isDevtronAsyncArgoCdInstallModeEnabledForApp(appId, envId int, forceSync bool) (bool, error) { return impl.globalEnvVariables.EnableAsyncArgoCdInstallDevtronChart && !forceSync, nil } -func (impl *TriggerServiceImpl) getClusterGRPCConfig(cluster repository2.Cluster) *gRPC.ClusterConfig { +func (impl *HandlerServiceImpl) getClusterGRPCConfig(cluster repository2.Cluster) *gRPC.ClusterConfig { clusterConfig := helper.ConvertClusterBeanToGrpcConfig(cluster) return clusterConfig } -func (impl *TriggerServiceImpl) overrideReferenceChartByteForHelmTypeApp(valuesOverrideResponse *app.ValuesOverrideResponse, +func (impl *HandlerServiceImpl) overrideReferenceChartByteForHelmTypeApp(valuesOverrideResponse *app.ValuesOverrideResponse, chartMetaData *chart.Metadata, referenceTemplatePath string, referenceChartByte []byte) ([]byte, error) { return referenceChartByte, nil } -func (impl *TriggerServiceImpl) getManifestPushService(storageType string) publish.ManifestPushService { +func (impl *HandlerServiceImpl) getManifestPushService(storageType string) publish.ManifestPushService { var manifestPushService publish.ManifestPushService if storageType == bean2.ManifestStorageGit { manifestPushService = impl.gitOpsManifestPushService @@ -47,25 +47,25 @@ func (impl *TriggerServiceImpl) getManifestPushService(storageType string) publi return manifestPushService } -func (impl *TriggerServiceImpl) preStageHandlingForTriggerStageInBulk(triggerRequest *bean.TriggerRequest) error { +func (impl *HandlerServiceImpl) preStageHandlingForTriggerStageInBulk(triggerRequest *bean.TriggerRequest) error { return nil } -func (impl *TriggerServiceImpl) manifestGenerationFailedTimelineHandling(triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, err error) { +func (impl *HandlerServiceImpl) manifestGenerationFailedTimelineHandling(triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, err error) { } -func (impl *TriggerServiceImpl) getHelmManifestForTriggerRelease(ctx context.Context, triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, +func (impl *HandlerServiceImpl) getHelmManifestForTriggerRelease(ctx context.Context, triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string) ([]byte, error) { return nil, nil } -func (impl *TriggerServiceImpl) buildManifestPushTemplateForNonGitStorageType(overrideRequest *bean3.ValuesOverrideRequest, +func (impl *HandlerServiceImpl) buildManifestPushTemplateForNonGitStorageType(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, err error, manifestPushConfig *repository.ManifestPushConfig, manifestPushTemplate *bean4.ManifestPushTemplate) error { return nil } -func (impl *TriggerServiceImpl) triggerReleaseSuccessHandling(triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, +func (impl *HandlerServiceImpl) triggerReleaseSuccessHandling(triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, helmManifest []byte) error { return nil } diff --git a/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go index defe63b75d..1bf2399ffc 100644 --- a/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go @@ -30,7 +30,7 @@ import ( "time" ) -func (impl *TriggerServiceImpl) TriggerPostStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) { +func (impl *HandlerServiceImpl) TriggerPostStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) { request.WorkflowType = bean2.CD_WORKFLOW_TYPE_POST // setting triggeredAt variable to have consistent data for various audit log places in db for deployment time triggeredAt := time.Now() @@ -164,7 +164,7 @@ func (impl *TriggerServiceImpl) TriggerPostStage(request bean.TriggerRequest) (* return manifestPushTempate, nil } -func (impl *TriggerServiceImpl) buildWfRequestErrorHandler(runner *pipelineConfig.CdWorkflowRunner, err error, triggeredBy int32) (*bean4.ManifestPushTemplate, error) { +func (impl *HandlerServiceImpl) buildWfRequestErrorHandler(runner *pipelineConfig.CdWorkflowRunner, err error, triggeredBy int32) (*bean4.ManifestPushTemplate, error) { dbErr := impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, err, triggeredBy) if dbErr != nil { impl.logger.Errorw("error while updating current runner status to failed, buildWfRequestErrorHandler", "runner", runner.Id, "err", dbErr, "releaseErr", err) diff --git a/pkg/deployment/trigger/devtronApps/PostStageTriggerService_ent.go b/pkg/deployment/trigger/devtronApps/PostStageTriggerService_ent.go index d99b813b1d..197aecd22d 100644 --- a/pkg/deployment/trigger/devtronApps/PostStageTriggerService_ent.go +++ b/pkg/deployment/trigger/devtronApps/PostStageTriggerService_ent.go @@ -26,13 +26,13 @@ import ( "time" ) -func (impl *TriggerServiceImpl) checkFeasibilityForPostStage(pipeline *pipelineConfig.Pipeline, request *bean.TriggerRequest, +func (impl *HandlerServiceImpl) checkFeasibilityForPostStage(pipeline *pipelineConfig.Pipeline, request *bean.TriggerRequest, env *repository.Environment, cdWf *pipelineConfig.CdWorkflow, triggeredBy int32) (interface{}, error) { //here return type is interface as ResourceFilterEvaluationAudit is not present in this version return nil, nil } -func (impl *TriggerServiceImpl) getManifestPushTemplateForPostStage(request bean.TriggerRequest, envDevploymentConfig *bean5.DeploymentConfig, +func (impl *HandlerServiceImpl) getManifestPushTemplateForPostStage(request bean.TriggerRequest, envDevploymentConfig *bean5.DeploymentConfig, jobHelmPackagePath string, cdStageWorkflowRequest *types.WorkflowRequest, cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner, pipeline *pipelineConfig.Pipeline, triggeredBy int32, triggeredAt time.Time) (*bean4.ManifestPushTemplate, error) { return nil, nil diff --git a/pkg/deployment/trigger/devtronApps/PrePostCommonService.go b/pkg/deployment/trigger/devtronApps/PrePostCommonService.go index d1cf588032..d3ff8b5e5f 100644 --- a/pkg/deployment/trigger/devtronApps/PrePostCommonService.go +++ b/pkg/deployment/trigger/devtronApps/PrePostCommonService.go @@ -26,7 +26,7 @@ import ( "time" ) -func (impl *TriggerServiceImpl) CancelStage(workflowRunnerId int, forceAbort bool, userId int32) (int, error) { +func (impl *HandlerServiceImpl) CancelStage(workflowRunnerId int, forceAbort bool, userId int32) (int, error) { workflowRunner, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(workflowRunnerId) if err != nil { impl.logger.Errorw("err", "err", err) @@ -113,7 +113,7 @@ func (impl *TriggerServiceImpl) CancelStage(workflowRunnerId int, forceAbort boo return workflowRunner.Id, nil } -func (impl *TriggerServiceImpl) updateWorkflowRunnerForForceAbort(workflowRunner *pipelineConfig.CdWorkflowRunner) error { +func (impl *HandlerServiceImpl) updateWorkflowRunnerForForceAbort(workflowRunner *pipelineConfig.CdWorkflowRunner) error { workflowRunner.Status = cdWorkflow2.WorkflowCancel workflowRunner.PodStatus = string(bean2.Failed) workflowRunner.Message = constants.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE @@ -125,7 +125,7 @@ func (impl *TriggerServiceImpl) updateWorkflowRunnerForForceAbort(workflowRunner return nil } -func (impl *TriggerServiceImpl) handleForceAbortCaseForCdStage(workflowRunner *pipelineConfig.CdWorkflowRunner, forceAbort bool) error { +func (impl *HandlerServiceImpl) handleForceAbortCaseForCdStage(workflowRunner *pipelineConfig.CdWorkflowRunner, forceAbort bool) error { isWorkflowInNonTerminalStage := workflowRunner.Status == string(v1alpha1.NodePending) || workflowRunner.Status == string(v1alpha1.NodeRunning) if !isWorkflowInNonTerminalStage { if forceAbort { @@ -141,7 +141,7 @@ func (impl *TriggerServiceImpl) handleForceAbortCaseForCdStage(workflowRunner *p return nil } -func (impl *TriggerServiceImpl) DownloadCdWorkflowArtifacts(buildId int) (*os.File, error) { +func (impl *HandlerServiceImpl) DownloadCdWorkflowArtifacts(buildId int) (*os.File, error) { wfr, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(buildId) if err != nil { impl.logger.Errorw("unable to fetch ciWorkflow", "err", err) @@ -224,7 +224,7 @@ func (impl *TriggerServiceImpl) DownloadCdWorkflowArtifacts(buildId int) (*os.Fi return file, nil } -func (impl *TriggerServiceImpl) GetRunningWorkflowLogs(environmentId int, pipelineId int, wfrId int) (*bufio.Reader, func() error, error) { +func (impl *HandlerServiceImpl) GetRunningWorkflowLogs(environmentId int, pipelineId int, wfrId int) (*bufio.Reader, func() error, error) { cdWorkflow, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(wfrId) if err != nil { impl.logger.Errorw("error on fetch wf runner", "err", err) @@ -256,7 +256,7 @@ func (impl *TriggerServiceImpl) GetRunningWorkflowLogs(environmentId int, pipeli return impl.getWorkflowLogs(pipelineId, cdWorkflow, clusterConfig, isExtCluster) } -func (impl *TriggerServiceImpl) getWorkflowLogs(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, runStageInEnv bool) (*bufio.Reader, func() error, error) { +func (impl *HandlerServiceImpl) getWorkflowLogs(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, runStageInEnv bool) (*bufio.Reader, func() error, error) { cdLogRequest := types.BuildLogRequest{ PodName: cdWorkflow.PodName, Namespace: cdWorkflow.Namespace, @@ -281,7 +281,7 @@ func (impl *TriggerServiceImpl) getWorkflowLogs(pipelineId int, cdWorkflow *pipe return logReader, cleanUp, err } -func (impl *TriggerServiceImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { +func (impl *HandlerServiceImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { impl.logger.Debug("getting historic logs", "pipelineId", pipelineId) cdConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() // TODO -fixme diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go index 64c373a235..ad0051e76d 100644 --- a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go @@ -61,7 +61,7 @@ import ( "time" ) -func (impl *TriggerServiceImpl) TriggerPreStage(request bean.TriggerRequest) (*bean6.ManifestPushTemplate, error) { +func (impl *HandlerServiceImpl) TriggerPreStage(request bean.TriggerRequest) (*bean6.ManifestPushTemplate, error) { request.WorkflowType = bean2.CD_WORKFLOW_TYPE_PRE // setting triggeredAt variable to have consistent data for various audit log places in db for deployment time triggeredAt := time.Now() @@ -177,7 +177,7 @@ func (impl *TriggerServiceImpl) TriggerPreStage(request bean.TriggerRequest) (*b return manifestPushTemplate, nil } -func (impl *TriggerServiceImpl) TriggerAutoCDOnPreStageSuccess(triggerContext bean.TriggerContext, cdPipelineId, ciArtifactId, workflowId int) error { +func (impl *HandlerServiceImpl) TriggerAutoCDOnPreStageSuccess(triggerContext bean.TriggerContext, cdPipelineId, ciArtifactId, workflowId int) error { pipeline, err := impl.pipelineRepository.FindById(cdPipelineId) if err != nil { return err @@ -216,7 +216,7 @@ func (impl *TriggerServiceImpl) TriggerAutoCDOnPreStageSuccess(triggerContext be } return nil } -func (impl *TriggerServiceImpl) checkDeploymentTriggeredAlready(wfId int) bool { +func (impl *HandlerServiceImpl) checkDeploymentTriggeredAlready(wfId int) bool { deploymentTriggeredAlready := false // TODO : need to check this logic for status check in case of multiple deployments requirement for same workflow workflowRunner, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(context.Background(), wfId, bean2.CD_WORKFLOW_TYPE_DEPLOY) @@ -228,7 +228,7 @@ func (impl *TriggerServiceImpl) checkDeploymentTriggeredAlready(wfId int) bool { return deploymentTriggeredAlready } -func (impl *TriggerServiceImpl) createStartingWfAndRunner(request bean.TriggerRequest, triggeredAt time.Time) (*pipelineConfig.CdWorkflow, *pipelineConfig.CdWorkflowRunner, error) { +func (impl *HandlerServiceImpl) createStartingWfAndRunner(request bean.TriggerRequest, triggeredAt time.Time) (*pipelineConfig.CdWorkflow, *pipelineConfig.CdWorkflowRunner, error) { triggeredBy := request.TriggeredBy artifact := request.Artifact pipeline := request.Pipeline @@ -272,7 +272,7 @@ func (impl *TriggerServiceImpl) createStartingWfAndRunner(request bean.TriggerRe return cdWf, runner, nil } -func (impl *TriggerServiceImpl) getEnvAndNsIfRunStageInEnv(ctx context.Context, request bean.TriggerRequest) (*repository4.Environment, string, error) { +func (impl *HandlerServiceImpl) getEnvAndNsIfRunStageInEnv(ctx context.Context, request bean.TriggerRequest) (*repository4.Environment, string, error) { workflowStage := request.WorkflowType pipeline := request.Pipeline var env *repository4.Environment @@ -297,7 +297,7 @@ func (impl *TriggerServiceImpl) getEnvAndNsIfRunStageInEnv(ctx context.Context, return env, namespace, nil } -func (impl *TriggerServiceImpl) checkVulnerabilityStatusAndFailWfIfNeeded(ctx context.Context, artifact *repository.CiArtifact, +func (impl *HandlerServiceImpl) checkVulnerabilityStatusAndFailWfIfNeeded(ctx context.Context, artifact *repository.CiArtifact, cdPipeline *pipelineConfig.Pipeline, runner *pipelineConfig.CdWorkflowRunner, triggeredBy int32) error { //checking vulnerability for the selected image vulnerabilityCheckRequest := adapter2.GetVulnerabilityCheckRequest(cdPipeline, artifact.ImageDigest) @@ -324,7 +324,7 @@ func (impl *TriggerServiceImpl) checkVulnerabilityStatusAndFailWfIfNeeded(ctx co } // setCopyContainerImagePluginDataAndReserveImages sets required fields in cdStageWorkflowRequest and reserve images generated by plugin -func (impl *TriggerServiceImpl) setCopyContainerImagePluginDataAndReserveImages(cdStageWorkflowRequest *types.WorkflowRequest, pipelineId int, pipelineStage string, artifact *repository.CiArtifact) ([]int, error) { +func (impl *HandlerServiceImpl) setCopyContainerImagePluginDataAndReserveImages(cdStageWorkflowRequest *types.WorkflowRequest, pipelineId int, pipelineStage string, artifact *repository.CiArtifact) ([]int, error) { copyContainerImagePluginDetail, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(buildCommonBean.COPY_CONTAINER_IMAGE) if err != nil && err != pg.ErrNoRows { @@ -407,7 +407,7 @@ func (impl *TriggerServiceImpl) setCopyContainerImagePluginDataAndReserveImages( return imagePathReservationIds, nil } -func (impl *TriggerServiceImpl) getDockerTagAndCustomTagIdForPlugin(pipelineStage string, pipelineId int, artifact *repository.CiArtifact) (string, int, error) { +func (impl *HandlerServiceImpl) getDockerTagAndCustomTagIdForPlugin(pipelineStage string, pipelineId int, artifact *repository.CiArtifact) (string, int, error) { var pipelineStageEntityType int if pipelineStage == types.PRE { pipelineStageEntityType = pipelineConfigBean.EntityTypePreCD @@ -442,7 +442,7 @@ func (impl *TriggerServiceImpl) getDockerTagAndCustomTagIdForPlugin(pipelineStag return DockerImageTag, customTagId, nil } -func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflowRunner, cdWf *pipelineConfig.CdWorkflow, cdPipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean5.DeploymentConfig, triggeredBy int32) (*types.WorkflowRequest, error) { +func (impl *HandlerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflowRunner, cdWf *pipelineConfig.CdWorkflow, cdPipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean5.DeploymentConfig, triggeredBy int32) (*types.WorkflowRequest, error) { if cdPipeline.App.Id == 0 { appModel, err := impl.appRepository.FindById(cdPipeline.AppId) if err != nil { @@ -910,7 +910,7 @@ getBuildRegistryConfigForArtifact performs the following logic to get Pre/Post C If the ci_pipeline_type type is CI_JOB We will always fetch the registry credentials from the ci_template_override table */ -func (impl *TriggerServiceImpl) getBuildRegistryConfigForArtifact(sourceCiPipeline *pipelineConfig.CiPipeline, artifact *repository.CiArtifact, appId int) (*types.DockerArtifactStoreBean, error) { +func (impl *HandlerServiceImpl) getBuildRegistryConfigForArtifact(sourceCiPipeline *pipelineConfig.CiPipeline, artifact *repository.CiArtifact, appId int) (*types.DockerArtifactStoreBean, error) { var buildRegistryConfig *types.DockerArtifactStoreBean var err error @@ -970,7 +970,7 @@ func (impl *TriggerServiceImpl) getBuildRegistryConfigForArtifact(sourceCiPipeli return buildRegistryConfig, nil } -func (impl *TriggerServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, destinationImages []string) ([]int, error) { +func (impl *HandlerServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, destinationImages []string) ([]int, error) { var imagePathReservationIds []int for _, image := range destinationImages { @@ -1004,7 +1004,7 @@ func setExtraEnvVariableInDeployStep(deploySteps []*pipelineConfigBean.StepObjec } } -func (impl *TriggerServiceImpl) getDeployStageDetails(pipelineId int) (pipelineConfig.CdWorkflowRunner, string, int, error) { +func (impl *HandlerServiceImpl) getDeployStageDetails(pipelineId int) (pipelineConfig.CdWorkflowRunner, string, int, error) { deployStageWfr := pipelineConfig.CdWorkflowRunner{} //getting deployment pipeline latest wfr by pipelineId deployStageWfr, err := impl.cdWorkflowRepository.FindLatestByPipelineIdAndRunnerType(pipelineId, bean2.CD_WORKFLOW_TYPE_DEPLOY) @@ -1025,7 +1025,7 @@ func (impl *TriggerServiceImpl) getDeployStageDetails(pipelineId int) (pipelineC return deployStageWfr, deployStageTriggeredByUserEmail, pipelineReleaseCounter, nil } -func (impl *TriggerServiceImpl) buildArtifactLocationForS3(cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) (string, string, string) { +func (impl *HandlerServiceImpl) buildArtifactLocationForS3(cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) (string, string, string) { cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() cdWorkflowConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() ArtifactLocation := fmt.Sprintf("s3://"+path.Join(cdWorkflowConfigLogsBucket, cdArtifactLocationFormat), cdWf.Id, runner.Id) @@ -1033,7 +1033,7 @@ func (impl *TriggerServiceImpl) buildArtifactLocationForS3(cdWf *pipelineConfig. return ArtifactLocation, cdWorkflowConfigLogsBucket, artifactFileName } -func (impl *TriggerServiceImpl) buildDefaultArtifactLocation(savedWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) string { +func (impl *HandlerServiceImpl) buildDefaultArtifactLocation(savedWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) string { cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() ArtifactLocation := fmt.Sprintf(cdArtifactLocationFormat, savedWf.Id, runner.Id) return ArtifactLocation @@ -1045,7 +1045,7 @@ func ReplaceImageTagWithDigest(image, digest string) string { return imageWithDigest } -func (impl *TriggerServiceImpl) sendPreStageNotification(ctx context.Context, cdWf *pipelineConfig.CdWorkflow, pipeline *pipelineConfig.Pipeline) error { +func (impl *HandlerServiceImpl) sendPreStageNotification(ctx context.Context, cdWf *pipelineConfig.CdWorkflow, pipeline *pipelineConfig.Pipeline) error { wfr, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, cdWf.Id, bean2.CD_WORKFLOW_TYPE_PRE) if err != nil { return err diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService_ent.go b/pkg/deployment/trigger/devtronApps/PreStageTriggerService_ent.go index 8f5b3a5c39..d285ba8224 100644 --- a/pkg/deployment/trigger/devtronApps/PreStageTriggerService_ent.go +++ b/pkg/deployment/trigger/devtronApps/PreStageTriggerService_ent.go @@ -14,33 +14,33 @@ import ( "time" ) -func (impl *TriggerServiceImpl) checkFeasibilityForPreStage(pipeline *pipelineConfig.Pipeline, request *bean2.TriggerRequest, +func (impl *HandlerServiceImpl) checkFeasibilityForPreStage(pipeline *pipelineConfig.Pipeline, request *bean2.TriggerRequest, env *repository.Environment, artifact *repository2.CiArtifact, triggeredBy int32) (interface{}, error) { //here return type is interface as ResourceFilterEvaluationAudit is not present in this version return nil, nil } -func (impl *TriggerServiceImpl) createAuditDataForDeploymentWindowBypass(request bean2.TriggerRequest, wfrId int) error { +func (impl *HandlerServiceImpl) createAuditDataForDeploymentWindowBypass(request bean2.TriggerRequest, wfrId int) error { return nil } -func (impl *TriggerServiceImpl) getManifestPushTemplateForPreStage(ctx context.Context, envDeploymentConfig *bean3.DeploymentConfig, +func (impl *HandlerServiceImpl) getManifestPushTemplateForPreStage(ctx context.Context, envDeploymentConfig *bean3.DeploymentConfig, pipeline *pipelineConfig.Pipeline, artifact *repository2.CiArtifact, jobHelmPackagePath string, cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner, triggeredBy int32, triggeredAt time.Time, request bean2.TriggerRequest) (*bean6.ManifestPushTemplate, error) { return nil, nil } -func (impl *TriggerServiceImpl) setCloningModeInCIProjectDetail(ciProjectDetail *bean.CiProjectDetails, appId int, +func (impl *HandlerServiceImpl) setCloningModeInCIProjectDetail(ciProjectDetail *bean.CiProjectDetails, appId int, m *pipelineConfig.CiPipelineMaterial) error { return nil } -func (impl *TriggerServiceImpl) getPreStageBuildRegistryConfigIfSourcePipelineNotPresent(appId int) (*types.DockerArtifactStoreBean, error) { +func (impl *HandlerServiceImpl) getPreStageBuildRegistryConfigIfSourcePipelineNotPresent(appId int) (*types.DockerArtifactStoreBean, error) { return nil, fmt.Errorf("soucePipeline is mandatory, corrupt data") } -func (impl *TriggerServiceImpl) handlerFilterEvaluationAudit(filterEvaluationAudit interface{}, +func (impl *HandlerServiceImpl) handlerFilterEvaluationAudit(filterEvaluationAudit interface{}, runner *pipelineConfig.CdWorkflowRunner) error { //here ip type of filterEvaluationAudit is interface as ResourceFilterEvaluationAudit is not present in this version return nil diff --git a/pkg/deployment/trigger/devtronApps/feasibilityChecker.go b/pkg/deployment/trigger/devtronApps/feasibilityChecker.go index 43926bb86a..c03968d5de 100644 --- a/pkg/deployment/trigger/devtronApps/feasibilityChecker.go +++ b/pkg/deployment/trigger/devtronApps/feasibilityChecker.go @@ -24,7 +24,7 @@ type FeasibilityManager interface { CheckFeasibility(triggerRequirementRequest *bean.TriggerRequirementRequestDto) error } -func (impl *TriggerServiceImpl) CheckFeasibility(triggerRequirementRequest *bean.TriggerRequirementRequestDto) error { +func (impl *HandlerServiceImpl) CheckFeasibility(triggerRequirementRequest *bean.TriggerRequirementRequestDto) error { // have not implemented right now, will be implemented in future for security vulnerability return nil } diff --git a/pkg/deployment/trigger/devtronApps/wire_devtronAppsDeployTrigger.go b/pkg/deployment/trigger/devtronApps/wire_devtronAppsDeployTrigger.go index 4bf4174cfe..953b46422f 100644 --- a/pkg/deployment/trigger/devtronApps/wire_devtronAppsDeployTrigger.go +++ b/pkg/deployment/trigger/devtronApps/wire_devtronAppsDeployTrigger.go @@ -23,6 +23,6 @@ import ( var DevtronAppsDeployTriggerWireSet = wire.NewSet( userDeploymentRequest.WireSet, - NewTriggerServiceImpl, - wire.Bind(new(TriggerService), new(*TriggerServiceImpl)), + NewHandlerServiceImpl, + wire.Bind(new(HandlerService), new(*HandlerServiceImpl)), ) diff --git a/pkg/eventProcessor/in/CDPipelineEventProcessorService.go b/pkg/eventProcessor/in/CDPipelineEventProcessorService.go index 5a54759107..cb18e00066 100644 --- a/pkg/eventProcessor/in/CDPipelineEventProcessorService.go +++ b/pkg/eventProcessor/in/CDPipelineEventProcessorService.go @@ -38,7 +38,7 @@ type CDPipelineEventProcessorImpl struct { pubSubClient *pubsub.PubSubClientServiceImpl cdWorkflowCommonService cd.CdWorkflowCommonService workflowStatusService status.WorkflowStatusService - cdTriggerService devtronApps.TriggerService + cdHandlerService devtronApps.HandlerService pipelineRepository pipelineConfig.PipelineRepository installedAppReadService installedAppReader.InstalledAppReadService } @@ -47,7 +47,7 @@ func NewCDPipelineEventProcessorImpl(logger *zap.SugaredLogger, pubSubClient *pubsub.PubSubClientServiceImpl, cdWorkflowCommonService cd.CdWorkflowCommonService, workflowStatusService status.WorkflowStatusService, - cdTriggerService devtronApps.TriggerService, + cdHandlerService devtronApps.HandlerService, pipelineRepository pipelineConfig.PipelineRepository, installedAppReadService installedAppReader.InstalledAppReadService) *CDPipelineEventProcessorImpl { cdPipelineEventProcessorImpl := &CDPipelineEventProcessorImpl{ @@ -55,7 +55,7 @@ func NewCDPipelineEventProcessorImpl(logger *zap.SugaredLogger, pubSubClient: pubSubClient, cdWorkflowCommonService: cdWorkflowCommonService, workflowStatusService: workflowStatusService, - cdTriggerService: cdTriggerService, + cdHandlerService: cdHandlerService, pipelineRepository: pipelineRepository, installedAppReadService: installedAppReadService, } @@ -78,7 +78,7 @@ func (impl *CDPipelineEventProcessorImpl) SubscribeCDBulkTriggerTopic() error { ReferenceId: pointer.String(msg.MsgId), Context: context2.Background(), } - _, _, _, err = impl.cdTriggerService.ManualCdTrigger(triggerContext, event.ValuesOverrideRequest) + _, _, _, err = impl.cdHandlerService.ManualCdTrigger(triggerContext, event.ValuesOverrideRequest) if err != nil { impl.logger.Errorw("Error triggering CD", "topic", pubsub.CD_BULK_DEPLOY_TRIGGER_TOPIC, "msg", msg.Data, "err", err) } diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index 5e97d5d99a..d7c1f8a9bb 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -79,7 +79,7 @@ type WorkflowEventProcessorImpl struct { cdHandler pipeline.CdHandler eventFactory client.EventFactory eventClient client.EventClient - cdTriggerService devtronApps.TriggerService + cdHandlerService devtronApps.HandlerService deployedAppService deployedApp.DeployedAppService webhookService pipeline.WebhookService validator *validator.Validate @@ -93,7 +93,7 @@ type WorkflowEventProcessorImpl struct { appServiceConfig *app.AppServiceConfig //ent only - ciTriggerService trigger.Service + ciHandlerService trigger.HandlerService // repositories import to be removed pipelineRepository pipelineConfig.PipelineRepository @@ -111,7 +111,7 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, workflowDagExecutor dag.WorkflowDagExecutor, ciHandler pipeline.CiHandler, cdHandler pipeline.CdHandler, eventFactory client.EventFactory, eventClient client.EventClient, - cdTriggerService devtronApps.TriggerService, + cdHandlerService devtronApps.HandlerService, deployedAppService deployedApp.DeployedAppService, webhookService pipeline.WebhookService, validator *validator.Validate, @@ -123,7 +123,7 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, ciArtifactRepository repository.CiArtifactRepository, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, deploymentConfigService common.DeploymentConfigService, - ciTriggerService trigger.Service) (*WorkflowEventProcessorImpl, error) { + ciHandlerService trigger.HandlerService) (*WorkflowEventProcessorImpl, error) { impl := &WorkflowEventProcessorImpl{ logger: logger, pubSubClient: pubSubClient, @@ -136,7 +136,7 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, eventFactory: eventFactory, eventClient: eventClient, workflowDagExecutor: workflowDagExecutor, - cdTriggerService: cdTriggerService, + cdHandlerService: cdHandlerService, deployedAppService: deployedAppService, webhookService: webhookService, validator: validator, @@ -150,7 +150,7 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, ciArtifactRepository: ciArtifactRepository, cdWorkflowRepository: cdWorkflowRepository, deploymentConfigService: deploymentConfigService, - ciTriggerService: ciTriggerService, + ciHandlerService: ciHandlerService, } appServiceConfig, err := app.GetAppServiceConfig() if err != nil { @@ -326,7 +326,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeTriggerBulkAction() error { ApplyAuth: false, TriggerContext: triggerContext, } - err = impl.cdTriggerService.TriggerStageForBulk(triggerRequest) + err = impl.cdHandlerService.TriggerStageForBulk(triggerRequest) if err != nil { impl.logger.Errorw("error in cd trigger ", "err", err) wf.WorkflowStatus = cdWorkflowModelBean.TRIGGER_ERROR @@ -400,7 +400,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCIWorkflowStatusUpdate() error return } - err = impl.ciTriggerService.CheckAndReTriggerCI(wfStatus) + err = impl.ciHandlerService.CheckAndReTriggerCI(wfStatus) if err != nil { impl.logger.Errorw("error in checking and re triggering ci", "err", err) //don't return as we have to update the workflow status diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index affa658f1d..a53e90dbd6 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -136,7 +136,7 @@ type WorkflowDagExecutorImpl struct { helmAppService client2.HelmAppService cdWorkflowCommonService cd.CdWorkflowCommonService - cdTriggerService devtronApps.TriggerService + cdHandlerService devtronApps.HandlerService userDeploymentRequestService service.UserDeploymentRequestService manifestCreationService manifest.ManifestCreationService @@ -150,7 +150,7 @@ type WorkflowDagExecutorImpl struct { envRepository repository5.EnvironmentRepository k8sCommonService k8sPkg.K8sCommonService workflowService executor.WorkflowService - ciTriggerService trigger.Service + ciHandlerService trigger.HandlerService } func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, @@ -171,7 +171,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi ciService pipeline.CiService, helmAppService client2.HelmAppService, cdWorkflowCommonService cd.CdWorkflowCommonService, - cdTriggerService devtronApps.TriggerService, + cdHandlerService devtronApps.HandlerService, userDeploymentRequestService service.UserDeploymentRequestService, manifestCreationService manifest.ManifestCreationService, commonArtifactService artifacts.CommonArtifactService, @@ -183,7 +183,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi envRepository repository5.EnvironmentRepository, k8sCommonService k8sPkg.K8sCommonService, workflowService executor.WorkflowService, - ciTriggerService trigger.Service, + ciHandlerService trigger.HandlerService, ) *WorkflowDagExecutorImpl { wde := &WorkflowDagExecutorImpl{logger: Logger, pipelineRepository: pipelineRepository, @@ -202,7 +202,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi pipelineStatusTimelineService: pipelineStatusTimelineService, helmAppService: helmAppService, cdWorkflowCommonService: cdWorkflowCommonService, - cdTriggerService: cdTriggerService, + cdHandlerService: cdHandlerService, userDeploymentRequestService: userDeploymentRequestService, manifestCreationService: manifestCreationService, commonArtifactService: commonArtifactService, @@ -216,7 +216,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi envRepository: envRepository, k8sCommonService: k8sCommonService, workflowService: workflowService, - ciTriggerService: ciTriggerService, + ciHandlerService: ciHandlerService, } config, err := types.GetCdConfig() if err != nil { @@ -324,7 +324,7 @@ func (impl *WorkflowDagExecutorImpl) UpdateCiWorkflowStatusFailure(timeoutForFai if isPodDeleted { ciWorkflow.Message = cdWorkflow2.POD_DELETED_MESSAGE // error logging handled inside handlePodDeleted - impl.ciTriggerService.HandlePodDeleted(ciWorkflow) + impl.ciHandlerService.HandlePodDeleted(ciWorkflow) } else { ciWorkflow.Message = "marked failed by job" } @@ -401,13 +401,13 @@ func (impl *WorkflowDagExecutorImpl) HandleCdStageReTrigger(runner *pipelineConf } if runner.WorkflowType == bean.CD_WORKFLOW_TYPE_PRE { - _, err = impl.cdTriggerService.TriggerPreStage(triggerRequest) + _, err = impl.cdHandlerService.TriggerPreStage(triggerRequest) if err != nil { impl.logger.Errorw("error in TriggerPreStage ", "err", err, "cdWorkflowRunnerId", runner.Id) return err } } else if runner.WorkflowType == bean.CD_WORKFLOW_TYPE_POST { - _, err = impl.cdTriggerService.TriggerPostStage(triggerRequest) + _, err = impl.cdHandlerService.TriggerPostStage(triggerRequest) if err != nil { impl.logger.Errorw("error in TriggerPostStage ", "err", err, "cdWorkflowRunnerId", runner.Id) return err @@ -560,7 +560,7 @@ func (impl *WorkflowDagExecutorImpl) ProcessDevtronAsyncInstallRequest(cdAsyncIn impl.logger.Errorw("error in getting deployment config by appId and envId", "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId, "err", err) return err } - releaseId, _, releaseErr := impl.cdTriggerService.TriggerRelease(newCtx, overrideRequest, envDeploymentConfig, cdAsyncInstallReq.TriggeredAt, cdAsyncInstallReq.TriggeredBy) + releaseId, _, releaseErr := impl.cdHandlerService.TriggerRelease(newCtx, overrideRequest, envDeploymentConfig, cdAsyncInstallReq.TriggeredAt, cdAsyncInstallReq.TriggeredBy) if releaseErr != nil { impl.logger.Errorw("error encountered in ProcessDevtronAsyncInstallRequest", "err", releaseErr, "cdWfrId", cdWfr.Id) impl.handleAsyncTriggerReleaseError(newCtx, releaseErr, cdWfr, overrideRequest) @@ -690,13 +690,13 @@ func (impl *WorkflowDagExecutorImpl) triggerIfAutoStageCdPipeline(request trigge // pre stage exists if request.Pipeline.PreTriggerType == pipelineConfig.TRIGGER_TYPE_AUTOMATIC { impl.logger.Debugw("trigger pre stage for pipeline", "artifactId", request.Artifact.Id, "pipelineId", request.Pipeline.Id) - _, err = impl.cdTriggerService.TriggerPreStage(request) // TODO handle error here + _, err = impl.cdHandlerService.TriggerPreStage(request) // TODO handle error here return err } } else if request.Pipeline.TriggerType == pipelineConfig.TRIGGER_TYPE_AUTOMATIC { // trigger deployment impl.logger.Debugw("trigger cd for pipeline", "artifactId", request.Artifact.Id, "pipelineId", request.Pipeline.Id) - err = impl.cdTriggerService.TriggerAutomaticDeployment(request) + err = impl.cdHandlerService.TriggerAutomaticDeployment(request) return err } return nil @@ -766,7 +766,7 @@ func (impl *WorkflowDagExecutorImpl) HandlePreStageSuccessEvent(triggerContext t } else { ciArtifactId = cdStageCompleteEvent.CiArtifactDTO.Id } - err = impl.cdTriggerService.TriggerAutoCDOnPreStageSuccess(triggerContext, cdStageCompleteEvent.CdPipelineId, ciArtifactId, cdStageCompleteEvent.WorkflowId) + err = impl.cdHandlerService.TriggerAutoCDOnPreStageSuccess(triggerContext, cdStageCompleteEvent.CdPipelineId, ciArtifactId, cdStageCompleteEvent.WorkflowId) if err != nil { impl.logger.Errorw("error in triggering cd on pre cd succcess", "err", err) return err @@ -810,7 +810,7 @@ func (impl *WorkflowDagExecutorImpl) HandleDeploymentSuccessEvent(triggerContext RefCdWorkflowRunnerId: 0, } triggerRequest.TriggerContext.Context = context.Background() - _, err = impl.cdTriggerService.TriggerPostStage(triggerRequest) + _, err = impl.cdHandlerService.TriggerPostStage(triggerRequest) if err != nil { impl.logger.Errorw("error in triggering post stage after successful deployment event", "err", err, "cdWorkflow", cdWorkflow) return err diff --git a/wire_gen.go b/wire_gen.go index e397a3a3a3..4d81fc08b8 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -130,7 +130,7 @@ import ( "github.com/devtron-labs/devtron/pkg/appStore/values/repository" service4 "github.com/devtron-labs/devtron/pkg/appStore/values/service" appWorkflow2 "github.com/devtron-labs/devtron/pkg/appWorkflow" - argoApplication3 "github.com/devtron-labs/devtron/pkg/argoApplication" + "github.com/devtron-labs/devtron/pkg/argoApplication" read22 "github.com/devtron-labs/devtron/pkg/argoApplication/read" config2 "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/asyncProvider" @@ -626,7 +626,7 @@ func InitializeApp() (*App, error) { } ciInfraGetter := ci.NewCiInfraGetter(sugaredLogger, infraConfigServiceImpl, infraConfigAuditServiceImpl) infraProviderImpl := infraProviders.NewInfraProviderImpl(sugaredLogger, infraGetter, ciInfraGetter) - workflowServiceImpl, err := executor.NewWorkflowServiceImpl(sugaredLogger, environmentRepositoryImpl, ciCdConfig, configReadServiceImpl, globalCMCSServiceImpl, argoWorkflowExecutorImpl, k8sServiceImpl, systemWorkflowExecutorImpl, k8sCommonServiceImpl, infraProviderImpl) + workflowServiceImpl, err := executor.NewWorkflowServiceImpl(sugaredLogger, environmentRepositoryImpl, ciCdConfig, configReadServiceImpl, globalCMCSServiceImpl, argoWorkflowExecutorImpl, systemWorkflowExecutorImpl, k8sCommonServiceImpl, infraProviderImpl, k8sServiceImpl) if err != nil { return nil, err } @@ -679,8 +679,8 @@ func InitializeApp() (*App, error) { return nil, err } blobStorageConfigServiceImpl := pipeline.NewBlobStorageConfigServiceImpl(sugaredLogger, k8sServiceImpl, ciCdConfig) - serviceImpl := trigger.NewServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciPipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateReadServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, ciCdPipelineOrchestratorImpl, attributesServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl, ciServiceImpl, ciWorkflowRepositoryImpl, clientImpl, ciLogServiceImpl, blobStorageConfigServiceImpl, clusterServiceImplExtended, environmentServiceImpl, k8sServiceImpl) - gitWebhookServiceImpl := gitWebhook.NewGitWebhookServiceImpl(sugaredLogger, gitWebhookRepositoryImpl, serviceImpl) + handlerServiceImpl := trigger.NewHandlerServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciPipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateReadServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, ciCdPipelineOrchestratorImpl, attributesServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl, ciServiceImpl, ciWorkflowRepositoryImpl, clientImpl, ciLogServiceImpl, blobStorageConfigServiceImpl, clusterServiceImplExtended, environmentServiceImpl, k8sServiceImpl) + gitWebhookServiceImpl := gitWebhook.NewGitWebhookServiceImpl(sugaredLogger, gitWebhookRepositoryImpl, handlerServiceImpl) gitWebhookRestHandlerImpl := restHandler.NewGitWebhookRestHandlerImpl(sugaredLogger, gitWebhookServiceImpl) ecrConfig, err := pipeline.GetEcrConfig() if err != nil { @@ -763,13 +763,13 @@ func InitializeApp() (*App, error) { scanToolExecutionHistoryMappingRepositoryImpl := repository24.NewScanToolExecutionHistoryMappingRepositoryImpl(db, sugaredLogger) cdWorkflowReadServiceImpl := read20.NewCdWorkflowReadServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) imageScanServiceImpl := imageScanning.NewImageScanServiceImpl(sugaredLogger, imageScanHistoryRepositoryImpl, imageScanResultRepositoryImpl, imageScanObjectMetaRepositoryImpl, cveStoreRepositoryImpl, imageScanDeployInfoRepositoryImpl, userServiceImpl, appRepositoryImpl, environmentServiceImpl, ciArtifactRepositoryImpl, policyServiceImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, scanToolMetadataRepositoryImpl, scanToolExecutionHistoryMappingRepositoryImpl, cvePolicyRepositoryImpl, cdWorkflowReadServiceImpl) - triggerServiceImpl, err := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, imageDigestPolicyServiceImpl, userServiceImpl, clientImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryReadServiceImpl, imageScanDeployInfoReadServiceImpl, imageScanDeployInfoServiceImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateReadServiceImpl, gitMaterialReadServiceImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, ciCdPipelineOrchestratorImpl, gitOperationServiceImpl, attributesServiceImpl, clusterRepositoryImpl, cdWorkflowRunnerServiceImpl, clusterServiceImplExtended, ciLogServiceImpl, workflowServiceImpl, blobStorageConfigServiceImpl) + devtronAppsHandlerServiceImpl, err := devtronApps.NewHandlerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, imageDigestPolicyServiceImpl, userServiceImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryReadServiceImpl, imageScanDeployInfoReadServiceImpl, imageScanDeployInfoServiceImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateReadServiceImpl, gitMaterialReadServiceImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, ciCdPipelineOrchestratorImpl, gitOperationServiceImpl, attributesServiceImpl, clusterRepositoryImpl, cdWorkflowRunnerServiceImpl, clusterServiceImplExtended, ciLogServiceImpl, workflowServiceImpl, blobStorageConfigServiceImpl) if err != nil { return nil, err } - pipelineConfigRestHandlerImpl := configure.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, deploymentTemplateValidationServiceImpl, chartServiceImpl, devtronAppGitOpConfigServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, generateManifestDeploymentTemplateServiceImpl, appWorkflowServiceImpl, gitMaterialReadServiceImpl, policyServiceImpl, imageScanResultReadServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingReadServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, ciCdPipelineOrchestratorImpl, gitProviderReadServiceImpl, teamReadServiceImpl, environmentRepositoryImpl, chartReadServiceImpl, serviceImpl, triggerServiceImpl) + pipelineConfigRestHandlerImpl := configure.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, deploymentTemplateValidationServiceImpl, chartServiceImpl, devtronAppGitOpConfigServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, generateManifestDeploymentTemplateServiceImpl, appWorkflowServiceImpl, gitMaterialReadServiceImpl, policyServiceImpl, imageScanResultReadServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingReadServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, ciCdPipelineOrchestratorImpl, gitProviderReadServiceImpl, teamReadServiceImpl, environmentRepositoryImpl, chartReadServiceImpl, handlerServiceImpl, devtronAppsHandlerServiceImpl) commonArtifactServiceImpl := artifacts.NewCommonArtifactServiceImpl(sugaredLogger, ciArtifactRepositoryImpl) - workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, cdWorkflowRunnerServiceImpl, ciServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl, runnable, imageScanHistoryRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, environmentRepositoryImpl, k8sCommonServiceImpl, workflowServiceImpl, serviceImpl) + workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, cdWorkflowRunnerServiceImpl, ciServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, devtronAppsHandlerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl, runnable, imageScanHistoryRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, environmentRepositoryImpl, k8sCommonServiceImpl, workflowServiceImpl, handlerServiceImpl) externalCiRestHandlerImpl := restHandler.NewExternalCiRestHandlerImpl(sugaredLogger, validate, userServiceImpl, enforcerImpl, workflowDagExecutorImpl) pubSubClientRestHandlerImpl := restHandler.NewPubSubClientRestHandlerImpl(pubSubClientServiceImpl, sugaredLogger, ciCdConfig) webhookRouterImpl := router.NewWebhookRouterImpl(gitWebhookRestHandlerImpl, pipelineConfigRestHandlerImpl, externalCiRestHandlerImpl, pubSubClientRestHandlerImpl) @@ -824,9 +824,9 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - argoApplicationServiceImpl := argoApplication3.NewArgoApplicationServiceImpl(sugaredLogger, clusterRepositoryImpl, k8sServiceImpl, helmAppClientImpl, helmAppServiceImpl, k8sApplicationServiceImpl, argoApplicationConfigServiceImpl, deploymentConfigServiceImpl) + argoApplicationServiceImpl := argoApplication.NewArgoApplicationServiceImpl(sugaredLogger, clusterRepositoryImpl, k8sServiceImpl, helmAppClientImpl, helmAppServiceImpl, k8sApplicationServiceImpl, argoApplicationConfigServiceImpl, deploymentConfigServiceImpl) argoApplicationReadServiceImpl := read22.NewArgoApplicationReadServiceImpl(sugaredLogger, clusterRepositoryImpl, k8sServiceImpl, helmAppClientImpl, helmAppServiceImpl) - argoApplicationServiceExtendedImpl := argoApplication3.NewArgoApplicationServiceExtendedServiceImpl(acdAuthConfig, argoApplicationServiceImpl, argoClientWrapperServiceImpl, argoApplicationReadServiceImpl, clusterServiceImplExtended) + argoApplicationServiceExtendedImpl := argoApplication.NewArgoApplicationServiceExtendedServiceImpl(acdAuthConfig, argoApplicationServiceImpl, argoClientWrapperServiceImpl, argoApplicationReadServiceImpl, clusterServiceImplExtended) installedAppResourceServiceImpl := resource.NewInstalledAppResourceServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, argoClientWrapperServiceImpl, acdAuthConfig, installedAppVersionHistoryRepositoryImpl, helmAppServiceImpl, helmAppReadServiceImpl, appStatusServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, k8sServiceImpl, deploymentConfigServiceImpl, ociRegistryConfigRepositoryImpl, argoApplicationServiceExtendedImpl) chartGroupEntriesRepositoryImpl := repository28.NewChartGroupEntriesRepositoryImpl(db, sugaredLogger) chartGroupReposotoryImpl := repository28.NewChartGroupReposotoryImpl(db, sugaredLogger) @@ -939,8 +939,8 @@ func InitializeApp() (*App, error) { telemetryRestHandlerImpl := restHandler.NewTelemetryRestHandlerImpl(sugaredLogger, telemetryEventClientImplExtended, enforcerImpl, userServiceImpl) telemetryRouterImpl := router.NewTelemetryRouterImpl(sugaredLogger, telemetryRestHandlerImpl) bulkUpdateRepositoryImpl := bulkUpdate.NewBulkUpdateRepository(db, sugaredLogger) - deployedAppServiceImpl := deployedApp.NewDeployedAppServiceImpl(sugaredLogger, k8sCommonServiceImpl, triggerServiceImpl, environmentRepositoryImpl, pipelineRepositoryImpl, cdWorkflowRepositoryImpl) - bulkUpdateServiceImpl := service7.NewBulkUpdateServiceImpl(bulkUpdateRepositoryImpl, sugaredLogger, environmentRepositoryImpl, pipelineRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineBuilderImpl, enforcerUtilImpl, ciHandlerImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, appWorkflowServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, deployedAppServiceImpl, cdPipelineEventPublishServiceImpl, serviceImpl) + deployedAppServiceImpl := deployedApp.NewDeployedAppServiceImpl(sugaredLogger, k8sCommonServiceImpl, devtronAppsHandlerServiceImpl, environmentRepositoryImpl, pipelineRepositoryImpl, cdWorkflowRepositoryImpl) + bulkUpdateServiceImpl := service7.NewBulkUpdateServiceImpl(bulkUpdateRepositoryImpl, sugaredLogger, environmentRepositoryImpl, pipelineRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineBuilderImpl, enforcerUtilImpl, ciHandlerImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, appWorkflowServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, deployedAppServiceImpl, cdPipelineEventPublishServiceImpl, handlerServiceImpl) bulkUpdateRestHandlerImpl := restHandler.NewBulkUpdateRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, bulkUpdateServiceImpl, chartServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, environmentServiceImpl, gitRegistryConfigImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl) bulkUpdateRouterImpl := router.NewBulkUpdateRouterImpl(bulkUpdateRestHandlerImpl) webhookSecretValidatorImpl := gitWebhook.NewWebhookSecretValidatorImpl(sugaredLogger) @@ -951,13 +951,13 @@ func InitializeApp() (*App, error) { webhookListenerRouterImpl := router.NewWebhookListenerRouterImpl(webhookEventHandlerImpl) appFilteringRestHandlerImpl := appList.NewAppFilteringRestHandlerImpl(sugaredLogger, teamServiceImpl, enforcerImpl, userServiceImpl, clusterServiceImplExtended, environmentServiceImpl, teamReadServiceImpl) appFilteringRouterImpl := appList2.NewAppFilteringRouterImpl(appFilteringRestHandlerImpl) - resourceTreeServiceImpl := resourceTree.NewServiceImpl(sugaredLogger, appListingServiceImpl, appStatusServiceImpl, argoApplicationServiceExtendedImpl, cdApplicationStatusUpdateHandlerImpl, helmAppReadServiceImpl, helmAppServiceImpl, k8sApplicationServiceImpl, k8sCommonServiceImpl, environmentReadServiceImpl) - appListingRestHandlerImpl := appList.NewAppListingRestHandlerImpl(appListingServiceImpl, enforcerImpl, pipelineBuilderImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, userServiceImpl, k8sCommonServiceImpl, installedAppDBExtendedServiceImpl, installedAppResourceServiceImpl, pipelineRepositoryImpl, k8sApplicationServiceImpl, deploymentConfigServiceImpl, resourceTreeServiceImpl) + serviceImpl := resourceTree.NewServiceImpl(sugaredLogger, appListingServiceImpl, appStatusServiceImpl, argoApplicationServiceExtendedImpl, cdApplicationStatusUpdateHandlerImpl, helmAppReadServiceImpl, helmAppServiceImpl, k8sApplicationServiceImpl, k8sCommonServiceImpl, environmentReadServiceImpl) + appListingRestHandlerImpl := appList.NewAppListingRestHandlerImpl(appListingServiceImpl, enforcerImpl, pipelineBuilderImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, userServiceImpl, k8sCommonServiceImpl, installedAppDBExtendedServiceImpl, installedAppResourceServiceImpl, pipelineRepositoryImpl, k8sApplicationServiceImpl, deploymentConfigServiceImpl, serviceImpl) appListingRouterImpl := appList2.NewAppListingRouterImpl(appListingRestHandlerImpl) appInfoRestHandlerImpl := appInfo.NewAppInfoRestHandlerImpl(sugaredLogger, appCrudOperationServiceImpl, userServiceImpl, validate, enforcerUtilImpl, enforcerImpl, helmAppServiceImpl, enforcerUtilHelmImpl, genericNoteServiceImpl, commonEnforcementUtilImpl) appInfoRouterImpl := appInfo2.NewAppInfoRouterImpl(sugaredLogger, appInfoRestHandlerImpl) pipelineDeploymentConfigServiceImpl := pipeline.NewPipelineDeploymentConfigServiceImpl(sugaredLogger, chartRepositoryImpl, pipelineRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, scopedVariableCMCSManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, configMapHistoryReadServiceImpl, envConfigOverrideReadServiceImpl) - pipelineTriggerRestHandlerImpl := trigger2.NewPipelineRestHandler(appServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, pipelineDeploymentConfigServiceImpl, deployedAppServiceImpl, triggerServiceImpl, workflowEventPublishServiceImpl) + pipelineTriggerRestHandlerImpl := trigger2.NewPipelineRestHandler(appServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, pipelineDeploymentConfigServiceImpl, deployedAppServiceImpl, devtronAppsHandlerServiceImpl, workflowEventPublishServiceImpl) sseSSE := sse.NewSSE() pipelineTriggerRouterImpl := trigger3.NewPipelineTriggerRouter(pipelineTriggerRestHandlerImpl, sseSSE) webhookDataRestHandlerImpl := webhook.NewWebhookDataRestHandlerImpl(sugaredLogger, userServiceImpl, ciPipelineMaterialRepositoryImpl, enforcerUtilImpl, enforcerImpl, clientImpl, webhookEventDataConfigImpl) @@ -1045,7 +1045,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - ciTriggerCronImpl := cron2.NewCiTriggerCronImpl(sugaredLogger, ciTriggerCronConfig, pipelineStageRepositoryImpl, ciArtifactRepositoryImpl, globalPluginRepositoryImpl, cronLoggerImpl, serviceImpl) + ciTriggerCronImpl := cron2.NewCiTriggerCronImpl(sugaredLogger, ciTriggerCronConfig, pipelineStageRepositoryImpl, ciArtifactRepositoryImpl, globalPluginRepositoryImpl, cronLoggerImpl, handlerServiceImpl) proxyConfig, err := proxy.GetProxyConfig() if err != nil { return nil, err @@ -1081,12 +1081,12 @@ func InitializeApp() (*App, error) { cdWorkflowServiceImpl := cd.NewCdWorkflowServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) cdWorkflowRunnerReadServiceImpl := read20.NewCdWorkflowRunnerReadServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) webhookServiceImpl := pipeline.NewWebhookServiceImpl(ciArtifactRepositoryImpl, sugaredLogger, ciPipelineRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowCommonServiceImpl, workFlowStageStatusServiceImpl, ciServiceImpl) - workflowEventProcessorImpl, err := in.NewWorkflowEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowServiceImpl, cdWorkflowReadServiceImpl, cdWorkflowRunnerServiceImpl, cdWorkflowRunnerReadServiceImpl, workflowDagExecutorImpl, ciHandlerImpl, cdHandlerImpl, eventSimpleFactoryImpl, eventRESTClientImpl, triggerServiceImpl, deployedAppServiceImpl, webhookServiceImpl, validate, environmentVariables, cdWorkflowCommonServiceImpl, cdPipelineConfigServiceImpl, userDeploymentRequestServiceImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, cdWorkflowRepositoryImpl, deploymentConfigServiceImpl, serviceImpl) + workflowEventProcessorImpl, err := in.NewWorkflowEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowServiceImpl, cdWorkflowReadServiceImpl, cdWorkflowRunnerServiceImpl, cdWorkflowRunnerReadServiceImpl, workflowDagExecutorImpl, ciHandlerImpl, cdHandlerImpl, eventSimpleFactoryImpl, eventRESTClientImpl, devtronAppsHandlerServiceImpl, deployedAppServiceImpl, webhookServiceImpl, validate, environmentVariables, cdWorkflowCommonServiceImpl, cdPipelineConfigServiceImpl, userDeploymentRequestServiceImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, cdWorkflowRepositoryImpl, deploymentConfigServiceImpl, handlerServiceImpl) if err != nil { return nil, err } ciPipelineEventProcessorImpl := in.NewCIPipelineEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, gitWebhookServiceImpl) - cdPipelineEventProcessorImpl := in.NewCDPipelineEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowCommonServiceImpl, workflowStatusServiceImpl, triggerServiceImpl, pipelineRepositoryImpl, installedAppReadServiceImpl) + cdPipelineEventProcessorImpl := in.NewCDPipelineEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowCommonServiceImpl, workflowStatusServiceImpl, devtronAppsHandlerServiceImpl, pipelineRepositoryImpl, installedAppReadServiceImpl) deployedApplicationEventProcessorImpl := in.NewDeployedApplicationEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, appServiceImpl, gitOpsConfigReadServiceImpl, installedAppDBExtendedServiceImpl, workflowDagExecutorImpl, cdWorkflowCommonServiceImpl, pipelineBuilderImpl, appStoreDeploymentServiceImpl, pipelineRepositoryImpl, installedAppReadServiceImpl, deploymentConfigServiceImpl) appStoreAppsEventProcessorImpl := in.NewAppStoreAppsEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, chartGroupServiceImpl, installedAppVersionHistoryRepositoryImpl) centralEventProcessor, err := eventProcessor.NewCentralEventProcessor(sugaredLogger, workflowEventProcessorImpl, ciPipelineEventProcessorImpl, cdPipelineEventProcessorImpl, deployedApplicationEventProcessorImpl, appStoreAppsEventProcessorImpl) From b21694cffac3226bdaada1beff32a1b46868f113 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 8 Apr 2025 12:29:47 +0530 Subject: [PATCH 34/37] file renaming --- .../trigger/devtronApps/HandlerService.go | 1330 +---------------- ..._ent1.go => deployStageHandlerCode_ent.go} | 16 + ...ggerService.go => postStageHandlerCode.go} | 0 ...ice_ent.go => postStageHandlerCode_ent.go} | 0 ...mmonService.go => prePostWfAndLogsCode.go} | 0 ...vice_ent.go => preStageHandlerCode_ent.go} | 0 ...erService.go => preStageHandlerService.go} | 0 7 files changed, 29 insertions(+), 1317 deletions(-) rename pkg/deployment/trigger/devtronApps/{HandlerService_ent1.go => deployStageHandlerCode_ent.go} (84%) rename pkg/deployment/trigger/devtronApps/{PostStageTriggerService.go => postStageHandlerCode.go} (100%) rename pkg/deployment/trigger/devtronApps/{PostStageTriggerService_ent.go => postStageHandlerCode_ent.go} (100%) rename pkg/deployment/trigger/devtronApps/{PrePostCommonService.go => prePostWfAndLogsCode.go} (100%) rename pkg/deployment/trigger/devtronApps/{PreStageTriggerService_ent.go => preStageHandlerCode_ent.go} (100%) rename pkg/deployment/trigger/devtronApps/{PreStageTriggerService.go => preStageHandlerService.go} (100%) diff --git a/pkg/deployment/trigger/devtronApps/HandlerService.go b/pkg/deployment/trigger/devtronApps/HandlerService.go index 7c915a3195..c92a009134 100644 --- a/pkg/deployment/trigger/devtronApps/HandlerService.go +++ b/pkg/deployment/trigger/devtronApps/HandlerService.go @@ -19,36 +19,25 @@ package devtronApps import ( "bufio" "context" - "errors" - "fmt" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" util5 "github.com/devtron-labs/common-lib/utils/k8s" bean3 "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/api/bean/gitOps" - bean6 "github.com/devtron-labs/devtron/api/helm-app/bean" "github.com/devtron-labs/devtron/api/helm-app/gRPC" client2 "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/client/argocdServer" - bean7 "github.com/devtron-labs/devtron/client/argocdServer/bean" client "github.com/devtron-labs/devtron/client/events" - "github.com/devtron-labs/devtron/internal/middleware" - "github.com/devtron-labs/devtron/internal/sql/models" repository3 "github.com/devtron-labs/devtron/internal/sql/repository" appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" repository4 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" bean4 "github.com/devtron-labs/devtron/pkg/app/bean" "github.com/devtron-labs/devtron/pkg/app/status" - statusBean "github.com/devtron-labs/devtron/pkg/app/status/bean" "github.com/devtron-labs/devtron/pkg/attributes" "github.com/devtron-labs/devtron/pkg/auth/user" - bean2 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/read" pipeline2 "github.com/devtron-labs/devtron/pkg/build/pipeline" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" @@ -60,49 +49,43 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "github.com/devtron-labs/devtron/pkg/deployment/manifest" - bean10 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/bean" - bean5 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" "github.com/devtron-labs/devtron/pkg/deployment/manifest/publish" - "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/adapter" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" - "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/helper" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/userDeploymentRequest/service" - clientErrors "github.com/devtron-labs/devtron/pkg/errors" "github.com/devtron-labs/devtron/pkg/eventProcessor/out" "github.com/devtron-labs/devtron/pkg/executor" "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" - k8s2 "github.com/devtron-labs/devtron/pkg/k8s" "github.com/devtron-labs/devtron/pkg/pipeline" - bean8 "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/history" "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/plugin" security2 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning" read2 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/read" - repository6 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/repository" "github.com/devtron-labs/devtron/pkg/sql" "github.com/devtron-labs/devtron/pkg/variables" "github.com/devtron-labs/devtron/pkg/workflow/cd" globalUtil "github.com/devtron-labs/devtron/util" util2 "github.com/devtron-labs/devtron/util/event" "github.com/devtron-labs/devtron/util/rbac" - "github.com/go-pg/pg" - "go.opentelemetry.io/otel" "go.uber.org/zap" - "golang.org/x/exp/slices" - "google.golang.org/grpc/codes" - status2 "google.golang.org/grpc/status" - "helm.sh/helm/v3/pkg/chart" - "net/http" "os" - "path" - "regexp" - "strconv" - "strings" "time" ) +/* +files in this package are - +HandlerService.go - containing If and impl with common used code +HandlerService_ent.go - containing ent If and impl with common used code +deployStageHandlerCode.go - code related to deploy stage trigger +deployStageHandlerCode_ent.go - ent code related to deploy stage trigger +preStageHandlerCode.go - code related to pre stage trigger +preStageHandlerCode_ent.go - ent code related to pre stage trigger +postStageHandlerCode.go - code related to post stage trigger +postStageHandlerCode_ent.go - ent code related to post stage trigger +prePostWfAndLogsCode.go - code containing pre/post wf handling(abort) and logs related code +*/ + type HandlerService interface { TriggerPostStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) TriggerPreStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) @@ -318,1112 +301,6 @@ func NewHandlerServiceImpl(logger *zap.SugaredLogger, return impl, nil } -func (impl *HandlerServiceImpl) TriggerStageForBulk(triggerRequest bean.TriggerRequest) error { - - preStage, err := impl.pipelineStageService.GetCdStageByCdPipelineIdAndStageType(triggerRequest.Pipeline.Id, repository.PIPELINE_STAGE_TYPE_PRE_CD, false) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in fetching CD pipeline stage", "cdPipelineId", triggerRequest.Pipeline.Id, "stage ", repository.PIPELINE_STAGE_TYPE_PRE_CD, "err", err) - return err - } - - // handle corrupt data (https://github.com/devtron-labs/devtron/issues/3826) - err, deleted := impl.deleteCorruptedPipelineStage(preStage, triggerRequest.TriggeredBy) - if err != nil { - impl.logger.Errorw("error in deleteCorruptedPipelineStage ", "cdPipelineId", triggerRequest.Pipeline.Id, "err", err, "preStage", preStage, "triggeredBy", triggerRequest.TriggeredBy) - return err - } - - triggerRequest.TriggerContext.Context = context.Background() - if len(triggerRequest.Pipeline.PreStageConfig) > 0 || (preStage != nil && !deleted) { - // pre stage exists - impl.logger.Debugw("trigger pre stage for pipeline", "artifactId", triggerRequest.Artifact.Id, "pipelineId", triggerRequest.Pipeline.Id) - triggerRequest.RefCdWorkflowRunnerId = 0 - err = impl.preStageHandlingForTriggerStageInBulk(&triggerRequest) - if err != nil { - return err - } - _, err = impl.TriggerPreStage(triggerRequest) // TODO handle error here - return err - } else { - // trigger deployment - impl.logger.Debugw("trigger cd for pipeline", "artifactId", triggerRequest.Artifact.Id, "pipelineId", triggerRequest.Pipeline.Id) - err = impl.TriggerAutomaticDeployment(triggerRequest) - return err - } -} - -func (impl *HandlerServiceImpl) getCdPipelineForManualCdTrigger(ctx context.Context, pipelineId int) (*pipelineConfig.Pipeline, error) { - _, span := otel.Tracer("HandlerService").Start(ctx, "getCdPipelineForManualCdTrigger") - defer span.End() - cdPipeline, err := impl.pipelineRepository.FindById(pipelineId) - if err != nil { - impl.logger.Errorw("manual trigger request with invalid pipelineId, ManualCdTrigger", "pipelineId", pipelineId, "err", err) - return nil, err - } - //checking if namespace exist or not - clusterIdToNsMap := map[int]string{ - cdPipeline.Environment.ClusterId: cdPipeline.Environment.Namespace, - } - - err = impl.helmAppService.CheckIfNsExistsForClusterIds(clusterIdToNsMap) - if err != nil { - impl.logger.Errorw("manual trigger request with invalid namespace, ManualCdTrigger", "pipelineId", pipelineId, "err", err) - return nil, err - } - return cdPipeline, nil -} - -func (impl *HandlerServiceImpl) validateDeploymentTriggerRequest(ctx context.Context, validateDeploymentTriggerObj *bean.ValidateDeploymentTriggerObj) error { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.validateDeploymentTriggerRequest") - defer span.End() - // custom GitOps repo url validation --> Start - err := impl.handleCustomGitOpsRepoValidation(validateDeploymentTriggerObj.Runner, validateDeploymentTriggerObj.CdPipeline, validateDeploymentTriggerObj.DeploymentConfig, validateDeploymentTriggerObj.TriggeredBy) - if err != nil { - impl.logger.Errorw("custom GitOps repository validation error, TriggerStage", "err", err) - return err - } - // custom GitOps repo url validation --> Ends - var isVulnerable bool - // if request is for rollback then bypass vulnerability validation - if !validateDeploymentTriggerObj.IsDeploymentTypeRollback() { - // checking vulnerability for deploying image - vulnerabilityCheckRequest := adapter.GetVulnerabilityCheckRequest(validateDeploymentTriggerObj.CdPipeline, validateDeploymentTriggerObj.ImageDigest) - isVulnerable, err = impl.imageScanService.GetArtifactVulnerabilityStatus(newCtx, vulnerabilityCheckRequest) - if err != nil { - impl.logger.Errorw("error in getting Artifact vulnerability status, ManualCdTrigger", "err", err) - return err - } - } - - if isVulnerable == true { - // if image vulnerable, update timeline status and return - if err = impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(validateDeploymentTriggerObj.Runner, errors.New(cdWorkflow.FOUND_VULNERABILITY), validateDeploymentTriggerObj.TriggeredBy); err != nil { - impl.logger.Errorw("error while updating current runner status to failed, TriggerDeployment", "wfrId", validateDeploymentTriggerObj.Runner.Id, "err", err) - } - return fmt.Errorf("found vulnerability for image digest %s", validateDeploymentTriggerObj.ImageDigest) - } - return nil -} - -// TODO: write a wrapper to handle auto and manual trigger -func (impl *HandlerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerContext, overrideRequest *bean3.ValuesOverrideRequest) (int, string, *bean4.ManifestPushTemplate, error) { - - triggerContext.TriggerType = bean.Manual - // setting triggeredAt variable to have consistent data for various audit log places in db for deployment time - triggeredAt := time.Now() - - releaseId := 0 - ctx := triggerContext.Context - cdPipeline, err := impl.getCdPipelineForManualCdTrigger(ctx, overrideRequest.PipelineId) - if err != nil { - if overrideRequest.WfrId != 0 { - err2 := impl.cdWorkflowCommonService.MarkDeploymentFailedForRunnerId(overrideRequest.WfrId, err, overrideRequest.UserId) - if err2 != nil { - impl.logger.Errorw("error while updating current runner status to failed, ManualCdTrigger", "cdWfr", overrideRequest.WfrId, "err2", err2) - } - } - return 0, "", nil, err - } - envDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(cdPipeline.AppId, cdPipeline.EnvironmentId) - if err != nil { - impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", cdPipeline.AppId, "envId", cdPipeline.EnvironmentId, "err", err) - return 0, "", nil, err - } - - adapter.SetPipelineFieldsInOverrideRequest(overrideRequest, cdPipeline, envDeploymentConfig) - ciArtifactId := overrideRequest.CiArtifactId - - _, span := otel.Tracer("orchestrator").Start(ctx, "ciArtifactRepository.Get") - artifact, err := impl.ciArtifactRepository.Get(ciArtifactId) - span.End() - if err != nil { - impl.logger.Errorw("error in getting CiArtifact", "CiArtifactId", overrideRequest.CiArtifactId, "err", err) - return 0, "", nil, err - } - - if artifact.IsMigrationRequired() { - // Migration of deprecated DataSource Type - migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(artifact.Id) - if migrationErr != nil { - impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id) - } - } - - _, imageTag, err := artifact.ExtractImageRepoAndTag() - if err != nil { - impl.logger.Errorw("error in getting image tag and repo", "err", err) - } - helmPackageName := fmt.Sprintf("%s-%s-%s", cdPipeline.App.AppName, cdPipeline.Environment.Name, imageTag) - var manifestPushTemplate *bean4.ManifestPushTemplate - - switch overrideRequest.CdWorkflowType { - case bean3.CD_WORKFLOW_TYPE_PRE: - var cdWf *pipelineConfig.CdWorkflow - if overrideRequest.CdWorkflowId == 0 { - cdWf = &pipelineConfig.CdWorkflow{ - CiArtifactId: artifact.Id, - PipelineId: cdPipeline.Id, - AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: 1, UpdatedOn: triggeredAt, UpdatedBy: 1}, - } - err := impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) - if err != nil { - return 0, "", nil, err - } - } else { - cdWf, err = impl.cdWorkflowRepository.FindById(overrideRequest.CdWorkflowId) - if err != nil { - impl.logger.Errorw("error in TriggerPreStage, ManualCdTrigger", "err", err) - return 0, "", nil, err - } - } - overrideRequest.CdWorkflowId = cdWf.Id - - _, span = otel.Tracer("orchestrator").Start(ctx, "TriggerPreStage") - triggerRequest := bean.TriggerRequest{ - CdWf: cdWf, - Artifact: artifact, - Pipeline: cdPipeline, - TriggeredBy: overrideRequest.UserId, - ApplyAuth: false, - TriggerContext: triggerContext, - RefCdWorkflowRunnerId: 0, - CdWorkflowRunnerId: overrideRequest.WfrId, - } - manifestPushTemplate, err = impl.TriggerPreStage(triggerRequest) - span.End() - if err != nil { - impl.logger.Errorw("error in TriggerPreStage, ManualCdTrigger", "err", err) - return 0, "", nil, err - } - case bean3.CD_WORKFLOW_TYPE_DEPLOY: - if overrideRequest.DeploymentType == models.DEPLOYMENTTYPE_UNKNOWN { - overrideRequest.DeploymentType = models.DEPLOYMENTTYPE_DEPLOY - } - - cdWf, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, overrideRequest.CdWorkflowId, bean3.CD_WORKFLOW_TYPE_PRE) - if err != nil && !util.IsErrNoRows(err) { - impl.logger.Errorw("error in getting cdWorkflow, ManualCdTrigger", "CdWorkflowId", overrideRequest.CdWorkflowId, "err", err) - return 0, "", nil, err - } - - cdWorkflowId := cdWf.CdWorkflowId - if cdWf.CdWorkflowId == 0 { - cdWf := &pipelineConfig.CdWorkflow{ - CiArtifactId: overrideRequest.CiArtifactId, - PipelineId: overrideRequest.PipelineId, - AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, - } - err := impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) - if err != nil { - impl.logger.Errorw("error in creating cdWorkflow, ManualCdTrigger", "PipelineId", overrideRequest.PipelineId, "err", err) - return 0, "", nil, err - } - cdWorkflowId = cdWf.Id - } - - runner := &pipelineConfig.CdWorkflowRunner{ - Name: cdPipeline.Name, - WorkflowType: bean3.CD_WORKFLOW_TYPE_DEPLOY, - ExecutorType: cdWorkflow.WORKFLOW_EXECUTOR_TYPE_AWF, - Status: cdWorkflow.WorkflowInitiated, //deployment Initiated for manual trigger - TriggeredBy: overrideRequest.UserId, - StartedOn: triggeredAt, - Namespace: impl.config.GetDefaultNamespace(), - CdWorkflowId: cdWorkflowId, - AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, - ReferenceId: triggerContext.ReferenceId, - } - savedWfr, err := impl.cdWorkflowRunnerService.SaveCDWorkflowRunnerWithStage(runner) - if err != nil { - impl.logger.Errorw("err in creating cdWorkflowRunner, ManualCdTrigger", "cdWorkflowId", cdWorkflowId, "err", err) - return 0, "", nil, err - } - runner.CdWorkflow = &pipelineConfig.CdWorkflow{ - Pipeline: cdPipeline, - } - overrideRequest.WfrId = savedWfr.Id - overrideRequest.CdWorkflowId = cdWorkflowId - // creating cd pipeline status timeline for deployment initialisation - timeline := impl.pipelineStatusTimelineService.NewDevtronAppPipelineStatusTimelineDbObject(runner.Id, timelineStatus.TIMELINE_STATUS_DEPLOYMENT_INITIATED, timelineStatus.TIMELINE_DESCRIPTION_DEPLOYMENT_INITIATED, overrideRequest.UserId) - _, span = otel.Tracer("orchestrator").Start(ctx, "cdPipelineStatusTimelineRepo.SaveTimelineForACDHelmApps") - _, err = impl.pipelineStatusTimelineService.SaveTimelineIfNotAlreadyPresent(timeline, nil) - - span.End() - if err != nil { - impl.logger.Errorw("error in creating timeline status for deployment initiation, ManualCdTrigger", "err", err, "timeline", timeline) - } - if isNotHibernateRequest(overrideRequest.DeploymentType) { - validateReqObj := adapter.NewValidateDeploymentTriggerObj(runner, cdPipeline, artifact.ImageDigest, envDeploymentConfig, overrideRequest.UserId, overrideRequest.IsRollbackDeployment) - validationErr := impl.validateDeploymentTriggerRequest(ctx, validateReqObj) - if validationErr != nil { - impl.logger.Errorw("validation error deployment request", "cdWfr", runner.Id, "err", validationErr) - return 0, "", nil, validationErr - } - } - // Deploy the release - var releaseErr error - releaseId, manifestPushTemplate, releaseErr = impl.handleCDTriggerRelease(ctx, overrideRequest, envDeploymentConfig, triggeredAt, overrideRequest.UserId) - // if releaseErr found, then the mark current deployment Failed and return - if releaseErr != nil { - err := impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, releaseErr, overrideRequest.UserId) - if err != nil { - impl.logger.Errorw("error while updating current runner status to failed", "cdWfr", runner.Id, "err", err) - } - return 0, "", nil, releaseErr - } - - case bean3.CD_WORKFLOW_TYPE_POST: - cdWfRunner, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, overrideRequest.CdWorkflowId, bean3.CD_WORKFLOW_TYPE_DEPLOY) - if err != nil && !util.IsErrNoRows(err) { - impl.logger.Errorw("err in getting cdWorkflowRunner, ManualCdTrigger", "cdWorkflowId", overrideRequest.CdWorkflowId, "err", err) - return 0, "", nil, err - } - - var cdWf *pipelineConfig.CdWorkflow - if cdWfRunner.CdWorkflowId == 0 { - cdWf = &pipelineConfig.CdWorkflow{ - CiArtifactId: ciArtifactId, - PipelineId: overrideRequest.PipelineId, - AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, - } - err := impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) - if err != nil { - impl.logger.Errorw("error in creating cdWorkflow, ManualCdTrigger", "CdWorkflowId", overrideRequest.CdWorkflowId, "err", err) - return 0, "", nil, err - } - overrideRequest.CdWorkflowId = cdWf.Id - } else { - _, span = otel.Tracer("orchestrator").Start(ctx, "cdWorkflowRepository.FindById") - cdWf, err = impl.cdWorkflowRepository.FindById(overrideRequest.CdWorkflowId) - span.End() - if err != nil && !util.IsErrNoRows(err) { - impl.logger.Errorw("error in getting cdWorkflow, ManualCdTrigger", "CdWorkflowId", overrideRequest.CdWorkflowId, "err", err) - return 0, "", nil, err - } - } - _, span = otel.Tracer("orchestrator").Start(ctx, "TriggerPostStage") - triggerRequest := bean.TriggerRequest{ - CdWf: cdWf, - Pipeline: cdPipeline, - TriggeredBy: overrideRequest.UserId, - RefCdWorkflowRunnerId: 0, - TriggerContext: triggerContext, - CdWorkflowRunnerId: overrideRequest.WfrId, - } - manifestPushTemplate, err = impl.TriggerPostStage(triggerRequest) - span.End() - if err != nil { - impl.logger.Errorw("error in TriggerPostStage, ManualCdTrigger", "CdWorkflowId", cdWf.Id, "err", err) - return 0, "", nil, err - } - default: - impl.logger.Errorw("invalid CdWorkflowType, ManualCdTrigger", "CdWorkflowType", overrideRequest.CdWorkflowType, "err", err) - return 0, "", nil, fmt.Errorf("invalid CdWorkflowType %s for the trigger request", string(overrideRequest.CdWorkflowType)) - } - return releaseId, helmPackageName, manifestPushTemplate, err -} - -func isNotHibernateRequest(deploymentType models.DeploymentType) bool { - return deploymentType != models.DEPLOYMENTTYPE_STOP && deploymentType != models.DEPLOYMENTTYPE_START -} - -// TODO: write a wrapper to handle auto and manual trigger -func (impl *HandlerServiceImpl) TriggerAutomaticDeployment(request bean.TriggerRequest) error { - // in case of manual trigger auth is already applied and for auto triggers there is no need for auth check here - triggeredBy := request.TriggeredBy - pipeline := request.Pipeline - artifact := request.Artifact - - //setting triggeredAt variable to have consistent data for various audit log places in db for deployment time - triggeredAt := time.Now() - cdWf := request.CdWf - ctx := context.Background() - - if cdWf == nil || (cdWf != nil && cdWf.CiArtifactId != artifact.Id) { - // cdWf != nil && cdWf.CiArtifactId != artifact.Id for auto trigger case when deployment is triggered with image generated by plugin - cdWf = &pipelineConfig.CdWorkflow{ - CiArtifactId: artifact.Id, - PipelineId: pipeline.Id, - AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: 1, UpdatedOn: triggeredAt, UpdatedBy: 1}, - } - err := impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) - if err != nil { - return err - } - } - - runner := &pipelineConfig.CdWorkflowRunner{ - Name: pipeline.Name, - WorkflowType: bean3.CD_WORKFLOW_TYPE_DEPLOY, - ExecutorType: cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM, - Status: cdWorkflow.WorkflowInitiated, // deployment Initiated for auto trigger - TriggeredBy: triggeredBy, - StartedOn: triggeredAt, - Namespace: impl.config.GetDefaultNamespace(), - CdWorkflowId: cdWf.Id, - AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: triggeredBy, UpdatedOn: triggeredAt, UpdatedBy: triggeredBy}, - ReferenceId: request.TriggerContext.ReferenceId, - } - savedWfr, err := impl.cdWorkflowRunnerService.SaveCDWorkflowRunnerWithStage(runner) - if err != nil { - return err - } - runner.CdWorkflow = &pipelineConfig.CdWorkflow{ - Pipeline: pipeline, - } - // creating cd pipeline status timeline for deployment initialisation - timeline := &pipelineConfig.PipelineStatusTimeline{ - CdWorkflowRunnerId: runner.Id, - Status: timelineStatus.TIMELINE_STATUS_DEPLOYMENT_INITIATED, - StatusDetail: "Deployment initiated successfully.", - StatusTime: time.Now(), - } - timeline.CreateAuditLog(1) - err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil) - if err != nil { - impl.logger.Errorw("error in creating timeline status for deployment initiation", "err", err, "timeline", timeline) - } - envDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(pipeline.AppId, pipeline.EnvironmentId) - if err != nil { - impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) - return err - } - validationErr := impl.validateDeploymentTriggerRequest(ctx, adapter.NewValidateDeploymentTriggerObj(runner, pipeline, artifact.ImageDigest, envDeploymentConfig, triggeredBy, false)) - if validationErr != nil { - impl.logger.Errorw("validation error deployment request", "cdWfr", runner.Id, "err", validationErr) - return validationErr - } - releaseErr := impl.TriggerCD(ctx, artifact, cdWf.Id, savedWfr.Id, pipeline, envDeploymentConfig, triggeredAt, triggeredBy) - // if releaseErr found, then the mark current deployment Failed and return - if releaseErr != nil { - err := impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, releaseErr, triggeredBy) - if err != nil { - impl.logger.Errorw("error while updating current runner status to failed, updatePreviousDeploymentStatus", "cdWfr", runner.Id, "err", err) - } - return releaseErr - } - return nil -} - -func (impl *HandlerServiceImpl) TriggerCD(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) error { - impl.logger.Debugw("automatic pipeline trigger attempt async", "artifactId", artifact.Id) - err := impl.triggerReleaseAsync(ctx, artifact, cdWorkflowId, wfrId, pipeline, envDeploymentConfig, triggeredAt, triggeredBy) - if err != nil { - impl.logger.Errorw("error in cd trigger", "err", err) - return err - } - return err -} - -func (impl *HandlerServiceImpl) triggerReleaseAsync(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) error { - err := impl.validateAndTrigger(ctx, pipeline, envDeploymentConfig, artifact, cdWorkflowId, wfrId, triggeredAt, triggeredBy) - if err != nil { - impl.logger.Errorw("error in trigger for pipeline", "pipelineId", strconv.Itoa(pipeline.Id)) - } - impl.logger.Debugw("trigger attempted for all pipeline ", "artifactId", artifact.Id) - return err -} - -func (impl *HandlerServiceImpl) validateAndTrigger(ctx context.Context, p *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time, triggeredBy int32) error { - //TODO: verify this logic - object := impl.enforcerUtil.GetAppRBACNameByAppId(p.AppId) - envApp := strings.Split(object, "/") - if len(envApp) != 2 { - impl.logger.Error("invalid req, app and env not found from rbac") - return errors.New("invalid req, app and env not found from rbac") - } - err := impl.releasePipeline(ctx, p, envDeploymentConfig, artifact, cdWorkflowId, wfrId, triggeredAt, triggeredBy) - return err -} - -func (impl *HandlerServiceImpl) releasePipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time, triggeredBy int32) error { - startTime := time.Now() - defer func() { - impl.logger.Debugw("auto trigger release process completed", "timeTaken", time.Since(startTime), "cdPipelineId", pipeline.Id, "artifactId", artifact.Id, "wfrId", wfrId) - }() - impl.logger.Debugw("auto triggering release for", "cdPipelineId", pipeline.Id, "artifactId", artifact.Id, "wfrId", wfrId) - pipeline, err := impl.pipelineRepository.FindById(pipeline.Id) - if err != nil { - impl.logger.Errorw("error in fetching pipeline by pipelineId", "err", err) - return err - } - - request := &bean3.ValuesOverrideRequest{ - PipelineId: pipeline.Id, - UserId: artifact.CreatedBy, - CiArtifactId: artifact.Id, - AppId: pipeline.AppId, - CdWorkflowId: cdWorkflowId, - ForceTrigger: true, - DeploymentWithConfig: bean3.DEPLOYMENT_CONFIG_TYPE_LAST_SAVED, - WfrId: wfrId, - } - - adapter.SetPipelineFieldsInOverrideRequest(request, pipeline, envDeploymentConfig) - - id, _, err := impl.handleCDTriggerRelease(ctx, request, envDeploymentConfig, triggeredAt, triggeredBy) - if err != nil { - impl.logger.Errorw("error in auto cd pipeline trigger", "pipelineId", pipeline.Id, "artifactId", artifact.Id, "err", err) - } else { - impl.logger.Infow("pipeline successfully triggered", "cdPipelineId", pipeline.Id, "artifactId", artifact.Id, "releaseId", id) - } - return err -} - -func (impl *HandlerServiceImpl) triggerAsyncRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, - envDeploymentConfig *bean9.DeploymentConfig, userDeploymentRequestId int, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.triggerAsyncRelease") - defer span.End() - // build merged values and save PCO history for the release - valuesOverrideResponse, err := impl.manifestCreationService.GetValuesOverrideForTrigger(newCtx, overrideRequest, envDeploymentConfig, triggeredAt) - // auditDeploymentTriggerHistory is performed irrespective of GetValuesOverrideForTrigger error - for auditing purposes - historyErr := impl.auditDeploymentTriggerHistory(overrideRequest.WfrId, valuesOverrideResponse, newCtx, triggeredAt, deployedBy) - if historyErr != nil { - impl.logger.Errorw("error in auditing deployment trigger history", "cdWfrId", overrideRequest.WfrId, "err", err) - return releaseNo, manifestPushTemplate, err - } - // handling GetValuesOverrideForTrigger error - if err != nil { - impl.logger.Errorw("error in fetching values for trigger", "err", err) - return releaseNo, manifestPushTemplate, err - } - // asynchronous mode of Helm/ArgoCd installation starts - return impl.workflowEventPublishService.TriggerAsyncRelease(userDeploymentRequestId, overrideRequest, valuesOverrideResponse, newCtx, deployedBy) -} - -func (impl *HandlerServiceImpl) handleCDTriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.handleCDTriggerRelease") - defer span.End() - // Handling for auto trigger - if overrideRequest.UserId == 0 { - overrideRequest.UserId = deployedBy - } - tx, err := impl.transactionUtilImpl.StartTx() - if err != nil { - impl.logger.Errorw("error in starting transaction to update userDeploymentRequest", "error", err) - return releaseNo, manifestPushTemplate, err - } - defer impl.transactionUtilImpl.RollbackTx(tx) - newDeploymentRequest := adapter.NewUserDeploymentRequest(overrideRequest, triggeredAt, overrideRequest.UserId) - // creating new user deployment request - userDeploymentRequestId, err := impl.userDeploymentRequestService.SaveNewDeployment(newCtx, tx, newDeploymentRequest) - if err != nil { - impl.logger.Errorw("error in saving new userDeploymentRequest", "overrideRequest", overrideRequest, "err", err) - return releaseNo, manifestPushTemplate, err - } - timeline := impl.pipelineStatusTimelineService.NewDevtronAppPipelineStatusTimelineDbObject(overrideRequest.WfrId, timelineStatus.TIMELINE_STATUS_DEPLOYMENT_REQUEST_VALIDATED, timelineStatus.TIMELINE_DESCRIPTION_DEPLOYMENT_REQUEST_VALIDATED, deployedBy) - // creating cd pipeline status timeline for deployment trigger request validated - _, err = impl.pipelineStatusTimelineService.SaveTimelineIfNotAlreadyPresent(timeline, tx) - err = impl.transactionUtilImpl.CommitTx(tx) - if err != nil { - impl.logger.Errorw("error in committing transaction to update userDeploymentRequest", "error", err) - return userDeploymentRequestId, manifestPushTemplate, err - } - isAsyncMode, err := impl.isDevtronAsyncInstallModeEnabled(overrideRequest) - if err != nil { - impl.logger.Errorw("error in checking async mode for devtron app", "err", err, "deploymentType", overrideRequest.DeploymentType, - "forceSyncDeployment", overrideRequest.ForceSyncDeployment, "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId) - return userDeploymentRequestId, manifestPushTemplate, err - } - if envDeploymentConfig.IsEmpty() { - deploymentConfig, dbErr := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(overrideRequest.AppId, overrideRequest.EnvId) - if dbErr != nil { - impl.logger.Errorw("error in getting deployment config by appId and envId", "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId, "err", dbErr) - return releaseNo, manifestPushTemplate, dbErr - } - envDeploymentConfig = deploymentConfig - } - if isAsyncMode { - return impl.triggerAsyncRelease(newCtx, overrideRequest, envDeploymentConfig, userDeploymentRequestId, triggeredAt, deployedBy) - } - // synchronous mode of installation starts - return impl.TriggerRelease(newCtx, overrideRequest, envDeploymentConfig, triggeredAt, deployedBy) -} - -func (impl *HandlerServiceImpl) auditDeploymentTriggerHistory(cdWfrId int, valuesOverrideResponse *app.ValuesOverrideResponse, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (err error) { - if valuesOverrideResponse.Pipeline == nil || valuesOverrideResponse.EnvOverride == nil { - impl.logger.Warnw("unable to save histories for deployment trigger, invalid valuesOverrideResponse received", "cdWfrId", cdWfrId) - return nil - } - err1 := impl.deployedConfigurationHistoryService.CreateHistoriesForDeploymentTrigger(ctx, valuesOverrideResponse.Pipeline, valuesOverrideResponse.PipelineStrategy, valuesOverrideResponse.EnvOverride, triggeredAt, triggeredBy) - if err1 != nil { - impl.logger.Errorw("error in saving histories for deployment trigger", "err", err1, "pipelineId", valuesOverrideResponse.Pipeline.Id, "cdWfrId", cdWfrId) - return nil - } - return nil -} - -// TriggerRelease will trigger Install/Upgrade request for Devtron App releases synchronously -func (impl *HandlerServiceImpl) TriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, - envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.TriggerRelease") - defer span.End() - triggerEvent, skipRequest, err := impl.buildTriggerEventForOverrideRequest(overrideRequest, triggeredAt) - if err != nil { - return releaseNo, manifestPushTemplate, err - } - impl.logger.Debugw("processing TriggerRelease", "wfrId", overrideRequest.WfrId, "triggerEvent", triggerEvent) - // request has already been served, skipping - if skipRequest { - impl.logger.Infow("request already served, skipping", "wfrId", overrideRequest.WfrId) - return releaseNo, manifestPushTemplate, nil - } - // build merged values and save PCO history for the release - valuesOverrideResponse, builtChartPath, err := impl.manifestCreationService.BuildManifestForTrigger(newCtx, overrideRequest, envDeploymentConfig, triggeredAt) - - // auditDeploymentTriggerHistory is performed irrespective of BuildManifestForTrigger error - for auditing purposes - historyErr := impl.auditDeploymentTriggerHistory(overrideRequest.WfrId, valuesOverrideResponse, newCtx, triggeredAt, triggeredBy) - if historyErr != nil { - impl.logger.Errorw("error in auditing deployment trigger history", "cdWfrId", overrideRequest.WfrId, "err", err) - return releaseNo, manifestPushTemplate, err - } - if err != nil { - impl.logger.Errorw("error in building merged manifest for trigger", "err", err) - impl.manifestGenerationFailedTimelineHandling(triggerEvent, overrideRequest, err) - return releaseNo, manifestPushTemplate, err - } - helmManifest, err := impl.getHelmManifestForTriggerRelease(ctx, triggerEvent, overrideRequest, valuesOverrideResponse, builtChartPath) - if err != nil { - impl.logger.Errorw("error, getHelmManifestForTriggerRelease", "err", err) - return releaseNo, manifestPushTemplate, err - } - impl.logger.Debugw("triggering pipeline for release", "wfrId", overrideRequest.WfrId, "builtChartPath", builtChartPath) - releaseNo, err = impl.triggerPipeline(overrideRequest, valuesOverrideResponse, builtChartPath, triggerEvent, newCtx) - if err != nil { - return 0, manifestPushTemplate, err - } - - err = impl.triggerReleaseSuccessHandling(triggerEvent, overrideRequest, valuesOverrideResponse, helmManifest) - if err != nil { - impl.logger.Errorw("error, triggerReleaseSuccessHandling", "triggerEvent", triggerEvent, "err", err) - return releaseNo, manifestPushTemplate, err - } - // creating cd pipeline status timeline for deployment triggered - for successfully triggered requests - timeline := impl.pipelineStatusTimelineService.NewDevtronAppPipelineStatusTimelineDbObject(overrideRequest.WfrId, timelineStatus.TIMELINE_STATUS_DEPLOYMENT_TRIGGERED, timelineStatus.TIMELINE_DESCRIPTION_DEPLOYMENT_COMPLETED, overrideRequest.UserId) - _, dbErr := impl.pipelineStatusTimelineService.SaveTimelineIfNotAlreadyPresent(timeline, nil) - if dbErr != nil { - impl.logger.Errorw("error in creating timeline status for deployment completed", "err", dbErr, "timeline", timeline) - } - impl.logger.Debugw("triggered pipeline for release successfully", "wfrId", overrideRequest.WfrId, "builtChartPath", builtChartPath) - return releaseNo, valuesOverrideResponse.ManifestPushTemplate, nil -} - -func (impl *HandlerServiceImpl) performGitOps(ctx context.Context, - overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, - builtChartPath string, triggerEvent bean.TriggerEvent) error { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.performGitOps") - defer span.End() - // update workflow runner status, used in app workflow view - err := impl.cdWorkflowCommonService.UpdateNonTerminalStatusInRunner(newCtx, overrideRequest.WfrId, overrideRequest.UserId, cdWorkflow.WorkflowInProgress) - if err != nil { - impl.logger.Errorw("error in updating the workflow runner status", "err", err) - return err - } - manifestPushTemplate, err := impl.buildManifestPushTemplate(overrideRequest, valuesOverrideResponse, builtChartPath) - if err != nil { - impl.logger.Errorw("error in building manifest push template", "err", err) - return err - } - manifestPushService := impl.getManifestPushService(triggerEvent.ManifestStorageType) - manifestPushResponse := manifestPushService.PushChart(newCtx, manifestPushTemplate) - if manifestPushResponse.Error != nil { - impl.logger.Errorw("error in pushing manifest to git/helm", "err", manifestPushResponse.Error, "git_repo_url", manifestPushTemplate.RepoUrl) - return manifestPushResponse.Error - } - if manifestPushResponse.IsNewGitRepoConfigured() { - // Update GitOps repo url after repo new repo created - valuesOverrideResponse.DeploymentConfig.SetRepoURL(manifestPushResponse.NewGitRepoUrl) - } - valuesOverrideResponse.ManifestPushTemplate = manifestPushTemplate - return nil -} - -func (impl *HandlerServiceImpl) buildTriggerEventForOverrideRequest(overrideRequest *bean3.ValuesOverrideRequest, triggeredAt time.Time) (triggerEvent bean.TriggerEvent, skipRequest bool, err error) { - triggerEvent = helper.NewTriggerEvent(overrideRequest.DeploymentAppType, triggeredAt, overrideRequest.UserId) - request := statusBean.NewTimelineGetRequest(). - WithCdWfrId(overrideRequest.WfrId). - ExcludingStatuses(timelineStatus.TIMELINE_STATUS_UNABLE_TO_FETCH_STATUS, - timelineStatus.TIMELINE_STATUS_KUBECTL_APPLY_STARTED, - timelineStatus.TIMELINE_STATUS_KUBECTL_APPLY_SYNCED) - timelineStatuses, err := impl.pipelineStatusTimelineService.GetTimelineStatusesFor(request) - if err != nil { - impl.logger.Errorw("error in getting last timeline status by cdWfrId", "cdWfrId", overrideRequest.WfrId, "err", err) - return triggerEvent, skipRequest, err - } else if !slices.Contains(timelineStatuses, timelineStatus.TIMELINE_STATUS_DEPLOYMENT_REQUEST_VALIDATED) { - impl.logger.Errorw("pre-condition missing: timeline for deployment request validation", "cdWfrId", overrideRequest.WfrId, "timelineStatuses", timelineStatuses) - return triggerEvent, skipRequest, fmt.Errorf("pre-condition missing: timeline for deployment request validation") - } else if timelineStatus.ContainsTerminalTimelineStatus(timelineStatuses) { - impl.logger.Info("deployment is already terminated", "cdWfrId", overrideRequest.WfrId, "timelineStatuses", timelineStatuses) - skipRequest = true - return triggerEvent, skipRequest, nil - } else if slices.Contains(timelineStatuses, timelineStatus.TIMELINE_STATUS_DEPLOYMENT_TRIGGERED) { - impl.logger.Info("deployment has been performed. skipping", "cdWfrId", overrideRequest.WfrId, "timelineStatuses", timelineStatuses) - skipRequest = true - return triggerEvent, skipRequest, nil - } - if slices.Contains(timelineStatuses, timelineStatus.TIMELINE_STATUS_GIT_COMMIT) || - slices.Contains(timelineStatuses, timelineStatus.TIMELINE_STATUS_ARGOCD_SYNC_INITIATED) { - // git commit has already been performed - triggerEvent.PerformChartPush = false - } - if slices.Contains(timelineStatuses, timelineStatus.TIMELINE_STATUS_ARGOCD_SYNC_COMPLETED) { - // ArgoCd sync has already been performed - triggerEvent.DeployArgoCdApp = false - } - return triggerEvent, skipRequest, nil -} - -func (impl *HandlerServiceImpl) triggerPipeline(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, triggerEvent bean.TriggerEvent, ctx context.Context) (releaseNo int, err error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.triggerPipeline") - defer span.End() - if triggerEvent.PerformChartPush { - impl.logger.Debugw("performing chart push operation in triggerPipeline", "cdWfrId", overrideRequest.WfrId) - err = impl.performGitOps(newCtx, overrideRequest, valuesOverrideResponse, builtChartPath, triggerEvent) - if err != nil { - impl.logger.Errorw("error in performing GitOps", "cdWfrId", overrideRequest.WfrId, "err", err) - return releaseNo, err - } - impl.logger.Debugw("chart push operation completed successfully", "cdWfrId", overrideRequest.WfrId) - } - - if triggerEvent.PerformDeploymentOnCluster { - err = impl.deployApp(newCtx, overrideRequest, valuesOverrideResponse, triggerEvent) - if err != nil { - impl.logger.Errorw("error in deploying app", "err", err) - return releaseNo, err - } - } - - go impl.writeCDTriggerEvent(overrideRequest, valuesOverrideResponse.Artifact, valuesOverrideResponse.PipelineOverride.PipelineReleaseCounter, valuesOverrideResponse.PipelineOverride.Id, overrideRequest.WfrId) - - _ = impl.markImageScanDeployed(newCtx, overrideRequest.AppId, overrideRequest.EnvId, overrideRequest.ClusterId, - valuesOverrideResponse.Artifact.ImageDigest, valuesOverrideResponse.Artifact.ScanEnabled, valuesOverrideResponse.Artifact.Image) - - middleware.CdTriggerCounter.WithLabelValues(overrideRequest.AppName, overrideRequest.EnvName).Inc() - - // Update previous deployment runner status (in transaction): Failed - dbErr := impl.cdWorkflowCommonService.SupersedePreviousDeployments(newCtx, overrideRequest.WfrId, overrideRequest.PipelineId, triggerEvent.TriggeredAt, overrideRequest.UserId) - if dbErr != nil { - impl.logger.Errorw("error while update previous cd workflow runners", "err", dbErr, "currentRunnerId", overrideRequest.WfrId, "pipelineId", overrideRequest.PipelineId) - return releaseNo, dbErr - } - return valuesOverrideResponse.PipelineOverride.PipelineReleaseCounter, nil -} - -func (impl *HandlerServiceImpl) buildManifestPushTemplate(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string) (*bean4.ManifestPushTemplate, error) { - - manifestPushTemplate := &bean4.ManifestPushTemplate{ - WorkflowRunnerId: overrideRequest.WfrId, - AppId: overrideRequest.AppId, - ChartRefId: valuesOverrideResponse.EnvOverride.Chart.ChartRefId, - EnvironmentId: valuesOverrideResponse.EnvOverride.Environment.Id, - EnvironmentName: valuesOverrideResponse.EnvOverride.Environment.Namespace, - UserId: overrideRequest.UserId, - PipelineOverrideId: valuesOverrideResponse.PipelineOverride.Id, - AppName: overrideRequest.AppName, - TargetEnvironmentId: valuesOverrideResponse.EnvOverride.TargetEnvironment, - BuiltChartPath: builtChartPath, - MergedValues: valuesOverrideResponse.MergedValues, - } - - manifestPushConfig, err := impl.manifestPushConfigRepository.GetManifestPushConfigByAppIdAndEnvId(overrideRequest.AppId, overrideRequest.EnvId) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in fetching manifest push config from db", "err", err) - return manifestPushTemplate, err - } - - if manifestPushConfig != nil && manifestPushConfig.Id != 0 { - if manifestPushConfig.StorageType == bean2.ManifestStorageGit { - // need to implement for git repo push - // currently manifest push config doesn't have git push config. GitOps config is derived from charts, chart_env_config_override and chart_ref table - } else { - err2 := impl.buildManifestPushTemplateForNonGitStorageType(overrideRequest, valuesOverrideResponse, builtChartPath, err, manifestPushConfig, manifestPushTemplate) - if err2 != nil { - return manifestPushTemplate, err2 - } - } - } else { - manifestPushTemplate.ChartReferenceTemplate = valuesOverrideResponse.EnvOverride.Chart.ReferenceTemplate - manifestPushTemplate.ChartName = valuesOverrideResponse.EnvOverride.Chart.ChartName - manifestPushTemplate.ChartVersion = valuesOverrideResponse.EnvOverride.Chart.ChartVersion - manifestPushTemplate.ChartLocation = valuesOverrideResponse.DeploymentConfig.GetChartLocation() - manifestPushTemplate.RepoUrl = valuesOverrideResponse.DeploymentConfig.GetRepoURL() - manifestPushTemplate.TargetRevision = valuesOverrideResponse.DeploymentConfig.GetTargetRevision() - manifestPushTemplate.ValuesFilePath = valuesOverrideResponse.DeploymentConfig.GetValuesFilePath() - manifestPushTemplate.ReleaseMode = valuesOverrideResponse.DeploymentConfig.ReleaseMode - manifestPushTemplate.IsCustomGitRepository = common.IsCustomGitOpsRepo(valuesOverrideResponse.DeploymentConfig.ConfigType) - manifestPushTemplate.IsArgoSyncSupported = valuesOverrideResponse.DeploymentConfig.IsArgoAppSyncAndRefreshSupported() - } - return manifestPushTemplate, nil -} - -func (impl *HandlerServiceImpl) deployApp(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, triggerEvent bean.TriggerEvent) error { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.deployApp") - defer span.End() - var referenceChartByte []byte - var err error - - if util.IsAcdApp(overrideRequest.DeploymentAppType) && triggerEvent.DeployArgoCdApp { - err = impl.deployArgoCdApp(newCtx, overrideRequest, valuesOverrideResponse) - if err != nil { - impl.logger.Errorw("error in deploying app on ArgoCd", "err", err) - return err - } - } else if util.IsHelmApp(overrideRequest.DeploymentAppType) { - _, referenceChartByte, err = impl.createHelmAppForCdPipeline(newCtx, overrideRequest, valuesOverrideResponse) - if err != nil { - impl.logger.Errorw("error in creating or updating helm application for cd pipeline", "err", err) - return err - } - } - impl.postDeployHook(overrideRequest, valuesOverrideResponse, referenceChartByte, err) - return nil -} - -func (impl *HandlerServiceImpl) createHelmAppForCdPipeline(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse) (bool, []byte, error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.createHelmAppForCdPipeline") - defer span.End() - pipelineModel := valuesOverrideResponse.Pipeline - envOverride := valuesOverrideResponse.EnvOverride - mergeAndSave := valuesOverrideResponse.MergedValues - chartMetaData, helmRevisionHistory, releaseIdentifier, err := impl.getHelmHistoryLimitAndChartMetadataForHelmAppCreation(ctx, valuesOverrideResponse) - if err != nil { - impl.logger.Errorw("error, getHelmHistoryLimitAndChartMetadataForHelmAppCreation", "valuesOverrideResponse", valuesOverrideResponse, "err", err) - return false, nil, err - } - referenceTemplate := envOverride.Chart.ReferenceTemplate - referenceTemplatePath := path.Join(bean5.RefChartDirPath, referenceTemplate) - var referenceChartByte []byte - if util.IsHelmApp(valuesOverrideResponse.DeploymentConfig.DeploymentAppType) { - sanitizedK8sVersion, err := impl.getSanitizedK8sVersion(referenceTemplate) - if err != nil { - return false, nil, err - } - referenceChartByte, err = impl.getReferenceChartByteForHelmTypeApp(envOverride, chartMetaData, referenceTemplatePath, overrideRequest, valuesOverrideResponse) - if err != nil { - impl.logger.Errorw("error, getReferenceChartByteForHelmTypeApp", "envOverride", envOverride, "err", err) - return false, nil, err - } - if pipelineModel.DeploymentAppCreated { - req := &gRPC.UpgradeReleaseRequest{ - ReleaseIdentifier: releaseIdentifier, - ValuesYaml: mergeAndSave, - HistoryMax: helmRevisionHistory, - ChartContent: &gRPC.ChartContent{Content: referenceChartByte}, - } - if len(sanitizedK8sVersion) > 0 { - req.K8SVersion = sanitizedK8sVersion - } - if impl.isDevtronAsyncHelmInstallModeEnabled(overrideRequest.ForceSyncDeployment) { - req.RunInCtx = true - } - // For cases where helm release was not found, kubelink will install the same configuration - updateApplicationResponse, err := impl.helmAppClient.UpdateApplication(newCtx, req) - if err != nil { - impl.logger.Errorw("error in updating helm application for cd pipelineModel", "err", err) - if util.IsErrorContextCancelled(err) { - return false, nil, cdWorkflow.ErrorDeploymentSuperseded - } else if util.IsErrorContextDeadlineExceeded(err) { - return false, nil, context.DeadlineExceeded - } - apiError := clientErrors.ConvertToApiError(err) - if apiError != nil { - return false, nil, apiError - } - return false, nil, err - } else { - impl.logger.Debugw("updated helm application", "response", updateApplicationResponse, "isSuccess", updateApplicationResponse.Success) - } - - } else { - - helmResponse, err := impl.helmInstallReleaseWithCustomChart(newCtx, releaseIdentifier, referenceChartByte, - mergeAndSave, sanitizedK8sVersion, overrideRequest.ForceSyncDeployment) - - // For connection related errors, no need to update the db - if err != nil && strings.Contains(err.Error(), "connection error") { - impl.logger.Errorw("error in helm install custom chart", "err", err) - return false, nil, err - } - - // IMP: update cd pipelineModel to mark deployment app created, even if helm install fails - // If the helm install fails, it still creates the app in failed state, so trying to - // re-create the app results in error from helm that cannot re-use name which is still in use - _, pgErr := impl.updatePipeline(pipelineModel, overrideRequest.UserId) - - if err != nil { - impl.logger.Errorw("error in helm install custom chart", "err", err) - if pgErr != nil { - impl.logger.Errorw("failed to update deployment app created flag in pipelineModel table", "err", err) - } - if util.IsErrorContextCancelled(err) { - return false, nil, cdWorkflow.ErrorDeploymentSuperseded - } else if util.IsErrorContextDeadlineExceeded(err) { - return false, nil, context.DeadlineExceeded - } - apiError := clientErrors.ConvertToApiError(err) - if apiError != nil { - return false, nil, apiError - } - return false, nil, err - } - - if pgErr != nil { - impl.logger.Errorw("failed to update deployment app created flag in pipelineModel table", "err", err) - return false, nil, err - } - - impl.logger.Debugw("received helm release response", "helmResponse", helmResponse, "isSuccess", helmResponse.Success) - } - - //update workflow runner status, used in app workflow view - err = impl.cdWorkflowCommonService.UpdateNonTerminalStatusInRunner(newCtx, overrideRequest.WfrId, overrideRequest.UserId, cdWorkflow.WorkflowInProgress) - if err != nil { - impl.logger.Errorw("error in updating the workflow runner status, createHelmAppForCdPipeline", "err", err) - return false, nil, err - } - } - return true, referenceChartByte, nil -} - -func (impl *HandlerServiceImpl) getHelmHistoryLimitAndChartMetadataForHelmAppCreation(ctx context.Context, - valuesOverrideResponse *app.ValuesOverrideResponse) (*chart.Metadata, int32, *gRPC.ReleaseIdentifier, error) { - pipelineModel := valuesOverrideResponse.Pipeline - envOverride := valuesOverrideResponse.EnvOverride - - var chartMetaData *chart.Metadata - releaseName := pipelineModel.DeploymentAppName - //getting cluster by id - cluster, err := impl.clusterRepository.FindById(envOverride.Environment.ClusterId) - if err != nil { - impl.logger.Errorw("error in getting cluster by id", "clusterId", envOverride.Environment.ClusterId, "err", err) - return nil, 0, nil, err - } else if cluster == nil { - impl.logger.Errorw("error in getting cluster by id, found nil object", "clusterId", envOverride.Environment.ClusterId) - return nil, 0, nil, err - } - - clusterConfig := impl.getClusterGRPCConfig(*cluster) - - releaseIdentifier := &gRPC.ReleaseIdentifier{ - ReleaseName: releaseName, - ReleaseNamespace: envOverride.Namespace, - ClusterConfig: clusterConfig, - } - - var helmRevisionHistory int32 - if valuesOverrideResponse.DeploymentConfig.ReleaseMode == util.PIPELINE_RELEASE_MODE_LINK { - detail, err := impl.helmAppClient.GetReleaseDetails(ctx, releaseIdentifier) - if err != nil { - impl.logger.Errorw("error in fetching release details", "clusterId", clusterConfig.ClusterId, "namespace", envOverride.Namespace, "releaseName", releaseName, "err", err) - return nil, 0, nil, err - } - chartMetaData = &chart.Metadata{ - Name: detail.ChartName, - Version: detail.ChartVersion, - } - //not modifying revision history in case of linked release - helmRevisionHistory = impl.helmAppService.GetRevisionHistoryMaxValue(bean6.SOURCE_LINKED_HELM_APP) - } else { - chartMetaData = &chart.Metadata{ - Name: pipelineModel.App.AppName, - Version: envOverride.Chart.ChartVersion, - } - helmRevisionHistory = impl.helmAppService.GetRevisionHistoryMaxValue(bean6.SOURCE_DEVTRON_APP) - } - - return chartMetaData, helmRevisionHistory, releaseIdentifier, nil -} - -func (impl *HandlerServiceImpl) deployArgoCdApp(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, - valuesOverrideResponse *app.ValuesOverrideResponse) error { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.deployArgoCdApp") - defer span.End() - name, err := impl.createArgoApplicationIfRequired(newCtx, valuesOverrideResponse.EnvOverride, valuesOverrideResponse.Pipeline, valuesOverrideResponse.DeploymentConfig, overrideRequest.UserId) - if err != nil { - impl.logger.Errorw("acd application create error on cd trigger", "err", err, "req", overrideRequest) - return err - } - impl.logger.Debugw("ArgoCd application created", "name", name) - updateAppInArgoCd, err := impl.updateArgoPipeline(newCtx, valuesOverrideResponse.Pipeline, valuesOverrideResponse.EnvOverride, valuesOverrideResponse.DeploymentConfig) - if err != nil { - impl.logger.Errorw("error in updating argocd app ", "err", err) - return err - } - if valuesOverrideResponse.DeploymentConfig.IsArgoAppSyncAndRefreshSupported() { - syncTime := time.Now() - targetRevision := valuesOverrideResponse.DeploymentConfig.GetTargetRevision() - err = impl.argoClientWrapperService.SyncArgoCDApplicationIfNeededAndRefresh(newCtx, valuesOverrideResponse.Pipeline.DeploymentAppName, targetRevision) - if err != nil { - impl.logger.Errorw("error in getting argo application with normal refresh", "argoAppName", valuesOverrideResponse.Pipeline.DeploymentAppName) - return fmt.Errorf("%s. err: %s", bean.ARGOCD_SYNC_ERROR, util.GetClientErrorDetailedMessage(err)) - } - if impl.ACDConfig.IsManualSyncEnabled() { - timeline := &pipelineConfig.PipelineStatusTimeline{ - CdWorkflowRunnerId: overrideRequest.WfrId, - StatusTime: syncTime, - Status: timelineStatus.TIMELINE_STATUS_ARGOCD_SYNC_COMPLETED, - StatusDetail: timelineStatus.TIMELINE_DESCRIPTION_ARGOCD_SYNC_COMPLETED, - } - timeline.CreateAuditLog(overrideRequest.UserId) - _, err = impl.pipelineStatusTimelineService.SaveTimelineIfNotAlreadyPresent(timeline, nil) - if err != nil { - impl.logger.Errorw("error in saving pipeline status timeline", "err", err) - } - } - } - if updateAppInArgoCd { - impl.logger.Debug("argo-cd successfully updated") - } else { - impl.logger.Debug("argo-cd failed to update, ignoring it") - } - return nil -} - -// update repoUrl, revision and argo app sync mode (auto/manual) if needed -func (impl *HandlerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envOverride *bean10.EnvConfigOverride, deploymentConfig *bean9.DeploymentConfig) (bool, error) { - if !deploymentConfig.IsArgoAppPatchSupported() { - impl.logger.Infow("argo app patch not supported", "pipelineId", pipeline.Id, "pipelineName", pipeline.Name) - return false, nil - } - if ctx == nil { - impl.logger.Errorw("err in syncing ACD, ctx is NULL", "pipelineId", pipeline.Id, "pipelineName", pipeline.Name) - return false, nil - } - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.updateArgoPipeline") - defer span.End() - argoAppName := pipeline.DeploymentAppName - impl.logger.Infow("received payload, updateArgoPipeline", "appId", pipeline.AppId, "pipelineName", pipeline.Name, "envId", envOverride.TargetEnvironment, "argoAppName", argoAppName) - argoApplication, err := impl.argoClientWrapperService.GetArgoAppByName(newCtx, argoAppName) - if err != nil { - impl.logger.Errorw("unable to get ArgoCd app", "app", argoAppName, "pipeline", pipeline.Name, "err", err) - return false, err - } - // if status, ok:=status.FromError(err);ok{ - appStatus, _ := status2.FromError(err) - if appStatus.Code() == codes.OK { - impl.logger.Debugw("argo app exists", "app", argoAppName, "pipeline", pipeline.Name) - if impl.argoClientWrapperService.IsArgoAppPatchRequired(argoApplication.Spec.Source, deploymentConfig.GetRepoURL(), deploymentConfig.GetTargetRevision(), deploymentConfig.GetChartLocation()) { - patchRequestDto := &bean7.ArgoCdAppPatchReqDto{ - ArgoAppName: argoAppName, - ChartLocation: deploymentConfig.GetChartLocation(), - GitRepoUrl: deploymentConfig.GetRepoURL(), - TargetRevision: deploymentConfig.GetTargetRevision(), - PatchType: bean7.PatchTypeMerge, - } - url, err := impl.gitOperationService.GetRepoUrlWithUserName(deploymentConfig.GetRepoURL()) - if err != nil { - return false, err - } - patchRequestDto.GitRepoUrl = url - err = impl.argoClientWrapperService.PatchArgoCdApp(newCtx, patchRequestDto) - if err != nil { - impl.logger.Errorw("error in patching argo pipeline", "err", err, "req", patchRequestDto) - return false, err - } - if deploymentConfig.GetRepoURL() != argoApplication.Spec.Source.RepoURL { - impl.logger.Infow("patching argo application's repo url", "argoAppName", argoAppName) - } - if deploymentConfig.GetTargetRevision() != argoApplication.Spec.Source.TargetRevision { - impl.logger.Infow("patching argo application's revision", "argoAppName", argoAppName) - } - impl.logger.Debugw("pipeline update req", "res", patchRequestDto) - } else { - impl.logger.Debug("pipeline no need to update ") - } - err := impl.argoClientWrapperService.UpdateArgoCDSyncModeIfNeeded(newCtx, argoApplication) - if err != nil { - impl.logger.Errorw("error in updating argocd sync mode", "err", err) - return false, err - } - return true, nil - } else if appStatus.Code() == codes.NotFound { - impl.logger.Errorw("argo app not found", "app", argoAppName, "pipeline", pipeline.Name) - return false, nil - } else { - impl.logger.Errorw("err in checking application on argoCD", "err", err, "pipeline", pipeline.Name) - return false, err - } -} - -func (impl *HandlerServiceImpl) createArgoApplicationIfRequired(ctx context.Context, envConfigOverride *bean10.EnvConfigOverride, - pipeline *pipelineConfig.Pipeline, deploymentConfig *bean9.DeploymentConfig, userId int32) (string, error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.createArgoApplicationIfRequired") - defer span.End() - envModel, err := impl.envRepository.FindById(envConfigOverride.TargetEnvironment) - if err != nil { - return "", err - } - argoAppName := pipeline.DeploymentAppName - if !deploymentConfig.IsArgoAppCreationRequired(pipeline.DeploymentAppCreated) { - return argoAppName, nil - } else { - impl.logger.Debugw("new pipeline found", "pipeline", pipeline) - // create - appNamespace := envConfigOverride.Namespace - if appNamespace == "" { - appNamespace = "default" - } - namespace := argocdServer.DevtronInstalationNs - - appRequest := &argocdServer.AppTemplate{ - ApplicationName: argoAppName, - Namespace: namespace, - TargetNamespace: appNamespace, - TargetServer: envModel.Cluster.ServerUrl, - Project: "default", - ValuesFile: helper.GetValuesFileForEnv(envModel.Id), - RepoPath: deploymentConfig.GetChartLocation(), - RepoUrl: deploymentConfig.GetRepoURL(), - AutoSyncEnabled: impl.ACDConfig.ArgoCDAutoSyncEnabled, - } - appRequest.RepoUrl, err = impl.gitOperationService.GetRepoUrlWithUserName(appRequest.RepoUrl) - if err != nil { - return "", err - } - createdArgoAppName, err := impl.argoK8sClient.CreateAcdApp(newCtx, appRequest, argocdServer.ARGOCD_APPLICATION_TEMPLATE) - if err != nil { - return "", err - } - // update cd pipeline to mark deployment app created - _, err = impl.updatePipeline(pipeline, userId) - if err != nil { - impl.logger.Errorw("error in update cd pipeline for deployment app created or not", "err", err) - return "", err - } - return createdArgoAppName, nil - } -} - -func (impl *HandlerServiceImpl) updatePipeline(pipeline *pipelineConfig.Pipeline, userId int32) (bool, error) { - err := impl.pipelineRepository.SetDeploymentAppCreatedInPipeline(true, pipeline.Id, userId) - if err != nil { - impl.logger.Errorw("error on updating cd pipeline for setting deployment app created", "err", err) - return false, err - } - return true, nil -} - -// helmInstallReleaseWithCustomChart performs helm install with custom chart -func (impl *HandlerServiceImpl) helmInstallReleaseWithCustomChart(ctx context.Context, releaseIdentifier *gRPC.ReleaseIdentifier, - referenceChartByte []byte, valuesYaml, k8sServerVersion string, forceSync bool) (*gRPC.HelmInstallCustomResponse, error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.helmInstallReleaseWithCustomChart") - defer span.End() - helmInstallRequest := gRPC.HelmInstallCustomRequest{ - ValuesYaml: valuesYaml, - ChartContent: &gRPC.ChartContent{Content: referenceChartByte}, - ReleaseIdentifier: releaseIdentifier, - } - if len(k8sServerVersion) > 0 { - helmInstallRequest.K8SVersion = k8sServerVersion - } - if impl.isDevtronAsyncHelmInstallModeEnabled(forceSync) { - helmInstallRequest.RunInCtx = true - } - // Request exec - return impl.helmAppClient.InstallReleaseWithCustomChart(newCtx, &helmInstallRequest) -} - func (impl *HandlerServiceImpl) writeCDTriggerEvent(overrideRequest *bean3.ValuesOverrideRequest, artifact *repository3.CiArtifact, releaseId, pipelineOverrideId, wfrId int) { event, err := impl.eventFactory.Build(util2.Trigger, &overrideRequest.PipelineId, overrideRequest.AppId, &overrideRequest.EnvId, util2.CD) @@ -1466,184 +343,3 @@ func (impl *HandlerServiceImpl) writeCDTriggerEvent(overrideRequest *bean3.Value impl.logger.Errorw("error in writing cd trigger event", "err", err) } } - -func (impl *HandlerServiceImpl) markImageScanDeployed(ctx context.Context, appId, envId, clusterId int, - imageDigest string, isScanEnabled bool, image string) error { - _, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.markImageScanDeployed") - defer span.End() - // TODO KB: send NATS event for self consumption - impl.logger.Debugw("mark image scan deployed for devtron app, from cd auto or manual trigger", "imageDigest", imageDigest) - executionHistory, err := impl.imageScanHistoryReadService.FindByImageAndDigest(imageDigest, image) - if err != nil && !errors.Is(err, pg.ErrNoRows) { - impl.logger.Errorw("error in fetching execution history", "err", err) - return err - } - if errors.Is(err, pg.ErrNoRows) || executionHistory == nil || executionHistory.Id == 0 { - if isScanEnabled { - // There should ImageScanHistory for ScanEnabled artifacts - impl.logger.Errorw("no execution history found for digest", "digest", imageDigest) - return fmt.Errorf("no execution history found for digest - %s", imageDigest) - } else { - // For ScanDisabled artifacts it should be an expected condition - impl.logger.Infow("no execution history found for digest", "digest", imageDigest) - return nil - } - } - impl.logger.Debugw("saving image_scan_deploy_info for cd auto or manual trigger", "executionHistory", executionHistory) - var ids []int - ids = append(ids, executionHistory.Id) - - ot, err := impl.imageScanDeployInfoReadService.FetchByAppIdAndEnvId(appId, envId, []string{repository6.ScanObjectType_APP}) - - if err == pg.ErrNoRows && !isScanEnabled { - // ignoring if no rows are found and scan is disabled - return nil - } - - if err != nil && err != pg.ErrNoRows { - return err - } else if err == pg.ErrNoRows && isScanEnabled { - imageScanDeployInfo := &repository6.ImageScanDeployInfo{ - ImageScanExecutionHistoryId: ids, - ScanObjectMetaId: appId, - ObjectType: repository6.ScanObjectType_APP, - EnvId: envId, - ClusterId: clusterId, - AuditLog: sql.AuditLog{ - CreatedOn: time.Now(), - CreatedBy: 1, - UpdatedOn: time.Now(), - UpdatedBy: 1, - }, - } - impl.logger.Debugw("mark image scan deployed for normal app, from cd auto or manual trigger", "imageScanDeployInfo", imageScanDeployInfo) - err = impl.imageScanDeployInfoService.Save(imageScanDeployInfo) - if err != nil { - impl.logger.Errorw("error in creating deploy info", "err", err) - } - } else { - // Updating Execution history for Latest Deployment to fetch out security Vulnerabilities for latest deployed info - if isScanEnabled { - ot.ImageScanExecutionHistoryId = ids - } else { - arr := []int{-1} - ot.ImageScanExecutionHistoryId = arr - } - err = impl.imageScanDeployInfoService.Update(ot) - if err != nil { - impl.logger.Errorw("error in updating deploy info for latest deployed image", "err", err) - } - } - return err -} - -func (impl *HandlerServiceImpl) isDevtronAsyncHelmInstallModeEnabled(forceSync bool) bool { - return impl.globalEnvVariables.EnableAsyncHelmInstallDevtronChart && !forceSync -} - -func (impl *HandlerServiceImpl) isDevtronAsyncInstallModeEnabled(overrideRequest *bean3.ValuesOverrideRequest) (bool, error) { - if util.IsHelmApp(overrideRequest.DeploymentAppType) { - return impl.isDevtronAsyncHelmInstallModeEnabled(overrideRequest.ForceSyncDeployment), nil - } else if util.IsAcdApp(overrideRequest.DeploymentAppType) { - return impl.isDevtronAsyncArgoCdInstallModeEnabledForApp(overrideRequest.AppId, - overrideRequest.EnvId, overrideRequest.ForceSyncDeployment) - } - return false, nil -} - -func (impl *HandlerServiceImpl) deleteCorruptedPipelineStage(pipelineStage *repository.PipelineStage, triggeredBy int32) (error, bool) { - if pipelineStage != nil { - stageReq := &bean8.PipelineStageDto{ - Id: pipelineStage.Id, - Type: pipelineStage.Type, - } - err, deleted := impl.pipelineStageService.DeletePipelineStageIfReq(stageReq, triggeredBy) - if err != nil { - impl.logger.Errorw("error in deleting the corrupted pipeline stage", "err", err, "pipelineStageReq", stageReq) - return err, false - } - return nil, deleted - } - return nil, false -} - -func (impl *HandlerServiceImpl) handleCustomGitOpsRepoValidation(runner *pipelineConfig.CdWorkflowRunner, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredBy int32) error { - if !util.IsAcdApp(pipeline.DeploymentAppName) { - return nil - } - isGitOpsConfigured := false - gitOpsConfig, err := impl.gitOpsConfigReadService.GetGitOpsConfigActive() - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error while getting active GitOpsConfig", "err", err) - } - if gitOpsConfig != nil && gitOpsConfig.Id > 0 { - isGitOpsConfigured = true - } - if isGitOpsConfigured && gitOpsConfig.AllowCustomRepository { - //chart, err := impl.chartRepository.FindLatestChartForAppByAppId(pipeline.AppId) - //if err != nil { - // impl.logger.Errorw("error in fetching latest chart for app by appId", "err", err, "appId", pipeline.AppId) - // return err - //} - if gitOps.IsGitOpsRepoNotConfigured(envDeploymentConfig.GetRepoURL()) { - if err = impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, errors.New(cdWorkflow.GITOPS_REPO_NOT_CONFIGURED), triggeredBy); err != nil { - impl.logger.Errorw("error while updating current runner status to failed, TriggerDeployment", "wfrId", runner.Id, "err", err) - } - apiErr := &util.ApiError{ - HttpStatusCode: http.StatusConflict, - UserMessage: cdWorkflow.GITOPS_REPO_NOT_CONFIGURED, - InternalMessage: cdWorkflow.GITOPS_REPO_NOT_CONFIGURED, - } - return apiErr - } - } - return nil -} - -func (impl *HandlerServiceImpl) getSanitizedK8sVersion(referenceTemplate string) (string, error) { - var sanitizedK8sVersion string - //handle specific case for all cronjob charts from cronjob-chart_1-2-0 to cronjob-chart_1-5-0 where semverCompare - //comparison func has wrong api version mentioned, so for already installed charts via these charts that comparison - //is always false, handles the gh issue:- https://github.com/devtron-labs/devtron/issues/4860 - cronJobChartRegex := regexp.MustCompile(bean.CronJobChartRegexExpression) - if cronJobChartRegex.MatchString(referenceTemplate) { - k8sServerVersion, err := impl.K8sUtil.GetKubeVersion() - if err != nil { - impl.logger.Errorw("exception caught in getting k8sServerVersion", "err", err) - return "", err - } - sanitizedK8sVersion = k8s2.StripPrereleaseFromK8sVersion(k8sServerVersion.String()) - } - return sanitizedK8sVersion, nil -} - -func (impl *HandlerServiceImpl) getReferenceChartByteForHelmTypeApp(envOverride *bean10.EnvConfigOverride, - chartMetaData *chart.Metadata, referenceTemplatePath string, overrideRequest *bean3.ValuesOverrideRequest, - valuesOverrideResponse *app.ValuesOverrideResponse) ([]byte, error) { - referenceChartByte := envOverride.Chart.ReferenceChart - // here updating reference chart into database. - if len(envOverride.Chart.ReferenceChart) == 0 { - refChartByte, err := impl.chartTemplateService.GetByteArrayRefChart(chartMetaData, referenceTemplatePath) - if err != nil { - impl.logger.Errorw("ref chart commit error on cd trigger", "err", err, "req", overrideRequest) - return nil, err - } - ch := envOverride.Chart - ch.ReferenceChart = refChartByte - ch.UpdatedOn = time.Now() - ch.UpdatedBy = overrideRequest.UserId - err = impl.chartRepository.Update(ch) - if err != nil { - impl.logger.Errorw("chart update error", "err", err, "req", overrideRequest) - return nil, err - } - referenceChartByte = refChartByte - } - var err error - referenceChartByte, err = impl.overrideReferenceChartByteForHelmTypeApp(valuesOverrideResponse, chartMetaData, referenceTemplatePath, referenceChartByte) - if err != nil { - impl.logger.Errorw("ref chart commit error on cd trigger", "err", err, "req", overrideRequest) - return nil, err - } - return referenceChartByte, nil -} diff --git a/pkg/deployment/trigger/devtronApps/HandlerService_ent1.go b/pkg/deployment/trigger/devtronApps/deployStageHandlerCode_ent.go similarity index 84% rename from pkg/deployment/trigger/devtronApps/HandlerService_ent1.go rename to pkg/deployment/trigger/devtronApps/deployStageHandlerCode_ent.go index 24b7dab264..347f980a07 100644 --- a/pkg/deployment/trigger/devtronApps/HandlerService_ent1.go +++ b/pkg/deployment/trigger/devtronApps/deployStageHandlerCode_ent.go @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package devtronApps import ( diff --git a/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go b/pkg/deployment/trigger/devtronApps/postStageHandlerCode.go similarity index 100% rename from pkg/deployment/trigger/devtronApps/PostStageTriggerService.go rename to pkg/deployment/trigger/devtronApps/postStageHandlerCode.go diff --git a/pkg/deployment/trigger/devtronApps/PostStageTriggerService_ent.go b/pkg/deployment/trigger/devtronApps/postStageHandlerCode_ent.go similarity index 100% rename from pkg/deployment/trigger/devtronApps/PostStageTriggerService_ent.go rename to pkg/deployment/trigger/devtronApps/postStageHandlerCode_ent.go diff --git a/pkg/deployment/trigger/devtronApps/PrePostCommonService.go b/pkg/deployment/trigger/devtronApps/prePostWfAndLogsCode.go similarity index 100% rename from pkg/deployment/trigger/devtronApps/PrePostCommonService.go rename to pkg/deployment/trigger/devtronApps/prePostWfAndLogsCode.go diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService_ent.go b/pkg/deployment/trigger/devtronApps/preStageHandlerCode_ent.go similarity index 100% rename from pkg/deployment/trigger/devtronApps/PreStageTriggerService_ent.go rename to pkg/deployment/trigger/devtronApps/preStageHandlerCode_ent.go diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go b/pkg/deployment/trigger/devtronApps/preStageHandlerService.go similarity index 100% rename from pkg/deployment/trigger/devtronApps/PreStageTriggerService.go rename to pkg/deployment/trigger/devtronApps/preStageHandlerService.go From 43fb95b70f9de2c6738a0aaa12c93fc6a8ff1b40 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 8 Apr 2025 12:34:38 +0530 Subject: [PATCH 35/37] deployStage unstacked file --- .../devtronApps/deployStageHandlerCode.go | 1352 +++++++++++++++++ 1 file changed, 1352 insertions(+) create mode 100644 pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go diff --git a/pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go b/pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go new file mode 100644 index 0000000000..c0d8a204a1 --- /dev/null +++ b/pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go @@ -0,0 +1,1352 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package devtronApps + +import ( + "context" + "errors" + "fmt" + bean3 "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/api/bean/gitOps" + bean6 "github.com/devtron-labs/devtron/api/helm-app/bean" + "github.com/devtron-labs/devtron/api/helm-app/gRPC" + "github.com/devtron-labs/devtron/client/argocdServer" + bean7 "github.com/devtron-labs/devtron/client/argocdServer/bean" + "github.com/devtron-labs/devtron/internal/middleware" + "github.com/devtron-labs/devtron/internal/sql/models" + repository3 "github.com/devtron-labs/devtron/internal/sql/repository" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" + "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/app" + bean4 "github.com/devtron-labs/devtron/pkg/app/bean" + statusBean "github.com/devtron-labs/devtron/pkg/app/status/bean" + bean2 "github.com/devtron-labs/devtron/pkg/bean" + "github.com/devtron-labs/devtron/pkg/deployment/common" + bean9 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" + bean10 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/bean" + bean5 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/adapter" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/helper" + clientErrors "github.com/devtron-labs/devtron/pkg/errors" + k8s2 "github.com/devtron-labs/devtron/pkg/k8s" + bean8 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/repository" + repository6 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/repository" + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/go-pg/pg" + "go.opentelemetry.io/otel" + "golang.org/x/exp/slices" + "google.golang.org/grpc/codes" + status2 "google.golang.org/grpc/status" + "helm.sh/helm/v3/pkg/chart" + "net/http" + "path" + "regexp" + "strconv" + "strings" + "time" +) + +func (impl *HandlerServiceImpl) TriggerStageForBulk(triggerRequest bean.TriggerRequest) error { + + preStage, err := impl.pipelineStageService.GetCdStageByCdPipelineIdAndStageType(triggerRequest.Pipeline.Id, repository.PIPELINE_STAGE_TYPE_PRE_CD, false) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching CD pipeline stage", "cdPipelineId", triggerRequest.Pipeline.Id, "stage ", repository.PIPELINE_STAGE_TYPE_PRE_CD, "err", err) + return err + } + + // handle corrupt data (https://github.com/devtron-labs/devtron/issues/3826) + err, deleted := impl.deleteCorruptedPipelineStage(preStage, triggerRequest.TriggeredBy) + if err != nil { + impl.logger.Errorw("error in deleteCorruptedPipelineStage ", "cdPipelineId", triggerRequest.Pipeline.Id, "err", err, "preStage", preStage, "triggeredBy", triggerRequest.TriggeredBy) + return err + } + + triggerRequest.TriggerContext.Context = context.Background() + if len(triggerRequest.Pipeline.PreStageConfig) > 0 || (preStage != nil && !deleted) { + // pre stage exists + impl.logger.Debugw("trigger pre stage for pipeline", "artifactId", triggerRequest.Artifact.Id, "pipelineId", triggerRequest.Pipeline.Id) + triggerRequest.RefCdWorkflowRunnerId = 0 + err = impl.preStageHandlingForTriggerStageInBulk(&triggerRequest) + if err != nil { + return err + } + _, err = impl.TriggerPreStage(triggerRequest) // TODO handle error here + return err + } else { + // trigger deployment + impl.logger.Debugw("trigger cd for pipeline", "artifactId", triggerRequest.Artifact.Id, "pipelineId", triggerRequest.Pipeline.Id) + err = impl.TriggerAutomaticDeployment(triggerRequest) + return err + } +} + +func (impl *HandlerServiceImpl) getCdPipelineForManualCdTrigger(ctx context.Context, pipelineId int) (*pipelineConfig.Pipeline, error) { + _, span := otel.Tracer("HandlerService").Start(ctx, "getCdPipelineForManualCdTrigger") + defer span.End() + cdPipeline, err := impl.pipelineRepository.FindById(pipelineId) + if err != nil { + impl.logger.Errorw("manual trigger request with invalid pipelineId, ManualCdTrigger", "pipelineId", pipelineId, "err", err) + return nil, err + } + //checking if namespace exist or not + clusterIdToNsMap := map[int]string{ + cdPipeline.Environment.ClusterId: cdPipeline.Environment.Namespace, + } + + err = impl.helmAppService.CheckIfNsExistsForClusterIds(clusterIdToNsMap) + if err != nil { + impl.logger.Errorw("manual trigger request with invalid namespace, ManualCdTrigger", "pipelineId", pipelineId, "err", err) + return nil, err + } + return cdPipeline, nil +} + +func (impl *HandlerServiceImpl) validateDeploymentTriggerRequest(ctx context.Context, validateDeploymentTriggerObj *bean.ValidateDeploymentTriggerObj) error { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.validateDeploymentTriggerRequest") + defer span.End() + // custom GitOps repo url validation --> Start + err := impl.handleCustomGitOpsRepoValidation(validateDeploymentTriggerObj.Runner, validateDeploymentTriggerObj.CdPipeline, validateDeploymentTriggerObj.DeploymentConfig, validateDeploymentTriggerObj.TriggeredBy) + if err != nil { + impl.logger.Errorw("custom GitOps repository validation error, TriggerStage", "err", err) + return err + } + // custom GitOps repo url validation --> Ends + var isVulnerable bool + // if request is for rollback then bypass vulnerability validation + if !validateDeploymentTriggerObj.IsDeploymentTypeRollback() { + // checking vulnerability for deploying image + vulnerabilityCheckRequest := adapter.GetVulnerabilityCheckRequest(validateDeploymentTriggerObj.CdPipeline, validateDeploymentTriggerObj.ImageDigest) + isVulnerable, err = impl.imageScanService.GetArtifactVulnerabilityStatus(newCtx, vulnerabilityCheckRequest) + if err != nil { + impl.logger.Errorw("error in getting Artifact vulnerability status, ManualCdTrigger", "err", err) + return err + } + } + + if isVulnerable == true { + // if image vulnerable, update timeline status and return + if err = impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(validateDeploymentTriggerObj.Runner, errors.New(cdWorkflow.FOUND_VULNERABILITY), validateDeploymentTriggerObj.TriggeredBy); err != nil { + impl.logger.Errorw("error while updating current runner status to failed, TriggerDeployment", "wfrId", validateDeploymentTriggerObj.Runner.Id, "err", err) + } + return fmt.Errorf("found vulnerability for image digest %s", validateDeploymentTriggerObj.ImageDigest) + } + return nil +} + +// TODO: write a wrapper to handle auto and manual trigger +func (impl *HandlerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerContext, overrideRequest *bean3.ValuesOverrideRequest) (int, string, *bean4.ManifestPushTemplate, error) { + + triggerContext.TriggerType = bean.Manual + // setting triggeredAt variable to have consistent data for various audit log places in db for deployment time + triggeredAt := time.Now() + + releaseId := 0 + ctx := triggerContext.Context + cdPipeline, err := impl.getCdPipelineForManualCdTrigger(ctx, overrideRequest.PipelineId) + if err != nil { + if overrideRequest.WfrId != 0 { + err2 := impl.cdWorkflowCommonService.MarkDeploymentFailedForRunnerId(overrideRequest.WfrId, err, overrideRequest.UserId) + if err2 != nil { + impl.logger.Errorw("error while updating current runner status to failed, ManualCdTrigger", "cdWfr", overrideRequest.WfrId, "err2", err2) + } + } + return 0, "", nil, err + } + envDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(cdPipeline.AppId, cdPipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", cdPipeline.AppId, "envId", cdPipeline.EnvironmentId, "err", err) + return 0, "", nil, err + } + + adapter.SetPipelineFieldsInOverrideRequest(overrideRequest, cdPipeline, envDeploymentConfig) + ciArtifactId := overrideRequest.CiArtifactId + + _, span := otel.Tracer("orchestrator").Start(ctx, "ciArtifactRepository.Get") + artifact, err := impl.ciArtifactRepository.Get(ciArtifactId) + span.End() + if err != nil { + impl.logger.Errorw("error in getting CiArtifact", "CiArtifactId", overrideRequest.CiArtifactId, "err", err) + return 0, "", nil, err + } + + if artifact.IsMigrationRequired() { + // Migration of deprecated DataSource Type + migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(artifact.Id) + if migrationErr != nil { + impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id) + } + } + + _, imageTag, err := artifact.ExtractImageRepoAndTag() + if err != nil { + impl.logger.Errorw("error in getting image tag and repo", "err", err) + } + helmPackageName := fmt.Sprintf("%s-%s-%s", cdPipeline.App.AppName, cdPipeline.Environment.Name, imageTag) + var manifestPushTemplate *bean4.ManifestPushTemplate + + switch overrideRequest.CdWorkflowType { + case bean3.CD_WORKFLOW_TYPE_PRE: + var cdWf *pipelineConfig.CdWorkflow + if overrideRequest.CdWorkflowId == 0 { + cdWf = &pipelineConfig.CdWorkflow{ + CiArtifactId: artifact.Id, + PipelineId: cdPipeline.Id, + AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: 1, UpdatedOn: triggeredAt, UpdatedBy: 1}, + } + err := impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) + if err != nil { + return 0, "", nil, err + } + } else { + cdWf, err = impl.cdWorkflowRepository.FindById(overrideRequest.CdWorkflowId) + if err != nil { + impl.logger.Errorw("error in TriggerPreStage, ManualCdTrigger", "err", err) + return 0, "", nil, err + } + } + overrideRequest.CdWorkflowId = cdWf.Id + + _, span = otel.Tracer("orchestrator").Start(ctx, "TriggerPreStage") + triggerRequest := bean.TriggerRequest{ + CdWf: cdWf, + Artifact: artifact, + Pipeline: cdPipeline, + TriggeredBy: overrideRequest.UserId, + ApplyAuth: false, + TriggerContext: triggerContext, + RefCdWorkflowRunnerId: 0, + CdWorkflowRunnerId: overrideRequest.WfrId, + } + manifestPushTemplate, err = impl.TriggerPreStage(triggerRequest) + span.End() + if err != nil { + impl.logger.Errorw("error in TriggerPreStage, ManualCdTrigger", "err", err) + return 0, "", nil, err + } + case bean3.CD_WORKFLOW_TYPE_DEPLOY: + if overrideRequest.DeploymentType == models.DEPLOYMENTTYPE_UNKNOWN { + overrideRequest.DeploymentType = models.DEPLOYMENTTYPE_DEPLOY + } + + cdWf, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, overrideRequest.CdWorkflowId, bean3.CD_WORKFLOW_TYPE_PRE) + if err != nil && !util.IsErrNoRows(err) { + impl.logger.Errorw("error in getting cdWorkflow, ManualCdTrigger", "CdWorkflowId", overrideRequest.CdWorkflowId, "err", err) + return 0, "", nil, err + } + + cdWorkflowId := cdWf.CdWorkflowId + if cdWf.CdWorkflowId == 0 { + cdWf := &pipelineConfig.CdWorkflow{ + CiArtifactId: overrideRequest.CiArtifactId, + PipelineId: overrideRequest.PipelineId, + AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, + } + err := impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) + if err != nil { + impl.logger.Errorw("error in creating cdWorkflow, ManualCdTrigger", "PipelineId", overrideRequest.PipelineId, "err", err) + return 0, "", nil, err + } + cdWorkflowId = cdWf.Id + } + + runner := &pipelineConfig.CdWorkflowRunner{ + Name: cdPipeline.Name, + WorkflowType: bean3.CD_WORKFLOW_TYPE_DEPLOY, + ExecutorType: cdWorkflow.WORKFLOW_EXECUTOR_TYPE_AWF, + Status: cdWorkflow.WorkflowInitiated, //deployment Initiated for manual trigger + TriggeredBy: overrideRequest.UserId, + StartedOn: triggeredAt, + Namespace: impl.config.GetDefaultNamespace(), + CdWorkflowId: cdWorkflowId, + AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, + ReferenceId: triggerContext.ReferenceId, + } + savedWfr, err := impl.cdWorkflowRunnerService.SaveCDWorkflowRunnerWithStage(runner) + if err != nil { + impl.logger.Errorw("err in creating cdWorkflowRunner, ManualCdTrigger", "cdWorkflowId", cdWorkflowId, "err", err) + return 0, "", nil, err + } + runner.CdWorkflow = &pipelineConfig.CdWorkflow{ + Pipeline: cdPipeline, + } + overrideRequest.WfrId = savedWfr.Id + overrideRequest.CdWorkflowId = cdWorkflowId + // creating cd pipeline status timeline for deployment initialisation + timeline := impl.pipelineStatusTimelineService.NewDevtronAppPipelineStatusTimelineDbObject(runner.Id, timelineStatus.TIMELINE_STATUS_DEPLOYMENT_INITIATED, timelineStatus.TIMELINE_DESCRIPTION_DEPLOYMENT_INITIATED, overrideRequest.UserId) + _, span = otel.Tracer("orchestrator").Start(ctx, "cdPipelineStatusTimelineRepo.SaveTimelineForACDHelmApps") + _, err = impl.pipelineStatusTimelineService.SaveTimelineIfNotAlreadyPresent(timeline, nil) + + span.End() + if err != nil { + impl.logger.Errorw("error in creating timeline status for deployment initiation, ManualCdTrigger", "err", err, "timeline", timeline) + } + if isNotHibernateRequest(overrideRequest.DeploymentType) { + validateReqObj := adapter.NewValidateDeploymentTriggerObj(runner, cdPipeline, artifact.ImageDigest, envDeploymentConfig, overrideRequest.UserId, overrideRequest.IsRollbackDeployment) + validationErr := impl.validateDeploymentTriggerRequest(ctx, validateReqObj) + if validationErr != nil { + impl.logger.Errorw("validation error deployment request", "cdWfr", runner.Id, "err", validationErr) + return 0, "", nil, validationErr + } + } + // Deploy the release + var releaseErr error + releaseId, manifestPushTemplate, releaseErr = impl.handleCDTriggerRelease(ctx, overrideRequest, envDeploymentConfig, triggeredAt, overrideRequest.UserId) + // if releaseErr found, then the mark current deployment Failed and return + if releaseErr != nil { + err := impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, releaseErr, overrideRequest.UserId) + if err != nil { + impl.logger.Errorw("error while updating current runner status to failed", "cdWfr", runner.Id, "err", err) + } + return 0, "", nil, releaseErr + } + + case bean3.CD_WORKFLOW_TYPE_POST: + cdWfRunner, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, overrideRequest.CdWorkflowId, bean3.CD_WORKFLOW_TYPE_DEPLOY) + if err != nil && !util.IsErrNoRows(err) { + impl.logger.Errorw("err in getting cdWorkflowRunner, ManualCdTrigger", "cdWorkflowId", overrideRequest.CdWorkflowId, "err", err) + return 0, "", nil, err + } + + var cdWf *pipelineConfig.CdWorkflow + if cdWfRunner.CdWorkflowId == 0 { + cdWf = &pipelineConfig.CdWorkflow{ + CiArtifactId: ciArtifactId, + PipelineId: overrideRequest.PipelineId, + AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, + } + err := impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) + if err != nil { + impl.logger.Errorw("error in creating cdWorkflow, ManualCdTrigger", "CdWorkflowId", overrideRequest.CdWorkflowId, "err", err) + return 0, "", nil, err + } + overrideRequest.CdWorkflowId = cdWf.Id + } else { + _, span = otel.Tracer("orchestrator").Start(ctx, "cdWorkflowRepository.FindById") + cdWf, err = impl.cdWorkflowRepository.FindById(overrideRequest.CdWorkflowId) + span.End() + if err != nil && !util.IsErrNoRows(err) { + impl.logger.Errorw("error in getting cdWorkflow, ManualCdTrigger", "CdWorkflowId", overrideRequest.CdWorkflowId, "err", err) + return 0, "", nil, err + } + } + _, span = otel.Tracer("orchestrator").Start(ctx, "TriggerPostStage") + triggerRequest := bean.TriggerRequest{ + CdWf: cdWf, + Pipeline: cdPipeline, + TriggeredBy: overrideRequest.UserId, + RefCdWorkflowRunnerId: 0, + TriggerContext: triggerContext, + CdWorkflowRunnerId: overrideRequest.WfrId, + } + manifestPushTemplate, err = impl.TriggerPostStage(triggerRequest) + span.End() + if err != nil { + impl.logger.Errorw("error in TriggerPostStage, ManualCdTrigger", "CdWorkflowId", cdWf.Id, "err", err) + return 0, "", nil, err + } + default: + impl.logger.Errorw("invalid CdWorkflowType, ManualCdTrigger", "CdWorkflowType", overrideRequest.CdWorkflowType, "err", err) + return 0, "", nil, fmt.Errorf("invalid CdWorkflowType %s for the trigger request", string(overrideRequest.CdWorkflowType)) + } + return releaseId, helmPackageName, manifestPushTemplate, err +} + +func isNotHibernateRequest(deploymentType models.DeploymentType) bool { + return deploymentType != models.DEPLOYMENTTYPE_STOP && deploymentType != models.DEPLOYMENTTYPE_START +} + +// TODO: write a wrapper to handle auto and manual trigger +func (impl *HandlerServiceImpl) TriggerAutomaticDeployment(request bean.TriggerRequest) error { + // in case of manual trigger auth is already applied and for auto triggers there is no need for auth check here + triggeredBy := request.TriggeredBy + pipeline := request.Pipeline + artifact := request.Artifact + + //setting triggeredAt variable to have consistent data for various audit log places in db for deployment time + triggeredAt := time.Now() + cdWf := request.CdWf + ctx := context.Background() + + if cdWf == nil || (cdWf != nil && cdWf.CiArtifactId != artifact.Id) { + // cdWf != nil && cdWf.CiArtifactId != artifact.Id for auto trigger case when deployment is triggered with image generated by plugin + cdWf = &pipelineConfig.CdWorkflow{ + CiArtifactId: artifact.Id, + PipelineId: pipeline.Id, + AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: 1, UpdatedOn: triggeredAt, UpdatedBy: 1}, + } + err := impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) + if err != nil { + return err + } + } + + runner := &pipelineConfig.CdWorkflowRunner{ + Name: pipeline.Name, + WorkflowType: bean3.CD_WORKFLOW_TYPE_DEPLOY, + ExecutorType: cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM, + Status: cdWorkflow.WorkflowInitiated, // deployment Initiated for auto trigger + TriggeredBy: triggeredBy, + StartedOn: triggeredAt, + Namespace: impl.config.GetDefaultNamespace(), + CdWorkflowId: cdWf.Id, + AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: triggeredBy, UpdatedOn: triggeredAt, UpdatedBy: triggeredBy}, + ReferenceId: request.TriggerContext.ReferenceId, + } + savedWfr, err := impl.cdWorkflowRunnerService.SaveCDWorkflowRunnerWithStage(runner) + if err != nil { + return err + } + runner.CdWorkflow = &pipelineConfig.CdWorkflow{ + Pipeline: pipeline, + } + // creating cd pipeline status timeline for deployment initialisation + timeline := &pipelineConfig.PipelineStatusTimeline{ + CdWorkflowRunnerId: runner.Id, + Status: timelineStatus.TIMELINE_STATUS_DEPLOYMENT_INITIATED, + StatusDetail: "Deployment initiated successfully.", + StatusTime: time.Now(), + } + timeline.CreateAuditLog(1) + err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil) + if err != nil { + impl.logger.Errorw("error in creating timeline status for deployment initiation", "err", err, "timeline", timeline) + } + envDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(pipeline.AppId, pipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) + return err + } + validationErr := impl.validateDeploymentTriggerRequest(ctx, adapter.NewValidateDeploymentTriggerObj(runner, pipeline, artifact.ImageDigest, envDeploymentConfig, triggeredBy, false)) + if validationErr != nil { + impl.logger.Errorw("validation error deployment request", "cdWfr", runner.Id, "err", validationErr) + return validationErr + } + releaseErr := impl.TriggerCD(ctx, artifact, cdWf.Id, savedWfr.Id, pipeline, envDeploymentConfig, triggeredAt, triggeredBy) + // if releaseErr found, then the mark current deployment Failed and return + if releaseErr != nil { + err := impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, releaseErr, triggeredBy) + if err != nil { + impl.logger.Errorw("error while updating current runner status to failed, updatePreviousDeploymentStatus", "cdWfr", runner.Id, "err", err) + } + return releaseErr + } + return nil +} + +func (impl *HandlerServiceImpl) TriggerCD(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) error { + impl.logger.Debugw("automatic pipeline trigger attempt async", "artifactId", artifact.Id) + err := impl.triggerReleaseAsync(ctx, artifact, cdWorkflowId, wfrId, pipeline, envDeploymentConfig, triggeredAt, triggeredBy) + if err != nil { + impl.logger.Errorw("error in cd trigger", "err", err) + return err + } + return err +} + +func (impl *HandlerServiceImpl) triggerReleaseAsync(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) error { + err := impl.validateAndTrigger(ctx, pipeline, envDeploymentConfig, artifact, cdWorkflowId, wfrId, triggeredAt, triggeredBy) + if err != nil { + impl.logger.Errorw("error in trigger for pipeline", "pipelineId", strconv.Itoa(pipeline.Id)) + } + impl.logger.Debugw("trigger attempted for all pipeline ", "artifactId", artifact.Id) + return err +} + +func (impl *HandlerServiceImpl) validateAndTrigger(ctx context.Context, p *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time, triggeredBy int32) error { + //TODO: verify this logic + object := impl.enforcerUtil.GetAppRBACNameByAppId(p.AppId) + envApp := strings.Split(object, "/") + if len(envApp) != 2 { + impl.logger.Error("invalid req, app and env not found from rbac") + return errors.New("invalid req, app and env not found from rbac") + } + err := impl.releasePipeline(ctx, p, envDeploymentConfig, artifact, cdWorkflowId, wfrId, triggeredAt, triggeredBy) + return err +} + +func (impl *HandlerServiceImpl) releasePipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time, triggeredBy int32) error { + startTime := time.Now() + defer func() { + impl.logger.Debugw("auto trigger release process completed", "timeTaken", time.Since(startTime), "cdPipelineId", pipeline.Id, "artifactId", artifact.Id, "wfrId", wfrId) + }() + impl.logger.Debugw("auto triggering release for", "cdPipelineId", pipeline.Id, "artifactId", artifact.Id, "wfrId", wfrId) + pipeline, err := impl.pipelineRepository.FindById(pipeline.Id) + if err != nil { + impl.logger.Errorw("error in fetching pipeline by pipelineId", "err", err) + return err + } + + request := &bean3.ValuesOverrideRequest{ + PipelineId: pipeline.Id, + UserId: artifact.CreatedBy, + CiArtifactId: artifact.Id, + AppId: pipeline.AppId, + CdWorkflowId: cdWorkflowId, + ForceTrigger: true, + DeploymentWithConfig: bean3.DEPLOYMENT_CONFIG_TYPE_LAST_SAVED, + WfrId: wfrId, + } + + adapter.SetPipelineFieldsInOverrideRequest(request, pipeline, envDeploymentConfig) + + id, _, err := impl.handleCDTriggerRelease(ctx, request, envDeploymentConfig, triggeredAt, triggeredBy) + if err != nil { + impl.logger.Errorw("error in auto cd pipeline trigger", "pipelineId", pipeline.Id, "artifactId", artifact.Id, "err", err) + } else { + impl.logger.Infow("pipeline successfully triggered", "cdPipelineId", pipeline.Id, "artifactId", artifact.Id, "releaseId", id) + } + return err +} + +func (impl *HandlerServiceImpl) triggerAsyncRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, + envDeploymentConfig *bean9.DeploymentConfig, userDeploymentRequestId int, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.triggerAsyncRelease") + defer span.End() + // build merged values and save PCO history for the release + valuesOverrideResponse, err := impl.manifestCreationService.GetValuesOverrideForTrigger(newCtx, overrideRequest, envDeploymentConfig, triggeredAt) + // auditDeploymentTriggerHistory is performed irrespective of GetValuesOverrideForTrigger error - for auditing purposes + historyErr := impl.auditDeploymentTriggerHistory(overrideRequest.WfrId, valuesOverrideResponse, newCtx, triggeredAt, deployedBy) + if historyErr != nil { + impl.logger.Errorw("error in auditing deployment trigger history", "cdWfrId", overrideRequest.WfrId, "err", err) + return releaseNo, manifestPushTemplate, err + } + // handling GetValuesOverrideForTrigger error + if err != nil { + impl.logger.Errorw("error in fetching values for trigger", "err", err) + return releaseNo, manifestPushTemplate, err + } + // asynchronous mode of Helm/ArgoCd installation starts + return impl.workflowEventPublishService.TriggerAsyncRelease(userDeploymentRequestId, overrideRequest, valuesOverrideResponse, newCtx, deployedBy) +} + +func (impl *HandlerServiceImpl) handleCDTriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.handleCDTriggerRelease") + defer span.End() + // Handling for auto trigger + if overrideRequest.UserId == 0 { + overrideRequest.UserId = deployedBy + } + tx, err := impl.transactionUtilImpl.StartTx() + if err != nil { + impl.logger.Errorw("error in starting transaction to update userDeploymentRequest", "error", err) + return releaseNo, manifestPushTemplate, err + } + defer impl.transactionUtilImpl.RollbackTx(tx) + newDeploymentRequest := adapter.NewUserDeploymentRequest(overrideRequest, triggeredAt, overrideRequest.UserId) + // creating new user deployment request + userDeploymentRequestId, err := impl.userDeploymentRequestService.SaveNewDeployment(newCtx, tx, newDeploymentRequest) + if err != nil { + impl.logger.Errorw("error in saving new userDeploymentRequest", "overrideRequest", overrideRequest, "err", err) + return releaseNo, manifestPushTemplate, err + } + timeline := impl.pipelineStatusTimelineService.NewDevtronAppPipelineStatusTimelineDbObject(overrideRequest.WfrId, timelineStatus.TIMELINE_STATUS_DEPLOYMENT_REQUEST_VALIDATED, timelineStatus.TIMELINE_DESCRIPTION_DEPLOYMENT_REQUEST_VALIDATED, deployedBy) + // creating cd pipeline status timeline for deployment trigger request validated + _, err = impl.pipelineStatusTimelineService.SaveTimelineIfNotAlreadyPresent(timeline, tx) + err = impl.transactionUtilImpl.CommitTx(tx) + if err != nil { + impl.logger.Errorw("error in committing transaction to update userDeploymentRequest", "error", err) + return userDeploymentRequestId, manifestPushTemplate, err + } + isAsyncMode, err := impl.isDevtronAsyncInstallModeEnabled(overrideRequest) + if err != nil { + impl.logger.Errorw("error in checking async mode for devtron app", "err", err, "deploymentType", overrideRequest.DeploymentType, + "forceSyncDeployment", overrideRequest.ForceSyncDeployment, "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId) + return userDeploymentRequestId, manifestPushTemplate, err + } + if envDeploymentConfig.IsEmpty() { + deploymentConfig, dbErr := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(overrideRequest.AppId, overrideRequest.EnvId) + if dbErr != nil { + impl.logger.Errorw("error in getting deployment config by appId and envId", "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId, "err", dbErr) + return releaseNo, manifestPushTemplate, dbErr + } + envDeploymentConfig = deploymentConfig + } + if isAsyncMode { + return impl.triggerAsyncRelease(newCtx, overrideRequest, envDeploymentConfig, userDeploymentRequestId, triggeredAt, deployedBy) + } + // synchronous mode of installation starts + return impl.TriggerRelease(newCtx, overrideRequest, envDeploymentConfig, triggeredAt, deployedBy) +} + +func (impl *HandlerServiceImpl) auditDeploymentTriggerHistory(cdWfrId int, valuesOverrideResponse *app.ValuesOverrideResponse, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (err error) { + if valuesOverrideResponse.Pipeline == nil || valuesOverrideResponse.EnvOverride == nil { + impl.logger.Warnw("unable to save histories for deployment trigger, invalid valuesOverrideResponse received", "cdWfrId", cdWfrId) + return nil + } + err1 := impl.deployedConfigurationHistoryService.CreateHistoriesForDeploymentTrigger(ctx, valuesOverrideResponse.Pipeline, valuesOverrideResponse.PipelineStrategy, valuesOverrideResponse.EnvOverride, triggeredAt, triggeredBy) + if err1 != nil { + impl.logger.Errorw("error in saving histories for deployment trigger", "err", err1, "pipelineId", valuesOverrideResponse.Pipeline.Id, "cdWfrId", cdWfrId) + return nil + } + return nil +} + +// TriggerRelease will trigger Install/Upgrade request for Devtron App releases synchronously +func (impl *HandlerServiceImpl) TriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, + envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.TriggerRelease") + defer span.End() + triggerEvent, skipRequest, err := impl.buildTriggerEventForOverrideRequest(overrideRequest, triggeredAt) + if err != nil { + return releaseNo, manifestPushTemplate, err + } + impl.logger.Debugw("processing TriggerRelease", "wfrId", overrideRequest.WfrId, "triggerEvent", triggerEvent) + // request has already been served, skipping + if skipRequest { + impl.logger.Infow("request already served, skipping", "wfrId", overrideRequest.WfrId) + return releaseNo, manifestPushTemplate, nil + } + // build merged values and save PCO history for the release + valuesOverrideResponse, builtChartPath, err := impl.manifestCreationService.BuildManifestForTrigger(newCtx, overrideRequest, envDeploymentConfig, triggeredAt) + + // auditDeploymentTriggerHistory is performed irrespective of BuildManifestForTrigger error - for auditing purposes + historyErr := impl.auditDeploymentTriggerHistory(overrideRequest.WfrId, valuesOverrideResponse, newCtx, triggeredAt, triggeredBy) + if historyErr != nil { + impl.logger.Errorw("error in auditing deployment trigger history", "cdWfrId", overrideRequest.WfrId, "err", err) + return releaseNo, manifestPushTemplate, err + } + if err != nil { + impl.logger.Errorw("error in building merged manifest for trigger", "err", err) + impl.manifestGenerationFailedTimelineHandling(triggerEvent, overrideRequest, err) + return releaseNo, manifestPushTemplate, err + } + helmManifest, err := impl.getHelmManifestForTriggerRelease(ctx, triggerEvent, overrideRequest, valuesOverrideResponse, builtChartPath) + if err != nil { + impl.logger.Errorw("error, getHelmManifestForTriggerRelease", "err", err) + return releaseNo, manifestPushTemplate, err + } + impl.logger.Debugw("triggering pipeline for release", "wfrId", overrideRequest.WfrId, "builtChartPath", builtChartPath) + releaseNo, err = impl.triggerPipeline(overrideRequest, valuesOverrideResponse, builtChartPath, triggerEvent, newCtx) + if err != nil { + return 0, manifestPushTemplate, err + } + + err = impl.triggerReleaseSuccessHandling(triggerEvent, overrideRequest, valuesOverrideResponse, helmManifest) + if err != nil { + impl.logger.Errorw("error, triggerReleaseSuccessHandling", "triggerEvent", triggerEvent, "err", err) + return releaseNo, manifestPushTemplate, err + } + // creating cd pipeline status timeline for deployment triggered - for successfully triggered requests + timeline := impl.pipelineStatusTimelineService.NewDevtronAppPipelineStatusTimelineDbObject(overrideRequest.WfrId, timelineStatus.TIMELINE_STATUS_DEPLOYMENT_TRIGGERED, timelineStatus.TIMELINE_DESCRIPTION_DEPLOYMENT_COMPLETED, overrideRequest.UserId) + _, dbErr := impl.pipelineStatusTimelineService.SaveTimelineIfNotAlreadyPresent(timeline, nil) + if dbErr != nil { + impl.logger.Errorw("error in creating timeline status for deployment completed", "err", dbErr, "timeline", timeline) + } + impl.logger.Debugw("triggered pipeline for release successfully", "wfrId", overrideRequest.WfrId, "builtChartPath", builtChartPath) + return releaseNo, valuesOverrideResponse.ManifestPushTemplate, nil +} + +func (impl *HandlerServiceImpl) performGitOps(ctx context.Context, + overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, + builtChartPath string, triggerEvent bean.TriggerEvent) error { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.performGitOps") + defer span.End() + // update workflow runner status, used in app workflow view + err := impl.cdWorkflowCommonService.UpdateNonTerminalStatusInRunner(newCtx, overrideRequest.WfrId, overrideRequest.UserId, cdWorkflow.WorkflowInProgress) + if err != nil { + impl.logger.Errorw("error in updating the workflow runner status", "err", err) + return err + } + manifestPushTemplate, err := impl.buildManifestPushTemplate(overrideRequest, valuesOverrideResponse, builtChartPath) + if err != nil { + impl.logger.Errorw("error in building manifest push template", "err", err) + return err + } + manifestPushService := impl.getManifestPushService(triggerEvent.ManifestStorageType) + manifestPushResponse := manifestPushService.PushChart(newCtx, manifestPushTemplate) + if manifestPushResponse.Error != nil { + impl.logger.Errorw("error in pushing manifest to git/helm", "err", manifestPushResponse.Error, "git_repo_url", manifestPushTemplate.RepoUrl) + return manifestPushResponse.Error + } + if manifestPushResponse.IsNewGitRepoConfigured() { + // Update GitOps repo url after repo new repo created + valuesOverrideResponse.DeploymentConfig.SetRepoURL(manifestPushResponse.NewGitRepoUrl) + } + valuesOverrideResponse.ManifestPushTemplate = manifestPushTemplate + return nil +} + +func (impl *HandlerServiceImpl) buildTriggerEventForOverrideRequest(overrideRequest *bean3.ValuesOverrideRequest, triggeredAt time.Time) (triggerEvent bean.TriggerEvent, skipRequest bool, err error) { + triggerEvent = helper.NewTriggerEvent(overrideRequest.DeploymentAppType, triggeredAt, overrideRequest.UserId) + request := statusBean.NewTimelineGetRequest(). + WithCdWfrId(overrideRequest.WfrId). + ExcludingStatuses(timelineStatus.TIMELINE_STATUS_UNABLE_TO_FETCH_STATUS, + timelineStatus.TIMELINE_STATUS_KUBECTL_APPLY_STARTED, + timelineStatus.TIMELINE_STATUS_KUBECTL_APPLY_SYNCED) + timelineStatuses, err := impl.pipelineStatusTimelineService.GetTimelineStatusesFor(request) + if err != nil { + impl.logger.Errorw("error in getting last timeline status by cdWfrId", "cdWfrId", overrideRequest.WfrId, "err", err) + return triggerEvent, skipRequest, err + } else if !slices.Contains(timelineStatuses, timelineStatus.TIMELINE_STATUS_DEPLOYMENT_REQUEST_VALIDATED) { + impl.logger.Errorw("pre-condition missing: timeline for deployment request validation", "cdWfrId", overrideRequest.WfrId, "timelineStatuses", timelineStatuses) + return triggerEvent, skipRequest, fmt.Errorf("pre-condition missing: timeline for deployment request validation") + } else if timelineStatus.ContainsTerminalTimelineStatus(timelineStatuses) { + impl.logger.Info("deployment is already terminated", "cdWfrId", overrideRequest.WfrId, "timelineStatuses", timelineStatuses) + skipRequest = true + return triggerEvent, skipRequest, nil + } else if slices.Contains(timelineStatuses, timelineStatus.TIMELINE_STATUS_DEPLOYMENT_TRIGGERED) { + impl.logger.Info("deployment has been performed. skipping", "cdWfrId", overrideRequest.WfrId, "timelineStatuses", timelineStatuses) + skipRequest = true + return triggerEvent, skipRequest, nil + } + if slices.Contains(timelineStatuses, timelineStatus.TIMELINE_STATUS_GIT_COMMIT) || + slices.Contains(timelineStatuses, timelineStatus.TIMELINE_STATUS_ARGOCD_SYNC_INITIATED) { + // git commit has already been performed + triggerEvent.PerformChartPush = false + } + if slices.Contains(timelineStatuses, timelineStatus.TIMELINE_STATUS_ARGOCD_SYNC_COMPLETED) { + // ArgoCd sync has already been performed + triggerEvent.DeployArgoCdApp = false + } + return triggerEvent, skipRequest, nil +} + +func (impl *HandlerServiceImpl) triggerPipeline(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, triggerEvent bean.TriggerEvent, ctx context.Context) (releaseNo int, err error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.triggerPipeline") + defer span.End() + if triggerEvent.PerformChartPush { + impl.logger.Debugw("performing chart push operation in triggerPipeline", "cdWfrId", overrideRequest.WfrId) + err = impl.performGitOps(newCtx, overrideRequest, valuesOverrideResponse, builtChartPath, triggerEvent) + if err != nil { + impl.logger.Errorw("error in performing GitOps", "cdWfrId", overrideRequest.WfrId, "err", err) + return releaseNo, err + } + impl.logger.Debugw("chart push operation completed successfully", "cdWfrId", overrideRequest.WfrId) + } + + if triggerEvent.PerformDeploymentOnCluster { + err = impl.deployApp(newCtx, overrideRequest, valuesOverrideResponse, triggerEvent) + if err != nil { + impl.logger.Errorw("error in deploying app", "err", err) + return releaseNo, err + } + } + + go impl.writeCDTriggerEvent(overrideRequest, valuesOverrideResponse.Artifact, valuesOverrideResponse.PipelineOverride.PipelineReleaseCounter, valuesOverrideResponse.PipelineOverride.Id, overrideRequest.WfrId) + + _ = impl.markImageScanDeployed(newCtx, overrideRequest.AppId, overrideRequest.EnvId, overrideRequest.ClusterId, + valuesOverrideResponse.Artifact.ImageDigest, valuesOverrideResponse.Artifact.ScanEnabled, valuesOverrideResponse.Artifact.Image) + + middleware.CdTriggerCounter.WithLabelValues(overrideRequest.AppName, overrideRequest.EnvName).Inc() + + // Update previous deployment runner status (in transaction): Failed + dbErr := impl.cdWorkflowCommonService.SupersedePreviousDeployments(newCtx, overrideRequest.WfrId, overrideRequest.PipelineId, triggerEvent.TriggeredAt, overrideRequest.UserId) + if dbErr != nil { + impl.logger.Errorw("error while update previous cd workflow runners", "err", dbErr, "currentRunnerId", overrideRequest.WfrId, "pipelineId", overrideRequest.PipelineId) + return releaseNo, dbErr + } + return valuesOverrideResponse.PipelineOverride.PipelineReleaseCounter, nil +} + +func (impl *HandlerServiceImpl) buildManifestPushTemplate(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string) (*bean4.ManifestPushTemplate, error) { + + manifestPushTemplate := &bean4.ManifestPushTemplate{ + WorkflowRunnerId: overrideRequest.WfrId, + AppId: overrideRequest.AppId, + ChartRefId: valuesOverrideResponse.EnvOverride.Chart.ChartRefId, + EnvironmentId: valuesOverrideResponse.EnvOverride.Environment.Id, + EnvironmentName: valuesOverrideResponse.EnvOverride.Environment.Namespace, + UserId: overrideRequest.UserId, + PipelineOverrideId: valuesOverrideResponse.PipelineOverride.Id, + AppName: overrideRequest.AppName, + TargetEnvironmentId: valuesOverrideResponse.EnvOverride.TargetEnvironment, + BuiltChartPath: builtChartPath, + MergedValues: valuesOverrideResponse.MergedValues, + } + + manifestPushConfig, err := impl.manifestPushConfigRepository.GetManifestPushConfigByAppIdAndEnvId(overrideRequest.AppId, overrideRequest.EnvId) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching manifest push config from db", "err", err) + return manifestPushTemplate, err + } + + if manifestPushConfig != nil && manifestPushConfig.Id != 0 { + if manifestPushConfig.StorageType == bean2.ManifestStorageGit { + // need to implement for git repo push + // currently manifest push config doesn't have git push config. GitOps config is derived from charts, chart_env_config_override and chart_ref table + } else { + err2 := impl.buildManifestPushTemplateForNonGitStorageType(overrideRequest, valuesOverrideResponse, builtChartPath, err, manifestPushConfig, manifestPushTemplate) + if err2 != nil { + return manifestPushTemplate, err2 + } + } + } else { + manifestPushTemplate.ChartReferenceTemplate = valuesOverrideResponse.EnvOverride.Chart.ReferenceTemplate + manifestPushTemplate.ChartName = valuesOverrideResponse.EnvOverride.Chart.ChartName + manifestPushTemplate.ChartVersion = valuesOverrideResponse.EnvOverride.Chart.ChartVersion + manifestPushTemplate.ChartLocation = valuesOverrideResponse.DeploymentConfig.GetChartLocation() + manifestPushTemplate.RepoUrl = valuesOverrideResponse.DeploymentConfig.GetRepoURL() + manifestPushTemplate.TargetRevision = valuesOverrideResponse.DeploymentConfig.GetTargetRevision() + manifestPushTemplate.ValuesFilePath = valuesOverrideResponse.DeploymentConfig.GetValuesFilePath() + manifestPushTemplate.ReleaseMode = valuesOverrideResponse.DeploymentConfig.ReleaseMode + manifestPushTemplate.IsCustomGitRepository = common.IsCustomGitOpsRepo(valuesOverrideResponse.DeploymentConfig.ConfigType) + manifestPushTemplate.IsArgoSyncSupported = valuesOverrideResponse.DeploymentConfig.IsArgoAppSyncAndRefreshSupported() + } + return manifestPushTemplate, nil +} + +func (impl *HandlerServiceImpl) deployApp(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, triggerEvent bean.TriggerEvent) error { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.deployApp") + defer span.End() + var referenceChartByte []byte + var err error + + if util.IsAcdApp(overrideRequest.DeploymentAppType) && triggerEvent.DeployArgoCdApp { + err = impl.deployArgoCdApp(newCtx, overrideRequest, valuesOverrideResponse) + if err != nil { + impl.logger.Errorw("error in deploying app on ArgoCd", "err", err) + return err + } + } else if util.IsHelmApp(overrideRequest.DeploymentAppType) { + _, referenceChartByte, err = impl.createHelmAppForCdPipeline(newCtx, overrideRequest, valuesOverrideResponse) + if err != nil { + impl.logger.Errorw("error in creating or updating helm application for cd pipeline", "err", err) + return err + } + } + impl.postDeployHook(overrideRequest, valuesOverrideResponse, referenceChartByte, err) + return nil +} + +func (impl *HandlerServiceImpl) createHelmAppForCdPipeline(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse) (bool, []byte, error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.createHelmAppForCdPipeline") + defer span.End() + pipelineModel := valuesOverrideResponse.Pipeline + envOverride := valuesOverrideResponse.EnvOverride + mergeAndSave := valuesOverrideResponse.MergedValues + chartMetaData, helmRevisionHistory, releaseIdentifier, err := impl.getHelmHistoryLimitAndChartMetadataForHelmAppCreation(ctx, valuesOverrideResponse) + if err != nil { + impl.logger.Errorw("error, getHelmHistoryLimitAndChartMetadataForHelmAppCreation", "valuesOverrideResponse", valuesOverrideResponse, "err", err) + return false, nil, err + } + referenceTemplate := envOverride.Chart.ReferenceTemplate + referenceTemplatePath := path.Join(bean5.RefChartDirPath, referenceTemplate) + var referenceChartByte []byte + if util.IsHelmApp(valuesOverrideResponse.DeploymentConfig.DeploymentAppType) { + sanitizedK8sVersion, err := impl.getSanitizedK8sVersion(referenceTemplate) + if err != nil { + return false, nil, err + } + referenceChartByte, err = impl.getReferenceChartByteForHelmTypeApp(envOverride, chartMetaData, referenceTemplatePath, overrideRequest, valuesOverrideResponse) + if err != nil { + impl.logger.Errorw("error, getReferenceChartByteForHelmTypeApp", "envOverride", envOverride, "err", err) + return false, nil, err + } + if pipelineModel.DeploymentAppCreated { + req := &gRPC.UpgradeReleaseRequest{ + ReleaseIdentifier: releaseIdentifier, + ValuesYaml: mergeAndSave, + HistoryMax: helmRevisionHistory, + ChartContent: &gRPC.ChartContent{Content: referenceChartByte}, + } + if len(sanitizedK8sVersion) > 0 { + req.K8SVersion = sanitizedK8sVersion + } + if impl.isDevtronAsyncHelmInstallModeEnabled(overrideRequest.ForceSyncDeployment) { + req.RunInCtx = true + } + // For cases where helm release was not found, kubelink will install the same configuration + updateApplicationResponse, err := impl.helmAppClient.UpdateApplication(newCtx, req) + if err != nil { + impl.logger.Errorw("error in updating helm application for cd pipelineModel", "err", err) + if util.IsErrorContextCancelled(err) { + return false, nil, cdWorkflow.ErrorDeploymentSuperseded + } else if util.IsErrorContextDeadlineExceeded(err) { + return false, nil, context.DeadlineExceeded + } + apiError := clientErrors.ConvertToApiError(err) + if apiError != nil { + return false, nil, apiError + } + return false, nil, err + } else { + impl.logger.Debugw("updated helm application", "response", updateApplicationResponse, "isSuccess", updateApplicationResponse.Success) + } + + } else { + + helmResponse, err := impl.helmInstallReleaseWithCustomChart(newCtx, releaseIdentifier, referenceChartByte, + mergeAndSave, sanitizedK8sVersion, overrideRequest.ForceSyncDeployment) + + // For connection related errors, no need to update the db + if err != nil && strings.Contains(err.Error(), "connection error") { + impl.logger.Errorw("error in helm install custom chart", "err", err) + return false, nil, err + } + + // IMP: update cd pipelineModel to mark deployment app created, even if helm install fails + // If the helm install fails, it still creates the app in failed state, so trying to + // re-create the app results in error from helm that cannot re-use name which is still in use + _, pgErr := impl.updatePipeline(pipelineModel, overrideRequest.UserId) + + if err != nil { + impl.logger.Errorw("error in helm install custom chart", "err", err) + if pgErr != nil { + impl.logger.Errorw("failed to update deployment app created flag in pipelineModel table", "err", err) + } + if util.IsErrorContextCancelled(err) { + return false, nil, cdWorkflow.ErrorDeploymentSuperseded + } else if util.IsErrorContextDeadlineExceeded(err) { + return false, nil, context.DeadlineExceeded + } + apiError := clientErrors.ConvertToApiError(err) + if apiError != nil { + return false, nil, apiError + } + return false, nil, err + } + + if pgErr != nil { + impl.logger.Errorw("failed to update deployment app created flag in pipelineModel table", "err", err) + return false, nil, err + } + + impl.logger.Debugw("received helm release response", "helmResponse", helmResponse, "isSuccess", helmResponse.Success) + } + + //update workflow runner status, used in app workflow view + err = impl.cdWorkflowCommonService.UpdateNonTerminalStatusInRunner(newCtx, overrideRequest.WfrId, overrideRequest.UserId, cdWorkflow.WorkflowInProgress) + if err != nil { + impl.logger.Errorw("error in updating the workflow runner status, createHelmAppForCdPipeline", "err", err) + return false, nil, err + } + } + return true, referenceChartByte, nil +} + +func (impl *HandlerServiceImpl) getHelmHistoryLimitAndChartMetadataForHelmAppCreation(ctx context.Context, + valuesOverrideResponse *app.ValuesOverrideResponse) (*chart.Metadata, int32, *gRPC.ReleaseIdentifier, error) { + pipelineModel := valuesOverrideResponse.Pipeline + envOverride := valuesOverrideResponse.EnvOverride + + var chartMetaData *chart.Metadata + releaseName := pipelineModel.DeploymentAppName + //getting cluster by id + cluster, err := impl.clusterRepository.FindById(envOverride.Environment.ClusterId) + if err != nil { + impl.logger.Errorw("error in getting cluster by id", "clusterId", envOverride.Environment.ClusterId, "err", err) + return nil, 0, nil, err + } else if cluster == nil { + impl.logger.Errorw("error in getting cluster by id, found nil object", "clusterId", envOverride.Environment.ClusterId) + return nil, 0, nil, err + } + + clusterConfig := impl.getClusterGRPCConfig(*cluster) + + releaseIdentifier := &gRPC.ReleaseIdentifier{ + ReleaseName: releaseName, + ReleaseNamespace: envOverride.Namespace, + ClusterConfig: clusterConfig, + } + + var helmRevisionHistory int32 + if valuesOverrideResponse.DeploymentConfig.ReleaseMode == util.PIPELINE_RELEASE_MODE_LINK { + detail, err := impl.helmAppClient.GetReleaseDetails(ctx, releaseIdentifier) + if err != nil { + impl.logger.Errorw("error in fetching release details", "clusterId", clusterConfig.ClusterId, "namespace", envOverride.Namespace, "releaseName", releaseName, "err", err) + return nil, 0, nil, err + } + chartMetaData = &chart.Metadata{ + Name: detail.ChartName, + Version: detail.ChartVersion, + } + //not modifying revision history in case of linked release + helmRevisionHistory = impl.helmAppService.GetRevisionHistoryMaxValue(bean6.SOURCE_LINKED_HELM_APP) + } else { + chartMetaData = &chart.Metadata{ + Name: pipelineModel.App.AppName, + Version: envOverride.Chart.ChartVersion, + } + helmRevisionHistory = impl.helmAppService.GetRevisionHistoryMaxValue(bean6.SOURCE_DEVTRON_APP) + } + + return chartMetaData, helmRevisionHistory, releaseIdentifier, nil +} + +func (impl *HandlerServiceImpl) deployArgoCdApp(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, + valuesOverrideResponse *app.ValuesOverrideResponse) error { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.deployArgoCdApp") + defer span.End() + name, err := impl.createArgoApplicationIfRequired(newCtx, valuesOverrideResponse.EnvOverride, valuesOverrideResponse.Pipeline, valuesOverrideResponse.DeploymentConfig, overrideRequest.UserId) + if err != nil { + impl.logger.Errorw("acd application create error on cd trigger", "err", err, "req", overrideRequest) + return err + } + impl.logger.Debugw("ArgoCd application created", "name", name) + updateAppInArgoCd, err := impl.updateArgoPipeline(newCtx, valuesOverrideResponse.Pipeline, valuesOverrideResponse.EnvOverride, valuesOverrideResponse.DeploymentConfig) + if err != nil { + impl.logger.Errorw("error in updating argocd app ", "err", err) + return err + } + if valuesOverrideResponse.DeploymentConfig.IsArgoAppSyncAndRefreshSupported() { + syncTime := time.Now() + targetRevision := valuesOverrideResponse.DeploymentConfig.GetTargetRevision() + err = impl.argoClientWrapperService.SyncArgoCDApplicationIfNeededAndRefresh(newCtx, valuesOverrideResponse.Pipeline.DeploymentAppName, targetRevision) + if err != nil { + impl.logger.Errorw("error in getting argo application with normal refresh", "argoAppName", valuesOverrideResponse.Pipeline.DeploymentAppName) + return fmt.Errorf("%s. err: %s", bean.ARGOCD_SYNC_ERROR, util.GetClientErrorDetailedMessage(err)) + } + if impl.ACDConfig.IsManualSyncEnabled() { + timeline := &pipelineConfig.PipelineStatusTimeline{ + CdWorkflowRunnerId: overrideRequest.WfrId, + StatusTime: syncTime, + Status: timelineStatus.TIMELINE_STATUS_ARGOCD_SYNC_COMPLETED, + StatusDetail: timelineStatus.TIMELINE_DESCRIPTION_ARGOCD_SYNC_COMPLETED, + } + timeline.CreateAuditLog(overrideRequest.UserId) + _, err = impl.pipelineStatusTimelineService.SaveTimelineIfNotAlreadyPresent(timeline, nil) + if err != nil { + impl.logger.Errorw("error in saving pipeline status timeline", "err", err) + } + } + } + if updateAppInArgoCd { + impl.logger.Debug("argo-cd successfully updated") + } else { + impl.logger.Debug("argo-cd failed to update, ignoring it") + } + return nil +} + +// update repoUrl, revision and argo app sync mode (auto/manual) if needed +func (impl *HandlerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envOverride *bean10.EnvConfigOverride, deploymentConfig *bean9.DeploymentConfig) (bool, error) { + if !deploymentConfig.IsArgoAppPatchSupported() { + impl.logger.Infow("argo app patch not supported", "pipelineId", pipeline.Id, "pipelineName", pipeline.Name) + return false, nil + } + if ctx == nil { + impl.logger.Errorw("err in syncing ACD, ctx is NULL", "pipelineId", pipeline.Id, "pipelineName", pipeline.Name) + return false, nil + } + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.updateArgoPipeline") + defer span.End() + argoAppName := pipeline.DeploymentAppName + impl.logger.Infow("received payload, updateArgoPipeline", "appId", pipeline.AppId, "pipelineName", pipeline.Name, "envId", envOverride.TargetEnvironment, "argoAppName", argoAppName) + argoApplication, err := impl.argoClientWrapperService.GetArgoAppByName(newCtx, argoAppName) + if err != nil { + impl.logger.Errorw("unable to get ArgoCd app", "app", argoAppName, "pipeline", pipeline.Name, "err", err) + return false, err + } + // if status, ok:=status.FromError(err);ok{ + appStatus, _ := status2.FromError(err) + if appStatus.Code() == codes.OK { + impl.logger.Debugw("argo app exists", "app", argoAppName, "pipeline", pipeline.Name) + if impl.argoClientWrapperService.IsArgoAppPatchRequired(argoApplication.Spec.Source, deploymentConfig.GetRepoURL(), deploymentConfig.GetTargetRevision(), deploymentConfig.GetChartLocation()) { + patchRequestDto := &bean7.ArgoCdAppPatchReqDto{ + ArgoAppName: argoAppName, + ChartLocation: deploymentConfig.GetChartLocation(), + GitRepoUrl: deploymentConfig.GetRepoURL(), + TargetRevision: deploymentConfig.GetTargetRevision(), + PatchType: bean7.PatchTypeMerge, + } + url, err := impl.gitOperationService.GetRepoUrlWithUserName(deploymentConfig.GetRepoURL()) + if err != nil { + return false, err + } + patchRequestDto.GitRepoUrl = url + err = impl.argoClientWrapperService.PatchArgoCdApp(newCtx, patchRequestDto) + if err != nil { + impl.logger.Errorw("error in patching argo pipeline", "err", err, "req", patchRequestDto) + return false, err + } + if deploymentConfig.GetRepoURL() != argoApplication.Spec.Source.RepoURL { + impl.logger.Infow("patching argo application's repo url", "argoAppName", argoAppName) + } + if deploymentConfig.GetTargetRevision() != argoApplication.Spec.Source.TargetRevision { + impl.logger.Infow("patching argo application's revision", "argoAppName", argoAppName) + } + impl.logger.Debugw("pipeline update req", "res", patchRequestDto) + } else { + impl.logger.Debug("pipeline no need to update ") + } + err := impl.argoClientWrapperService.UpdateArgoCDSyncModeIfNeeded(newCtx, argoApplication) + if err != nil { + impl.logger.Errorw("error in updating argocd sync mode", "err", err) + return false, err + } + return true, nil + } else if appStatus.Code() == codes.NotFound { + impl.logger.Errorw("argo app not found", "app", argoAppName, "pipeline", pipeline.Name) + return false, nil + } else { + impl.logger.Errorw("err in checking application on argoCD", "err", err, "pipeline", pipeline.Name) + return false, err + } +} + +func (impl *HandlerServiceImpl) createArgoApplicationIfRequired(ctx context.Context, envConfigOverride *bean10.EnvConfigOverride, + pipeline *pipelineConfig.Pipeline, deploymentConfig *bean9.DeploymentConfig, userId int32) (string, error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.createArgoApplicationIfRequired") + defer span.End() + envModel, err := impl.envRepository.FindById(envConfigOverride.TargetEnvironment) + if err != nil { + return "", err + } + argoAppName := pipeline.DeploymentAppName + if !deploymentConfig.IsArgoAppCreationRequired(pipeline.DeploymentAppCreated) { + return argoAppName, nil + } else { + impl.logger.Debugw("new pipeline found", "pipeline", pipeline) + // create + appNamespace := envConfigOverride.Namespace + if appNamespace == "" { + appNamespace = "default" + } + namespace := argocdServer.DevtronInstalationNs + + appRequest := &argocdServer.AppTemplate{ + ApplicationName: argoAppName, + Namespace: namespace, + TargetNamespace: appNamespace, + TargetServer: envModel.Cluster.ServerUrl, + Project: "default", + ValuesFile: helper.GetValuesFileForEnv(envModel.Id), + RepoPath: deploymentConfig.GetChartLocation(), + RepoUrl: deploymentConfig.GetRepoURL(), + AutoSyncEnabled: impl.ACDConfig.ArgoCDAutoSyncEnabled, + } + appRequest.RepoUrl, err = impl.gitOperationService.GetRepoUrlWithUserName(appRequest.RepoUrl) + if err != nil { + return "", err + } + createdArgoAppName, err := impl.argoK8sClient.CreateAcdApp(newCtx, appRequest, argocdServer.ARGOCD_APPLICATION_TEMPLATE) + if err != nil { + return "", err + } + // update cd pipeline to mark deployment app created + _, err = impl.updatePipeline(pipeline, userId) + if err != nil { + impl.logger.Errorw("error in update cd pipeline for deployment app created or not", "err", err) + return "", err + } + return createdArgoAppName, nil + } +} + +func (impl *HandlerServiceImpl) updatePipeline(pipeline *pipelineConfig.Pipeline, userId int32) (bool, error) { + err := impl.pipelineRepository.SetDeploymentAppCreatedInPipeline(true, pipeline.Id, userId) + if err != nil { + impl.logger.Errorw("error on updating cd pipeline for setting deployment app created", "err", err) + return false, err + } + return true, nil +} + +// helmInstallReleaseWithCustomChart performs helm install with custom chart +func (impl *HandlerServiceImpl) helmInstallReleaseWithCustomChart(ctx context.Context, releaseIdentifier *gRPC.ReleaseIdentifier, + referenceChartByte []byte, valuesYaml, k8sServerVersion string, forceSync bool) (*gRPC.HelmInstallCustomResponse, error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.helmInstallReleaseWithCustomChart") + defer span.End() + helmInstallRequest := gRPC.HelmInstallCustomRequest{ + ValuesYaml: valuesYaml, + ChartContent: &gRPC.ChartContent{Content: referenceChartByte}, + ReleaseIdentifier: releaseIdentifier, + } + if len(k8sServerVersion) > 0 { + helmInstallRequest.K8SVersion = k8sServerVersion + } + if impl.isDevtronAsyncHelmInstallModeEnabled(forceSync) { + helmInstallRequest.RunInCtx = true + } + // Request exec + return impl.helmAppClient.InstallReleaseWithCustomChart(newCtx, &helmInstallRequest) +} + +func (impl *HandlerServiceImpl) markImageScanDeployed(ctx context.Context, appId, envId, clusterId int, + imageDigest string, isScanEnabled bool, image string) error { + _, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.markImageScanDeployed") + defer span.End() + // TODO KB: send NATS event for self consumption + impl.logger.Debugw("mark image scan deployed for devtron app, from cd auto or manual trigger", "imageDigest", imageDigest) + executionHistory, err := impl.imageScanHistoryReadService.FindByImageAndDigest(imageDigest, image) + if err != nil && !errors.Is(err, pg.ErrNoRows) { + impl.logger.Errorw("error in fetching execution history", "err", err) + return err + } + if errors.Is(err, pg.ErrNoRows) || executionHistory == nil || executionHistory.Id == 0 { + if isScanEnabled { + // There should ImageScanHistory for ScanEnabled artifacts + impl.logger.Errorw("no execution history found for digest", "digest", imageDigest) + return fmt.Errorf("no execution history found for digest - %s", imageDigest) + } else { + // For ScanDisabled artifacts it should be an expected condition + impl.logger.Infow("no execution history found for digest", "digest", imageDigest) + return nil + } + } + impl.logger.Debugw("saving image_scan_deploy_info for cd auto or manual trigger", "executionHistory", executionHistory) + var ids []int + ids = append(ids, executionHistory.Id) + + ot, err := impl.imageScanDeployInfoReadService.FetchByAppIdAndEnvId(appId, envId, []string{repository6.ScanObjectType_APP}) + + if err == pg.ErrNoRows && !isScanEnabled { + // ignoring if no rows are found and scan is disabled + return nil + } + + if err != nil && err != pg.ErrNoRows { + return err + } else if err == pg.ErrNoRows && isScanEnabled { + imageScanDeployInfo := &repository6.ImageScanDeployInfo{ + ImageScanExecutionHistoryId: ids, + ScanObjectMetaId: appId, + ObjectType: repository6.ScanObjectType_APP, + EnvId: envId, + ClusterId: clusterId, + AuditLog: sql.AuditLog{ + CreatedOn: time.Now(), + CreatedBy: 1, + UpdatedOn: time.Now(), + UpdatedBy: 1, + }, + } + impl.logger.Debugw("mark image scan deployed for normal app, from cd auto or manual trigger", "imageScanDeployInfo", imageScanDeployInfo) + err = impl.imageScanDeployInfoService.Save(imageScanDeployInfo) + if err != nil { + impl.logger.Errorw("error in creating deploy info", "err", err) + } + } else { + // Updating Execution history for Latest Deployment to fetch out security Vulnerabilities for latest deployed info + if isScanEnabled { + ot.ImageScanExecutionHistoryId = ids + } else { + arr := []int{-1} + ot.ImageScanExecutionHistoryId = arr + } + err = impl.imageScanDeployInfoService.Update(ot) + if err != nil { + impl.logger.Errorw("error in updating deploy info for latest deployed image", "err", err) + } + } + return err +} + +func (impl *HandlerServiceImpl) isDevtronAsyncHelmInstallModeEnabled(forceSync bool) bool { + return impl.globalEnvVariables.EnableAsyncHelmInstallDevtronChart && !forceSync +} + +func (impl *HandlerServiceImpl) isDevtronAsyncInstallModeEnabled(overrideRequest *bean3.ValuesOverrideRequest) (bool, error) { + if util.IsHelmApp(overrideRequest.DeploymentAppType) { + return impl.isDevtronAsyncHelmInstallModeEnabled(overrideRequest.ForceSyncDeployment), nil + } else if util.IsAcdApp(overrideRequest.DeploymentAppType) { + return impl.isDevtronAsyncArgoCdInstallModeEnabledForApp(overrideRequest.AppId, + overrideRequest.EnvId, overrideRequest.ForceSyncDeployment) + } + return false, nil +} + +func (impl *HandlerServiceImpl) deleteCorruptedPipelineStage(pipelineStage *repository.PipelineStage, triggeredBy int32) (error, bool) { + if pipelineStage != nil { + stageReq := &bean8.PipelineStageDto{ + Id: pipelineStage.Id, + Type: pipelineStage.Type, + } + err, deleted := impl.pipelineStageService.DeletePipelineStageIfReq(stageReq, triggeredBy) + if err != nil { + impl.logger.Errorw("error in deleting the corrupted pipeline stage", "err", err, "pipelineStageReq", stageReq) + return err, false + } + return nil, deleted + } + return nil, false +} + +func (impl *HandlerServiceImpl) handleCustomGitOpsRepoValidation(runner *pipelineConfig.CdWorkflowRunner, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredBy int32) error { + if !util.IsAcdApp(pipeline.DeploymentAppName) { + return nil + } + isGitOpsConfigured := false + gitOpsConfig, err := impl.gitOpsConfigReadService.GetGitOpsConfigActive() + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error while getting active GitOpsConfig", "err", err) + } + if gitOpsConfig != nil && gitOpsConfig.Id > 0 { + isGitOpsConfigured = true + } + if isGitOpsConfigured && gitOpsConfig.AllowCustomRepository { + //chart, err := impl.chartRepository.FindLatestChartForAppByAppId(pipeline.AppId) + //if err != nil { + // impl.logger.Errorw("error in fetching latest chart for app by appId", "err", err, "appId", pipeline.AppId) + // return err + //} + if gitOps.IsGitOpsRepoNotConfigured(envDeploymentConfig.GetRepoURL()) { + if err = impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, errors.New(cdWorkflow.GITOPS_REPO_NOT_CONFIGURED), triggeredBy); err != nil { + impl.logger.Errorw("error while updating current runner status to failed, TriggerDeployment", "wfrId", runner.Id, "err", err) + } + apiErr := &util.ApiError{ + HttpStatusCode: http.StatusConflict, + UserMessage: cdWorkflow.GITOPS_REPO_NOT_CONFIGURED, + InternalMessage: cdWorkflow.GITOPS_REPO_NOT_CONFIGURED, + } + return apiErr + } + } + return nil +} + +func (impl *HandlerServiceImpl) getSanitizedK8sVersion(referenceTemplate string) (string, error) { + var sanitizedK8sVersion string + //handle specific case for all cronjob charts from cronjob-chart_1-2-0 to cronjob-chart_1-5-0 where semverCompare + //comparison func has wrong api version mentioned, so for already installed charts via these charts that comparison + //is always false, handles the gh issue:- https://github.com/devtron-labs/devtron/issues/4860 + cronJobChartRegex := regexp.MustCompile(bean.CronJobChartRegexExpression) + if cronJobChartRegex.MatchString(referenceTemplate) { + k8sServerVersion, err := impl.K8sUtil.GetKubeVersion() + if err != nil { + impl.logger.Errorw("exception caught in getting k8sServerVersion", "err", err) + return "", err + } + sanitizedK8sVersion = k8s2.StripPrereleaseFromK8sVersion(k8sServerVersion.String()) + } + return sanitizedK8sVersion, nil +} + +func (impl *HandlerServiceImpl) getReferenceChartByteForHelmTypeApp(envOverride *bean10.EnvConfigOverride, + chartMetaData *chart.Metadata, referenceTemplatePath string, overrideRequest *bean3.ValuesOverrideRequest, + valuesOverrideResponse *app.ValuesOverrideResponse) ([]byte, error) { + referenceChartByte := envOverride.Chart.ReferenceChart + // here updating reference chart into database. + if len(envOverride.Chart.ReferenceChart) == 0 { + refChartByte, err := impl.chartTemplateService.GetByteArrayRefChart(chartMetaData, referenceTemplatePath) + if err != nil { + impl.logger.Errorw("ref chart commit error on cd trigger", "err", err, "req", overrideRequest) + return nil, err + } + ch := envOverride.Chart + ch.ReferenceChart = refChartByte + ch.UpdatedOn = time.Now() + ch.UpdatedBy = overrideRequest.UserId + err = impl.chartRepository.Update(ch) + if err != nil { + impl.logger.Errorw("chart update error", "err", err, "req", overrideRequest) + return nil, err + } + referenceChartByte = refChartByte + } + var err error + referenceChartByte, err = impl.overrideReferenceChartByteForHelmTypeApp(valuesOverrideResponse, chartMetaData, referenceTemplatePath, referenceChartByte) + if err != nil { + impl.logger.Errorw("ref chart commit error on cd trigger", "err", err, "req", overrideRequest) + return nil, err + } + return referenceChartByte, nil +} From 0ac8f1fae65bbdd4b8a361cdc35f8f0ab3db1759 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 8 Apr 2025 13:05:07 +0530 Subject: [PATCH 36/37] file renamed --- .../{preStageHandlerService.go => preStageHandlerCode.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pkg/deployment/trigger/devtronApps/{preStageHandlerService.go => preStageHandlerCode.go} (100%) diff --git a/pkg/deployment/trigger/devtronApps/preStageHandlerService.go b/pkg/deployment/trigger/devtronApps/preStageHandlerCode.go similarity index 100% rename from pkg/deployment/trigger/devtronApps/preStageHandlerService.go rename to pkg/deployment/trigger/devtronApps/preStageHandlerCode.go From 555f96ba4fc8d98ff13d3c68eeb7ae31524ea227 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Fri, 11 Apr 2025 16:42:42 +0530 Subject: [PATCH 37/37] updated BuildxCacheFlags description --- cmd/external-app/wire_gen.go | 2 +- env_gen.json | 2 +- env_gen.md | 412 ++++++++++++++-------------- pkg/build/trigger/HandlerService.go | 4 +- wire_gen.go | 2 +- 5 files changed, 211 insertions(+), 211 deletions(-) diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index f18942e46e..14934dea14 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run -mod=mod github.com/google/wire/cmd/wire +//go:generate go run github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject diff --git a/env_gen.json b/env_gen.json index ad32c0da18..6fd874d675 100644 --- a/env_gen.json +++ b/env_gen.json @@ -1 +1 @@ -[{"Category":"CD","Fields":[{"Env":"ARGO_APP_MANUAL_SYNC_TIME","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_HELM_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_PIPELINE_STATUS_TIMEOUT_DURATION","EnvType":"string","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS","EnvType":"int","EnvValue":"12","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_CHART_ARGO_CD_INSTALL_REQUEST_TIMEOUT","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT","EnvType":"int","EnvValue":"6","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXPOSE_CD_METRICS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FEATURE_MIGRATE_ARGOCD_APPLICATION_ENABLE","EnvType":"bool","EnvValue":"false","EnvDescription":"enable migration of external argocd application to devtron pipeline","Example":"","Deprecated":"false"},{"Env":"HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME","EnvType":"string","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IS_INTERNAL_USE","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MIGRATE_DEPLOYMENT_CONFIG_DATA","EnvType":"bool","EnvValue":"false","EnvDescription":"migrate deployment config data from charts table to deployment_config table","Example":"","Deprecated":"false"},{"Env":"PIPELINE_DEGRADED_TIME","EnvType":"string","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_DEVTRON_APP","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_EXTERNAL_HELM_APP","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_HELM_APP","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_LINKED_HELM_APP","EnvType":"int","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUN_HELM_INSTALL_IN_ASYNC_MODE_HELM_APPS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SHOULD_CHECK_NAMESPACE_ON_CLONE","EnvType":"bool","EnvValue":"false","EnvDescription":"should we check if namespace exists or not while cloning app","Example":"","Deprecated":"false"},{"Env":"USE_DEPLOYMENT_CONFIG_DATA","EnvType":"bool","EnvValue":"false","EnvDescription":"use deployment config data from deployment_config table","Example":"","Deprecated":"true"}]},{"Category":"CI_RUNNER","Fields":[{"Env":"AZURE_ACCOUNT_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_ACCOUNT_NAME","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_CACHE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_LOG","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_CONNECTION_INSECURE","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_URL","EnvType":"string","EnvValue":"http://devtron-minio.devtroncd:9000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BASE_LOG_LOCATION_PATH","EnvType":"string","EnvValue":"/home/devtron/","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_GCP_CREDENTIALS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_PROVIDER","EnvType":"","EnvValue":"S3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ACCESS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_BUCKET_VERSIONED","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT_INSECURE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_SECRET_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_CACHE_PATH","EnvType":"string","EnvValue":"/var/lib/devtron/buildx","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_K8S_DRIVER_OPTIONS","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_PROVENANCE_MODE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILD_LOG_TTL_VALUE_IN_SECS","EnvType":"int","EnvValue":"3600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CACHE_LIMIT","EnvType":"int64","EnvValue":"5000000000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_DEFAULT_ADDRESS_POOL_BASE_CIDR","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_DEFAULT_ADDRESS_POOL_SIZE","EnvType":"int","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_LIMIT_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_LIMIT_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"dedicated","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"ci","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_REQ_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_REQ_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_WORKFLOW_EXECUTOR_TYPE","EnvType":"","EnvValue":"AWF","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_WORKFLOW_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"cd-runner","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_DEFAULT_ADDRESS_POOL_BASE_CIDR","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_DEFAULT_ADDRESS_POOL_SIZE","EnvType":"int","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_IGNORE_DOCKER_CACHE","EnvType":"bool","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_LOGS_KEY_PREFIX","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_RUNNER_DOCKER_MTU_VALUE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_SUCCESS_AUTO_TRIGGER_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_VOLUME_MOUNTS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_WORKFLOW_EXECUTOR_TYPE","EnvType":"","EnvValue":"AWF","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_ARTIFACT_KEY_LOCATION","EnvType":"string","EnvValue":"arsenal-v1/ci-artifacts","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_BUCKET","EnvType":"string","EnvValue":"devtron-pro-ci-logs","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_KEY_PREFIX","EnvType":"string","EnvValue":"arsenal-v1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET","EnvType":"string","EnvValue":"ci-caching","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET_REGION","EnvType":"string","EnvValue":"us-east-2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_ARTIFACT_KEY_LOCATION","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_LOGS_BUCKET_REGION","EnvType":"string","EnvValue":"us-east-2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_NAMESPACE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_TIMEOUT","EnvType":"int64","EnvValue":"3600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CI_IMAGE","EnvType":"string","EnvValue":"686244538589.dkr.ecr.us-east-2.amazonaws.com/cirunner:47","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtron-ci","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_TARGET_PLATFORM","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DOCKER_BUILD_CACHE_PATH","EnvType":"string","EnvValue":"/var/lib/docker","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_BUILD_CONTEXT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_WORKFLOW_EXECUTION_STAGE","EnvType":"bool","EnvValue":"true","EnvDescription":"if enabled then we will display build stages separately for CI/Job/Pre-Post CD","Example":"true","Deprecated":"false"},{"Env":"EXTERNAL_BLOB_STORAGE_CM_NAME","EnvType":"string","EnvValue":"blob-storage-cm","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_BLOB_STORAGE_SECRET_NAME","EnvType":"string","EnvValue":"blob-storage-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"dedicated","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"ci","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_API_SECRET","EnvType":"string","EnvValue":"devtroncd-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_PAYLOAD","EnvType":"string","EnvValue":"{\"ciProjectDetails\":[{\"gitRepository\":\"https://github.com/vikram1601/getting-started-nodejs.git\",\"checkoutPath\":\"./abc\",\"commitHash\":\"239077135f8cdeeccb7857e2851348f558cb53d3\",\"commitTime\":\"2022-10-30T20:00:00\",\"branch\":\"master\",\"message\":\"Update README.md\",\"author\":\"User Name \"}],\"dockerImage\":\"445808685819.dkr.ecr.us-east-2.amazonaws.com/orch:23907713-2\"}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_WEB_HOOK_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IGNORE_CM_CS_IN_CI_JOB","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_RETRY_COUNT","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_RETRY_INTERVAL","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCANNER_ENDPOINT","EnvType":"string","EnvValue":"http://image-scanner-new-demo-devtroncd-service.devtroncd:80","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_MAX_RETRIES","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_RETRY_DELAY","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IN_APP_LOGGING_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_CD_WORKFLOW_RUNNER_RETRIES","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_CI_WORKFLOW_RETRIES","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MODE","EnvType":"string","EnvValue":"DEV","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"localhost:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ORCH_HOST","EnvType":"string","EnvValue":"http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ORCH_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PRE_CI_CACHE_PATH","EnvType":"string","EnvValue":"/devtroncd-cache","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SHOW_DOCKER_BUILD_ARGS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SKIP_CI_JOB_BUILD_CACHE_PUSH_PULL","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SKIP_CREATING_ECR_REPO","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TERMINATION_GRACE_PERIOD_SECS","EnvType":"int","EnvValue":"180","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_ARTIFACT_LISTING_QUERY_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_BLOB_STORAGE_CONFIG_IN_CI_WORKFLOW","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_BUILDX","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_DOCKER_API_TO_GET_DIGEST","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_EXTERNAL_NODE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_IMAGE_TAG_FROM_GIT_PROVIDER_FOR_TAG_BASED_BUILD","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"WF_CONTROLLER_INSTANCE_ID","EnvType":"string","EnvValue":"devtron-runner","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"WORKFLOW_CACHE_CONFIG","EnvType":"string","EnvValue":"{}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"WORKFLOW_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"ci-runner","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"DEVTRON","Fields":[{"Env":"-","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ADDITIONAL_NODE_GROUP_LABELS","EnvType":"","EnvValue":"","EnvDescription":"Add comma separated list of additional node group labels to default labels","Example":"karpenter.sh/nodepool,cloud.google.com/gke-nodepool","Deprecated":"false"},{"Env":"APP_SYNC_IMAGE","EnvType":"string","EnvValue":"quay.io/devtron/chart-sync:1227622d-132-3775","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_JOB_RESOURCES_OBJ","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"chart-sync","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_SHUTDOWN_WAIT_DURATION","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_AUTO_SYNC_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_GIT_COMMIT_RETRY_COUNT_ON_CONFLICT","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_GIT_COMMIT_RETRY_DELAY_ON_CONFLICT","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_REPO_REGISTER_RETRY_COUNT","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_REPO_REGISTER_RETRY_DELAY","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ASYNC_BUILDX_CACHE_EXPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BATCH_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_CACHE_MODE_MIN","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_PORT","EnvType":"string","EnvValue":"8000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CExpirationTime","EnvType":"int","EnvValue":"600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_TRIGGER_CRON_TIME","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_WORKFLOW_STATUS_UPDATE_CRON","EnvType":"string","EnvValue":"*/5 * * * *","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLI_CMD_TIMEOUT_GLOBAL_SECONDS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLUSTER_STATUS_CRON_TIME","EnvType":"int","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_TIMEOUT","EnvType":"float64","EnvValue":"3600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_BOM_URL","EnvType":"string","EnvValue":"https://raw.githubusercontent.com/devtron-labs/devtron/%s/charts/devtron/devtron-bom.yaml","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_DEX_SECRET_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_CHART_NAME","EnvType":"string","EnvValue":"devtron-operator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_NAME","EnvType":"string","EnvValue":"devtron","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_REPO_NAME","EnvType":"string","EnvValue":"devtron","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_REPO_URL","EnvType":"string","EnvValue":"https://helm.devtron.ai","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_INSTALLATION_TYPE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_MODULES_IDENTIFIER_IN_HELM_VALUES","EnvType":"string","EnvValue":"installer.modules","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_SECRET_NAME","EnvType":"string","EnvValue":"devtron-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_VERSION_IDENTIFIER_IN_HELM_VALUES","EnvType":"string","EnvValue":"installer.release","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_CID","EnvType":"string","EnvValue":"example-app","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_CLIENT_ID","EnvType":"string","EnvValue":"argo-cd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_CSTOREKEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_JWTKEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_RURL","EnvType":"string","EnvValue":"http://127.0.0.1:8080/callback","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_SCOPES","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_SECRET","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ECR_REPO_NAME_PREFIX","EnvType":"string","EnvValue":"test/","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_ASYNC_ARGO_CD_INSTALL_DEVTRON_CHART","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_ASYNC_INSTALL_DEVTRON_CHART","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EPHEMERAL_SERVER_VERSION_REGEX","EnvType":"string","EnvValue":"v[1-9]\\.\\b(2[3-9]\\|[3-9][0-9])\\b.*","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EVENT_URL","EnvType":"string","EnvValue":"http://localhost:3000/notify","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXECUTE_WIRE_NIL_CHECKER","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXPOSE_CI_METRICS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FEATURE_RESTART_WORKLOAD_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FEATURE_RESTART_WORKLOAD_WORKER_POOL_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FORCE_SECURITY_SCANNING","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GITOPS_REPO_PREFIX","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GO_RUNTIME_ENV","EnvType":"string","EnvValue":"production","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_ORG_ID","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_PASSWORD","EnvType":"string","EnvValue":"prom-operator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_PORT","EnvType":"string","EnvValue":"8090","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_USERNAME","EnvType":"string","EnvValue":"admin","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"HIDE_IMAGE_TAGGING_HARD_DELETE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IGNORE_AUTOCOMPLETE_AUTH_CHECK","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"INSTALLED_MODULES","EnvType":"","EnvValue":"","EnvDescription":"List of installed modules given in helm values/yaml are written in cm and used by devtron to know which modules are given","Example":"security.trivy,security.clair","Deprecated":"false"},{"Env":"INSTALLER_CRD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_GROUP_NAME","EnvType":"string","EnvValue":"installer.devtron.ai","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_RESOURCE","EnvType":"string","EnvValue":"installers","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_VERSION","EnvType":"string","EnvValue":"v1alpha1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IS_AIR_GAP_ENVIRONMENT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"JwtExpirationTime","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST","EnvType":"int","EnvValue":"25","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_IDLE_CONN_TIMEOUT","EnvType":"int","EnvValue":"300","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_KEEPALIVE","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_TIMEOUT","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TLS_HANDSHAKE_TIMEOUT","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE","EnvType":"int","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_SEND_MSG_SIZE","EnvType":"int","EnvValue":"4","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LENS_TIMEOUT","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LENS_URL","EnvType":"string","EnvValue":"http://lens-milandevtron-service:80","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LIMIT_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LIMIT_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOGGER_DEV_MODE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_SESSION_PER_USER","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MODULE_METADATA_API_URL","EnvType":"string","EnvValue":"https://api.devtron.ai/module?name=%s","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MODULE_STATUS_HANDLING_CRON_DURATION_MIN","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NOTIFICATION_MEDIUM","EnvType":"NotificationMedium","EnvValue":"rest","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"OTEL_COLLECTOR_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PARALLELISM_LIMIT_FOR_TAG_PROCESSING","EnvType":"int","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PLUGIN_NAME","EnvType":"string","EnvValue":"Pull images from container repository","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PROPAGATE_EXTRA_LABELS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PROXY_SERVICE_CONFIG","EnvType":"string","EnvValue":"{}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REQ_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REQ_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RESTRICT_TERMINAL_ACCESS_FOR_NON_SUPER_USER","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUNTIME_CONFIG_LOCAL_DEV","EnvType":"LocalDevMode","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_FORMAT","EnvType":"string","EnvValue":"@{{%s}}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_HANDLE_PRIMITIVES","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_NAME_REGEX","EnvType":"string","EnvValue":"^[a-zA-Z][a-zA-Z0-9_-]{0,62}[a-zA-Z0-9]$","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SOCKET_DISCONNECT_DELAY_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SOCKET_HEARTBEAT_SECONDS","EnvType":"int","EnvValue":"25","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SYSTEM_VAR_PREFIX","EnvType":"string","EnvValue":"DEVTRON_","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"default","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_INACTIVE_DURATION_IN_MINS","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_STATUS_SYNC_In_SECS","EnvType":"int","EnvValue":"600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_APP","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_LOG_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_PASSWORD","EnvType":"string","EnvValue":"postgrespw","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_PORT","EnvType":"string","EnvValue":"55000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_USER","EnvType":"string","EnvValue":"postgres","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TIMEOUT_FOR_FAILED_CI_BUILD","EnvType":"string","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TIMEOUT_IN_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USER_SESSION_DURATION_SECONDS","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_ARTIFACT_LISTING_API_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_CUSTOM_HTTP_TRANSPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_GIT_CLI","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_RBAC_CREATION_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"VARIABLE_CACHE_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"VARIABLE_EXPRESSION_REGEX","EnvType":"string","EnvValue":"@{{([^}]+)}}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"WEBHOOK_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"GITOPS","Fields":[{"Env":"ACD_CM","EnvType":"string","EnvValue":"argocd-cm","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ACD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ACD_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ACD_USERNAME","EnvType":"string","EnvValue":"admin","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GITOPS_SECRET_NAME","EnvType":"string","EnvValue":"devtron-gitops-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RESOURCE_LIST_FOR_REPLICAS","EnvType":"string","EnvValue":"Deployment,Rollout,StatefulSet,ReplicaSet","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RESOURCE_LIST_FOR_REPLICAS_BATCH_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"INFRA_SETUP","Fields":[{"Env":"DASHBOARD_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DASHBOARD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DASHBOARD_PORT","EnvType":"string","EnvValue":"3000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_HOST","EnvType":"string","EnvValue":"http://localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_PORT","EnvType":"string","EnvValue":"5556","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_PROTOCOL","EnvType":"string","EnvValue":"REST","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_TIMEOUT","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_URL","EnvType":"string","EnvValue":"127.0.0.1:7070","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"HELM_CLIENT_URL","EnvType":"string","EnvValue":"127.0.0.1:50051","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"POSTGRES","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"Application name","Example":"","Deprecated":"false"},{"Env":"CASBIN_DATABASE","EnvType":"string","EnvValue":"casbin","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"address of postgres service","Example":"postgresql-postgresql.devtroncd","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"postgres database to be made connection with","Example":"orchestrator, casbin, git_sensor, lens","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"{password}","EnvDescription":"password for postgres, associated with PG_USER","Example":"confidential ;)","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"port of postgresql service","Example":"5432","Deprecated":"false"},{"Env":"PG_READ_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"postgres","EnvDescription":"user for postgres","Example":"postgres","Deprecated":"false"},{"Env":"PG_WRITE_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"RBAC","Fields":[{"Env":"ENFORCER_CACHE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENFORCER_CACHE_EXPIRATION_IN_SEC","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENFORCER_MAX_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_CASBIN_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"}]}] \ No newline at end of file +[{"Category":"CD","Fields":[{"Env":"ARGO_APP_MANUAL_SYNC_TIME","EnvType":"int","EnvValue":"3","EnvDescription":"retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins)","Example":"","Deprecated":"false"},{"Env":"CD_HELM_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"Cron time to check the pipeline status ","Example":"","Deprecated":"false"},{"Env":"CD_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"Cron time for CD pipeline status","Example":"","Deprecated":"false"},{"Env":"CD_PIPELINE_STATUS_TIMEOUT_DURATION","EnvType":"string","EnvValue":"20","EnvDescription":"Timeout for CD pipeline to get healthy","Example":"","Deprecated":"false"},{"Env":"DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS","EnvType":"int","EnvValue":"12","EnvDescription":"This flag is used to fetch the deployment status of the application. It retrieves the status of deployments that occurred between 12 hours and 10 minutes prior to the current time. It fetches non-terminal statuses.","Example":"","Deprecated":"false"},{"Env":"DEVTRON_CHART_ARGO_CD_INSTALL_REQUEST_TIMEOUT","EnvType":"int","EnvValue":"1","EnvDescription":"Context timeout for gitops concurrent async deployments","Example":"","Deprecated":"false"},{"Env":"DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT","EnvType":"int","EnvValue":"6","EnvDescription":"Context timeout for no gitops concurrent async deployments","Example":"","Deprecated":"false"},{"Env":"EXPOSE_CD_METRICS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FEATURE_MIGRATE_ARGOCD_APPLICATION_ENABLE","EnvType":"bool","EnvValue":"false","EnvDescription":"enable migration of external argocd application to devtron pipeline","Example":"","Deprecated":"false"},{"Env":"HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME","EnvType":"string","EnvValue":"120","EnvDescription":"eligible time for checking helm app status periodically and update in db, value is in seconds., default is 120, if wfr is updated within configured time i.e. HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME then do not include for this cron cycle.","Example":"","Deprecated":"false"},{"Env":"IS_INTERNAL_USE","EnvType":"bool","EnvValue":"true","EnvDescription":"If enabled then cd pipeline and helm apps will not need the deployment app type mandatorily. Couple this flag with HIDE_GITOPS_OR_HELM_OPTION (in Dashborad) and if gitops is configured and allowed for the env, pipeline/ helm app will gitops else no-gitops.","Example":"","Deprecated":"false"},{"Env":"MIGRATE_DEPLOYMENT_CONFIG_DATA","EnvType":"bool","EnvValue":"false","EnvDescription":"migrate deployment config data from charts table to deployment_config table","Example":"","Deprecated":"false"},{"Env":"PIPELINE_DEGRADED_TIME","EnvType":"string","EnvValue":"10","EnvDescription":"Time to mark a pipeline degraded if not healthy in defined time","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_DEVTRON_APP","EnvType":"int","EnvValue":"1","EnvDescription":"Count for devtron application rivision history","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_EXTERNAL_HELM_APP","EnvType":"int","EnvValue":"0","EnvDescription":"Count for external helm application rivision history","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_HELM_APP","EnvType":"int","EnvValue":"1","EnvDescription":"To set the history limit for the helm app being deployed through devtron","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_LINKED_HELM_APP","EnvType":"int","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUN_HELM_INSTALL_IN_ASYNC_MODE_HELM_APPS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SHOULD_CHECK_NAMESPACE_ON_CLONE","EnvType":"bool","EnvValue":"false","EnvDescription":"should we check if namespace exists or not while cloning app","Example":"","Deprecated":"false"},{"Env":"USE_DEPLOYMENT_CONFIG_DATA","EnvType":"bool","EnvValue":"false","EnvDescription":"use deployment config data from deployment_config table","Example":"","Deprecated":"true"}]},{"Category":"CI_RUNNER","Fields":[{"Env":"AZURE_ACCOUNT_KEY","EnvType":"string","EnvValue":"","EnvDescription":"If blob storage is bieng used of azure then pass the secret key to access the bucket","Example":"","Deprecated":"false"},{"Env":"AZURE_ACCOUNT_NAME","EnvType":"string","EnvValue":"","EnvDescription":"Account name for azure blob storage","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_CACHE","EnvType":"string","EnvValue":"","EnvDescription":"Cache bucket name for azure blob storage","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_LOG","EnvType":"string","EnvValue":"","EnvDescription":"Log bucket for azure blob storage","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_CONNECTION_INSECURE","EnvType":"bool","EnvValue":"true","EnvDescription":"Azure gateway connection allows insecure if true","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_URL","EnvType":"string","EnvValue":"http://devtron-minio.devtroncd:9000","EnvDescription":"Sent to CI runner for blob","Example":"","Deprecated":"false"},{"Env":"BASE_LOG_LOCATION_PATH","EnvType":"string","EnvValue":"/home/devtron/","EnvDescription":"Used to store, download logs of ci workflow, artifact","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_GCP_CREDENTIALS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"GCP cred json for GCS blob storage","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_PROVIDER","EnvType":"","EnvValue":"S3","EnvDescription":"Blob storage provider name(AWS/GCP/Azure)","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ACCESS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"S3 access key for s3 blob storage","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_BUCKET_VERSIONED","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable buctet versioning for blob storage","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT","EnvType":"string","EnvValue":"","EnvDescription":"S3 endpoint URL for s3 blob storage","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT_INSECURE","EnvType":"bool","EnvValue":"false","EnvDescription":"To use insecure s3 endpoint","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_SECRET_KEY","EnvType":"string","EnvValue":"","EnvDescription":"Secret key for s3 blob storage","Example":"","Deprecated":"false"},{"Env":"BUILDX_CACHE_PATH","EnvType":"string","EnvValue":"/var/lib/devtron/buildx","EnvDescription":"Path for the buildx cache","Example":"","Deprecated":"false"},{"Env":"BUILDX_K8S_DRIVER_OPTIONS","EnvType":"string","EnvValue":"","EnvDescription":"To enable the k8s driver and pass args for k8s driver in buildx","Example":"","Deprecated":"false"},{"Env":"BUILDX_PROVENANCE_MODE","EnvType":"string","EnvValue":"","EnvDescription":"provinance is set to true by default by docker. this will add some build related data in generated build manifest.it also adds some unknown:unknown key:value pair which may not be compatible by some container registries. with buildx k8s driver , provinenance=true is causing issue when push manifest to quay registry, so setting it to false","Example":"","Deprecated":"false"},{"Env":"BUILD_LOG_TTL_VALUE_IN_SECS","EnvType":"int","EnvValue":"3600","EnvDescription":"This is the time that the pods of ci/pre-cd/post-cd live after completion state.","Example":"","Deprecated":"false"},{"Env":"CACHE_LIMIT","EnvType":"int64","EnvValue":"5000000000","EnvDescription":"Cache limit.","Example":"","Deprecated":"false"},{"Env":"CD_DEFAULT_ADDRESS_POOL_BASE_CIDR","EnvType":"string","EnvValue":"","EnvDescription":"To pass the IP cidr for Pre/Post cd ","Example":"","Deprecated":"false"},{"Env":"CD_DEFAULT_ADDRESS_POOL_SIZE","EnvType":"int","EnvValue":"","EnvDescription":"The subnet size to allocate from the base pool for CD","Example":"","Deprecated":"false"},{"Env":"CD_LIMIT_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"CPU Resource Limit Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_LIMIT_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"Memory Resource Limit Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"Node label selector for Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"dedicated","EnvDescription":"Toleration key for Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"ci","EnvDescription":"Toleration value for Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_REQ_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"CPU Resource Rquest Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_REQ_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"Memory Resource Rquest Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_WORKFLOW_EXECUTOR_TYPE","EnvType":"","EnvValue":"AWF","EnvDescription":"Executor type for Pre/Post CD(AWF,System)","Example":"","Deprecated":"false"},{"Env":"CD_WORKFLOW_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"cd-runner","EnvDescription":"Service account to be used in Pre/Post CD pod","Example":"","Deprecated":"false"},{"Env":"CI_DEFAULT_ADDRESS_POOL_BASE_CIDR","EnvType":"string","EnvValue":"","EnvDescription":"To pass the IP cidr for CI","Example":"","Deprecated":"false"},{"Env":"CI_DEFAULT_ADDRESS_POOL_SIZE","EnvType":"int","EnvValue":"","EnvDescription":"The subnet size to allocate from the base pool for CI","Example":"","Deprecated":"false"},{"Env":"CI_IGNORE_DOCKER_CACHE","EnvType":"bool","EnvValue":"","EnvDescription":"Ignoring docker cache ","Example":"","Deprecated":"false"},{"Env":"CI_LOGS_KEY_PREFIX","EnvType":"string","EnvValue":"","EnvDescription":"Prefix for build logs","Example":"","Deprecated":"false"},{"Env":"CI_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"Node label selector for CI","Example":"","Deprecated":"false"},{"Env":"CI_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"Toleration key for CI","Example":"","Deprecated":"false"},{"Env":"CI_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"","EnvDescription":"Toleration value for CI","Example":"","Deprecated":"false"},{"Env":"CI_RUNNER_DOCKER_MTU_VALUE","EnvType":"int","EnvValue":"-1","EnvDescription":"this is to control the bytes of inofrmation passed in a network packet in ci-runner. default is -1 (defaults to the underlying node mtu value)","Example":"","Deprecated":"false"},{"Env":"CI_SUCCESS_AUTO_TRIGGER_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"this is to control the no of linked pipelines should be hanled in one go when a ci-success event of an parent ci is received","Example":"","Deprecated":"false"},{"Env":"CI_VOLUME_MOUNTS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"additional volume mount data for CI and JOB","Example":"","Deprecated":"false"},{"Env":"CI_WORKFLOW_EXECUTOR_TYPE","EnvType":"","EnvValue":"AWF","EnvDescription":"Executor type for CI(AWF,System)","Example":"","Deprecated":"false"},{"Env":"DEFAULT_ARTIFACT_KEY_LOCATION","EnvType":"string","EnvValue":"arsenal-v1/ci-artifacts","EnvDescription":"Key location for artifacts being created","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_BUCKET","EnvType":"string","EnvValue":"devtron-pro-ci-logs","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_KEY_PREFIX","EnvType":"string","EnvValue":"arsenal-v1","EnvDescription":"Bucket prefix for build logs","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET","EnvType":"string","EnvValue":"ci-caching","EnvDescription":"Bucket name for build cache","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET_REGION","EnvType":"string","EnvValue":"us-east-2","EnvDescription":"Build Cache bucket region","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_ARTIFACT_KEY_LOCATION","EnvType":"string","EnvValue":"","EnvDescription":"Bucket prefix for build cache","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_LOGS_BUCKET_REGION","EnvType":"string","EnvValue":"us-east-2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_NAMESPACE","EnvType":"string","EnvValue":"","EnvDescription":"Namespace for devtron stack","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_TIMEOUT","EnvType":"int64","EnvValue":"3600","EnvDescription":"Timeout for Pre/Post-Cd to be completed","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CI_IMAGE","EnvType":"string","EnvValue":"686244538589.dkr.ecr.us-east-2.amazonaws.com/cirunner:47","EnvDescription":"To pass the ci-runner image","Example":"","Deprecated":"false"},{"Env":"DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtron-ci","EnvDescription":"Timeout for CI to be completed","Example":"","Deprecated":"false"},{"Env":"DEFAULT_TARGET_PLATFORM","EnvType":"string","EnvValue":"","EnvDescription":"Default architecture for buildx","Example":"","Deprecated":"false"},{"Env":"DOCKER_BUILD_CACHE_PATH","EnvType":"string","EnvValue":"/var/lib/docker","EnvDescription":"Path to store cache of docker build (/var/lib/docker-\u003e for legacy docker build, /var/lib/devtron-\u003e for buildx)","Example":"","Deprecated":"false"},{"Env":"ENABLE_BUILD_CONTEXT","EnvType":"bool","EnvValue":"false","EnvDescription":"To Enable build context in Devtron.","Example":"","Deprecated":"false"},{"Env":"ENABLE_WORKFLOW_EXECUTION_STAGE","EnvType":"bool","EnvValue":"true","EnvDescription":"if enabled then we will display build stages separately for CI/Job/Pre-Post CD","Example":"true","Deprecated":"false"},{"Env":"EXTERNAL_BLOB_STORAGE_CM_NAME","EnvType":"string","EnvValue":"blob-storage-cm","EnvDescription":"name of the config map(contains bucket name, etc.) in external cluster when there is some operation related to external cluster, for example:-downloading cd artifact pushed in external cluster's env and we need to download from there, downloads ci logs pushed in external cluster's blob","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_BLOB_STORAGE_SECRET_NAME","EnvType":"string","EnvValue":"blob-storage-secret","EnvDescription":"name of the secret(contains password, accessId,passKeys, etc.) in external cluster when there is some operation related to external cluster, for example:-downloading cd artifact pushed in external cluster's env and we need to download from there, downloads ci logs pushed in external cluster's blob","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"This is an array of strings used when submitting a workflow for pre or post-CD execution. If the ","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"dedicated","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"ci","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_API_SECRET","EnvType":"string","EnvValue":"devtroncd-secret","EnvDescription":"External CI API secret.","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_PAYLOAD","EnvType":"string","EnvValue":"{\"ciProjectDetails\":[{\"gitRepository\":\"https://github.com/vikram1601/getting-started-nodejs.git\",\"checkoutPath\":\"./abc\",\"commitHash\":\"239077135f8cdeeccb7857e2851348f558cb53d3\",\"commitTime\":\"2022-10-30T20:00:00\",\"branch\":\"master\",\"message\":\"Update README.md\",\"author\":\"User Name \"}],\"dockerImage\":\"445808685819.dkr.ecr.us-east-2.amazonaws.com/orch:23907713-2\"}","EnvDescription":"External CI payload with project details.","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_WEB_HOOK_URL","EnvType":"string","EnvValue":"","EnvDescription":"default is {{HOST_URL}}/orchestrator/webhook/ext-ci. It is used for external ci.","Example":"","Deprecated":"false"},{"Env":"IGNORE_CM_CS_IN_CI_JOB","EnvType":"bool","EnvValue":"false","EnvDescription":"Ignore CM/CS in CI-pipeline as Job","Example":"","Deprecated":"false"},{"Env":"IMAGE_RETRY_COUNT","EnvType":"int","EnvValue":"0","EnvDescription":"push artifact(image) in ci retry count ","Example":"","Deprecated":"false"},{"Env":"IMAGE_RETRY_INTERVAL","EnvType":"int","EnvValue":"5","EnvDescription":"image retry interval takes value in seconds","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCANNER_ENDPOINT","EnvType":"string","EnvValue":"http://image-scanner-new-demo-devtroncd-service.devtroncd:80","EnvDescription":"Image-scanner micro-service URL","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_MAX_RETRIES","EnvType":"int","EnvValue":"3","EnvDescription":"Max retry count for image-scanning","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_RETRY_DELAY","EnvType":"int","EnvValue":"5","EnvDescription":"Delay for the image-scaning to start","Example":"","Deprecated":"false"},{"Env":"IN_APP_LOGGING_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"Used in case of argo workflow is enabled. If enabled logs push will be managed by us, else will be managed by argo workflow.","Example":"","Deprecated":"false"},{"Env":"MAX_CD_WORKFLOW_RUNNER_RETRIES","EnvType":"int","EnvValue":"0","EnvDescription":"Maximum time pre/post-cd-workflow create pod if it fails to complete","Example":"","Deprecated":"false"},{"Env":"MAX_CI_WORKFLOW_RETRIES","EnvType":"int","EnvValue":"0","EnvDescription":"Maximum time CI-workflow create pod if it fails to complete","Example":"","Deprecated":"false"},{"Env":"MODE","EnvType":"string","EnvValue":"DEV","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"localhost:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ORCH_HOST","EnvType":"string","EnvValue":"http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats","EnvDescription":"Orchestrator micro-service URL ","Example":"","Deprecated":"false"},{"Env":"ORCH_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"Orchestrator token","Example":"","Deprecated":"false"},{"Env":"PRE_CI_CACHE_PATH","EnvType":"string","EnvValue":"/devtroncd-cache","EnvDescription":"Cache path for Pre CI tasks","Example":"","Deprecated":"false"},{"Env":"SHOW_DOCKER_BUILD_ARGS","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable showing the args passed for CI in build logs","Example":"","Deprecated":"false"},{"Env":"SKIP_CI_JOB_BUILD_CACHE_PUSH_PULL","EnvType":"bool","EnvValue":"false","EnvDescription":"To skip cache Push/Pull for ci job","Example":"","Deprecated":"false"},{"Env":"SKIP_CREATING_ECR_REPO","EnvType":"bool","EnvValue":"false","EnvDescription":"By disabling this ECR repo won't get created if it's not available on ECR from build configuration","Example":"","Deprecated":"false"},{"Env":"TERMINATION_GRACE_PERIOD_SECS","EnvType":"int","EnvValue":"180","EnvDescription":"this is the time given to workflow pods to shutdown. (grace full termination time)","Example":"","Deprecated":"false"},{"Env":"USE_ARTIFACT_LISTING_QUERY_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"To use the V2 query for listing artifacts","Example":"","Deprecated":"false"},{"Env":"USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable blob storage in pre and post cd","Example":"","Deprecated":"false"},{"Env":"USE_BLOB_STORAGE_CONFIG_IN_CI_WORKFLOW","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable blob storage in pre and post ci","Example":"","Deprecated":"false"},{"Env":"USE_BUILDX","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable buildx feature globally","Example":"","Deprecated":"false"},{"Env":"USE_DOCKER_API_TO_GET_DIGEST","EnvType":"bool","EnvValue":"false","EnvDescription":"when user do not pass the digest then this flag controls , finding the image digest using docker API or not. if set to true we get the digest from docker API call else use docker pull command. [logic in ci-runner]","Example":"","Deprecated":"false"},{"Env":"USE_EXTERNAL_NODE","EnvType":"bool","EnvValue":"false","EnvDescription":"It is used in case of Pre/ Post Cd with run in application mode. If enabled the node lebels are read from EXTERNAL_CD_NODE_LABEL_SELECTOR else from CD_NODE_LABEL_SELECTOR MODE: if the vale is DEV, it will read the local kube config file or else from the cluser location.","Example":"","Deprecated":"false"},{"Env":"USE_IMAGE_TAG_FROM_GIT_PROVIDER_FOR_TAG_BASED_BUILD","EnvType":"bool","EnvValue":"false","EnvDescription":"To use the same tag in container image as that of git tag","Example":"","Deprecated":"false"},{"Env":"WF_CONTROLLER_INSTANCE_ID","EnvType":"string","EnvValue":"devtron-runner","EnvDescription":"Workflow controller instance ID.","Example":"","Deprecated":"false"},{"Env":"WORKFLOW_CACHE_CONFIG","EnvType":"string","EnvValue":"{}","EnvDescription":"flag is used to configure how Docker caches are handled during a CI/CD ","Example":"","Deprecated":"false"},{"Env":"WORKFLOW_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"ci-runner","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"DEVTRON","Fields":[{"Env":"-","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ADDITIONAL_NODE_GROUP_LABELS","EnvType":"","EnvValue":"","EnvDescription":"Add comma separated list of additional node group labels to default labels","Example":"karpenter.sh/nodepool,cloud.google.com/gke-nodepool","Deprecated":"false"},{"Env":"APP_SYNC_IMAGE","EnvType":"string","EnvValue":"quay.io/devtron/chart-sync:1227622d-132-3775","EnvDescription":"For the app sync image, this image will be used in app-manual sync job","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_JOB_RESOURCES_OBJ","EnvType":"string","EnvValue":"","EnvDescription":"To pass the resource of app sync","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"chart-sync","EnvDescription":"Service account to be used in app sync Job","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_SHUTDOWN_WAIT_DURATION","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_AUTO_SYNC_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"If enabled all argocd application will have auto sync enabled","Example":"","Deprecated":"false"},{"Env":"ARGO_GIT_COMMIT_RETRY_COUNT_ON_CONFLICT","EnvType":"int","EnvValue":"3","EnvDescription":"retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins)","Example":"","Deprecated":"false"},{"Env":"ARGO_GIT_COMMIT_RETRY_DELAY_ON_CONFLICT","EnvType":"int","EnvValue":"1","EnvDescription":"Delay on retrying the maifest commit the on gitops","Example":"","Deprecated":"false"},{"Env":"ARGO_REPO_REGISTER_RETRY_COUNT","EnvType":"int","EnvValue":"3","EnvDescription":"Argo app registration in argo retries on deployment","Example":"","Deprecated":"false"},{"Env":"ARGO_REPO_REGISTER_RETRY_DELAY","EnvType":"int","EnvValue":"10","EnvDescription":"Argo app registration in argo cd on deployment delay between retry","Example":"","Deprecated":"false"},{"Env":"ASYNC_BUILDX_CACHE_EXPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable async container image cache export","Example":"","Deprecated":"false"},{"Env":"BATCH_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"there is feature to get URL's of services/ingresses. so to extract those, we need to parse all the servcie and ingress objects of the application. this BATCH_SIZE flag controls the no of these objects get parsed in one go.","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_CACHE_MODE_MIN","EnvType":"bool","EnvValue":"false","EnvDescription":"To set build cache mode to minimum in buildx","Example":"","Deprecated":"false"},{"Env":"CD_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"Host for the devtron stack","Example":"","Deprecated":"false"},{"Env":"CD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_PORT","EnvType":"string","EnvValue":"8000","EnvDescription":"Port for pre/post-cd","Example":"","Deprecated":"false"},{"Env":"CExpirationTime","EnvType":"int","EnvValue":"600","EnvDescription":"Caching expiration time.","Example":"","Deprecated":"false"},{"Env":"CI_TRIGGER_CRON_TIME","EnvType":"int","EnvValue":"2","EnvDescription":"For image poll plugin","Example":"","Deprecated":"false"},{"Env":"CI_WORKFLOW_STATUS_UPDATE_CRON","EnvType":"string","EnvValue":"*/5 * * * *","EnvDescription":"Cron schedule for CI pipeline status","Example":"","Deprecated":"false"},{"Env":"CLI_CMD_TIMEOUT_GLOBAL_SECONDS","EnvType":"int","EnvValue":"0","EnvDescription":"Used in git cli opeartion timeout","Example":"","Deprecated":"false"},{"Env":"CLUSTER_STATUS_CRON_TIME","EnvType":"int","EnvValue":"15","EnvDescription":"Cron schedule for cluster status on resource browser","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_TIMEOUT","EnvType":"float64","EnvValue":"3600","EnvDescription":"Timeout for CI to be completed","Example":"","Deprecated":"false"},{"Env":"DEVTRON_BOM_URL","EnvType":"string","EnvValue":"https://raw.githubusercontent.com/devtron-labs/devtron/%s/charts/devtron/devtron-bom.yaml","EnvDescription":"Path to devtron-bom.yaml of devtron charts, used for module installation and devtron upgrade","Example":"","Deprecated":"false"},{"Env":"DEVTRON_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_DEX_SECRET_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"Namespace of dex secret","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_CHART_NAME","EnvType":"string","EnvValue":"devtron-operator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_NAME","EnvType":"string","EnvValue":"devtron","EnvDescription":"Name of the Devtron Helm release. ","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"Namespace of the Devtron Helm release","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_REPO_NAME","EnvType":"string","EnvValue":"devtron","EnvDescription":"Is used to install modules (stack manager)","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_REPO_URL","EnvType":"string","EnvValue":"https://helm.devtron.ai","EnvDescription":"Is used to install modules (stack manager)","Example":"","Deprecated":"false"},{"Env":"DEVTRON_INSTALLATION_TYPE","EnvType":"string","EnvValue":"","EnvDescription":"Devtron Installation type(EA/Full)","Example":"","Deprecated":"false"},{"Env":"DEVTRON_MODULES_IDENTIFIER_IN_HELM_VALUES","EnvType":"string","EnvValue":"installer.modules","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_SECRET_NAME","EnvType":"string","EnvValue":"devtron-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_VERSION_IDENTIFIER_IN_HELM_VALUES","EnvType":"string","EnvValue":"installer.release","EnvDescription":"devtron operator version identifier in helm values yaml","Example":"","Deprecated":"false"},{"Env":"DEX_CID","EnvType":"string","EnvValue":"example-app","EnvDescription":"dex client id ","Example":"","Deprecated":"false"},{"Env":"DEX_CLIENT_ID","EnvType":"string","EnvValue":"argo-cd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_CSTOREKEY","EnvType":"string","EnvValue":"","EnvDescription":"DEX CSTOREKEY.","Example":"","Deprecated":"false"},{"Env":"DEX_JWTKEY","EnvType":"string","EnvValue":"","EnvDescription":"DEX JWT key. ","Example":"","Deprecated":"false"},{"Env":"DEX_RURL","EnvType":"string","EnvValue":"http://127.0.0.1:8080/callback","EnvDescription":"Dex redirect URL(http://argocd-dex-server.devtroncd:8080/callback)","Example":"","Deprecated":"false"},{"Env":"DEX_SCOPES","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_SECRET","EnvType":"string","EnvValue":"","EnvDescription":"Dex secret","Example":"","Deprecated":"false"},{"Env":"DEX_URL","EnvType":"string","EnvValue":"","EnvDescription":"Dex service endpoint with dex path(http://argocd-dex-server.devtroncd:5556/dex)","Example":"","Deprecated":"false"},{"Env":"ECR_REPO_NAME_PREFIX","EnvType":"string","EnvValue":"test/","EnvDescription":"Prefix for ECR repo to be created in does not exist","Example":"","Deprecated":"false"},{"Env":"ENABLE_ASYNC_ARGO_CD_INSTALL_DEVTRON_CHART","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable async installation of gitops application","Example":"","Deprecated":"false"},{"Env":"ENABLE_ASYNC_INSTALL_DEVTRON_CHART","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable async installation of no-gitops application","Example":"","Deprecated":"false"},{"Env":"EPHEMERAL_SERVER_VERSION_REGEX","EnvType":"string","EnvValue":"v[1-9]\\.\\b(2[3-9]\\|[3-9][0-9])\\b.*","EnvDescription":"ephemeral containers support version regex that is compared with k8sServerVersion","Example":"","Deprecated":"false"},{"Env":"EVENT_URL","EnvType":"string","EnvValue":"http://localhost:3000/notify","EnvDescription":"Notifier service url","Example":"","Deprecated":"false"},{"Env":"EXECUTE_WIRE_NIL_CHECKER","EnvType":"bool","EnvValue":"false","EnvDescription":"checks for any nil pointer in wire.go","Example":"","Deprecated":"false"},{"Env":"EXPOSE_CI_METRICS","EnvType":"bool","EnvValue":"false","EnvDescription":"To expose CI metrics","Example":"","Deprecated":"false"},{"Env":"FEATURE_RESTART_WORKLOAD_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"restart workload retrieval batch size ","Example":"","Deprecated":"false"},{"Env":"FEATURE_RESTART_WORKLOAD_WORKER_POOL_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"restart workload retrieval pool size","Example":"","Deprecated":"false"},{"Env":"FORCE_SECURITY_SCANNING","EnvType":"bool","EnvValue":"false","EnvDescription":"By enabling this no one can disable image scaning on ci-pipeline from UI","Example":"","Deprecated":"false"},{"Env":"GITOPS_REPO_PREFIX","EnvType":"string","EnvValue":"","EnvDescription":"Prefix for Gitops repo being creation for argocd application","Example":"","Deprecated":"false"},{"Env":"GO_RUNTIME_ENV","EnvType":"string","EnvValue":"production","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"Host URL for the grafana dashboard","Example":"","Deprecated":"false"},{"Env":"GRAFANA_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"Namespace for grafana","Example":"","Deprecated":"false"},{"Env":"GRAFANA_ORG_ID","EnvType":"int","EnvValue":"2","EnvDescription":"Org ID for grafana for application metrics","Example":"","Deprecated":"false"},{"Env":"GRAFANA_PASSWORD","EnvType":"string","EnvValue":"prom-operator","EnvDescription":"Password for grafana dashboard","Example":"","Deprecated":"false"},{"Env":"GRAFANA_PORT","EnvType":"string","EnvValue":"8090","EnvDescription":"Port for grafana micro-service","Example":"","Deprecated":"false"},{"Env":"GRAFANA_URL","EnvType":"string","EnvValue":"","EnvDescription":"Host URL for the grafana dashboard","Example":"","Deprecated":"false"},{"Env":"GRAFANA_USERNAME","EnvType":"string","EnvValue":"admin","EnvDescription":"Username for grafana ","Example":"","Deprecated":"false"},{"Env":"HIDE_IMAGE_TAGGING_HARD_DELETE","EnvType":"bool","EnvValue":"false","EnvDescription":"Flag to hide the hard delete option in the image tagging service","Example":"","Deprecated":"false"},{"Env":"IGNORE_AUTOCOMPLETE_AUTH_CHECK","EnvType":"bool","EnvValue":"false","EnvDescription":"flag for ignoring auth check in autocomplete apis.","Example":"","Deprecated":"false"},{"Env":"INSTALLED_MODULES","EnvType":"","EnvValue":"","EnvDescription":"List of installed modules given in helm values/yaml are written in cm and used by devtron to know which modules are given","Example":"security.trivy,security.clair","Deprecated":"false"},{"Env":"INSTALLER_CRD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"namespace where Custom Resource Definitions get installed","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_GROUP_NAME","EnvType":"string","EnvValue":"installer.devtron.ai","EnvDescription":"Devtron installer CRD group name, partially deprecated.","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_RESOURCE","EnvType":"string","EnvValue":"installers","EnvDescription":"Devtron installer CRD resource name, partially deprecated","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_VERSION","EnvType":"string","EnvValue":"v1alpha1","EnvDescription":"version of the CRDs. default is v1alpha1","Example":"","Deprecated":"false"},{"Env":"IS_AIR_GAP_ENVIRONMENT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"JwtExpirationTime","EnvType":"int","EnvValue":"120","EnvDescription":"JWT expiration time.","Example":"","Deprecated":"false"},{"Env":"K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST","EnvType":"int","EnvValue":"25","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_IDLE_CONN_TIMEOUT","EnvType":"int","EnvValue":"300","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_KEEPALIVE","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_TIMEOUT","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TLS_HANDSHAKE_TIMEOUT","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE","EnvType":"int","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_SEND_MSG_SIZE","EnvType":"int","EnvValue":"4","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LENS_TIMEOUT","EnvType":"int","EnvValue":"0","EnvDescription":"Lens microservice timeout.","Example":"","Deprecated":"false"},{"Env":"LENS_URL","EnvType":"string","EnvValue":"http://lens-milandevtron-service:80","EnvDescription":"Lens micro-service URL","Example":"","Deprecated":"false"},{"Env":"LIMIT_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LIMIT_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOGGER_DEV_MODE","EnvType":"bool","EnvValue":"false","EnvDescription":"Enables a different logger theme.","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_SESSION_PER_USER","EnvType":"int","EnvValue":"5","EnvDescription":"max no of cluster terminal pods can be created by an user","Example":"","Deprecated":"false"},{"Env":"MODULE_METADATA_API_URL","EnvType":"string","EnvValue":"https://api.devtron.ai/module?name=%s","EnvDescription":"Modules list and meta info will be fetched from this server, that is central api server of devtron.","Example":"","Deprecated":"false"},{"Env":"MODULE_STATUS_HANDLING_CRON_DURATION_MIN","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NOTIFICATION_MEDIUM","EnvType":"NotificationMedium","EnvValue":"rest","EnvDescription":"notification medium","Example":"","Deprecated":"false"},{"Env":"OTEL_COLLECTOR_URL","EnvType":"string","EnvValue":"","EnvDescription":"Opentelemetry URL ","Example":"","Deprecated":"false"},{"Env":"PARALLELISM_LIMIT_FOR_TAG_PROCESSING","EnvType":"int","EnvValue":"","EnvDescription":"App manual sync job parallel tag processing count.","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PLUGIN_NAME","EnvType":"string","EnvValue":"Pull images from container repository","EnvDescription":"Handles image retrieval from a container repository and triggers subsequent CI processes upon detecting new images.Current default plugin name: Pull Images from Container Repository.","Example":"","Deprecated":"false"},{"Env":"PROPAGATE_EXTRA_LABELS","EnvType":"bool","EnvValue":"false","EnvDescription":"Add additional propagate labels like api.devtron.ai/appName, api.devtron.ai/envName, api.devtron.ai/project along with the user defined ones.","Example":"","Deprecated":"false"},{"Env":"PROXY_SERVICE_CONFIG","EnvType":"string","EnvValue":"{}","EnvDescription":"Proxy configuration for micro-service to be accessible on orhcestrator ingress","Example":"","Deprecated":"false"},{"Env":"REQ_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REQ_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RESTRICT_TERMINAL_ACCESS_FOR_NON_SUPER_USER","EnvType":"bool","EnvValue":"false","EnvDescription":"To restrict the cluster terminal from user having non-super admin acceess","Example":"","Deprecated":"false"},{"Env":"RUNTIME_CONFIG_LOCAL_DEV","EnvType":"LocalDevMode","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable scoped variable option","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_FORMAT","EnvType":"string","EnvValue":"@{{%s}}","EnvDescription":"Its a scope format for varialbe name.","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_HANDLE_PRIMITIVES","EnvType":"bool","EnvValue":"false","EnvDescription":"This describe should we handle primitives or not in scoped variable template parsing.","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_NAME_REGEX","EnvType":"string","EnvValue":"^[a-zA-Z][a-zA-Z0-9_-]{0,62}[a-zA-Z0-9]$","EnvDescription":"Regex for scoped variable name that must passed this regex.","Example":"","Deprecated":"false"},{"Env":"SOCKET_DISCONNECT_DELAY_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"The server closes a session when a client receiving connection have not been seen for a while.This delay is configured by this setting. By default the session is closed when a receiving connection wasn't seen for 5 seconds.","Example":"","Deprecated":"false"},{"Env":"SOCKET_HEARTBEAT_SECONDS","EnvType":"int","EnvValue":"25","EnvDescription":"In order to keep proxies and load balancers from closing long running http requests we need to pretend that the connection is active and send a heartbeat packet once in a while. This setting controls how often this is done. By default a heartbeat packet is sent every 25 seconds.","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SYSTEM_VAR_PREFIX","EnvType":"string","EnvValue":"DEVTRON_","EnvDescription":"Scoped variable prefix, variable name must have this prefix.","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"default","EnvDescription":"Cluster terminal default namespace","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_INACTIVE_DURATION_IN_MINS","EnvType":"int","EnvValue":"10","EnvDescription":"Timeout for cluster terminal to be inactive","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_STATUS_SYNC_In_SECS","EnvType":"int","EnvValue":"600","EnvDescription":"this is the time interval at which the status of the cluster terminal pod","Example":"","Deprecated":"false"},{"Env":"TEST_APP","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_LOG_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_PASSWORD","EnvType":"string","EnvValue":"postgrespw","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_PORT","EnvType":"string","EnvValue":"55000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_USER","EnvType":"string","EnvValue":"postgres","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TIMEOUT_FOR_FAILED_CI_BUILD","EnvType":"string","EnvValue":"15","EnvDescription":"Timeout for Failed CI build ","Example":"","Deprecated":"false"},{"Env":"TIMEOUT_IN_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"timeout to compute the urls from services and ingress objects of an application","Example":"","Deprecated":"false"},{"Env":"USER_SESSION_DURATION_SECONDS","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_ARTIFACT_LISTING_API_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"To use the V2 API for listing artifacts in Listing the images in pipeline","Example":"","Deprecated":"false"},{"Env":"USE_CUSTOM_HTTP_TRANSPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_GIT_CLI","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable git cli","Example":"","Deprecated":"false"},{"Env":"USE_RBAC_CREATION_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"To use the V2 for RBAC creation","Example":"","Deprecated":"false"},{"Env":"VARIABLE_CACHE_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"This is used to control caching of all the scope variables defined in the system.","Example":"","Deprecated":"false"},{"Env":"VARIABLE_EXPRESSION_REGEX","EnvType":"string","EnvValue":"@{{([^}]+)}}","EnvDescription":"Scoped variable expression regex","Example":"","Deprecated":"false"},{"Env":"WEBHOOK_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"If you want to continue using jenkins for CI then please provide this for authentication of requests","Example":"","Deprecated":"false"}]},{"Category":"GITOPS","Fields":[{"Env":"ACD_CM","EnvType":"string","EnvValue":"argocd-cm","EnvDescription":"Name of the argocd CM","Example":"","Deprecated":"false"},{"Env":"ACD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"To pass the argocd namespace","Example":"","Deprecated":"false"},{"Env":"ACD_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"Password for the Argocd (deprecated)","Example":"","Deprecated":"false"},{"Env":"ACD_USERNAME","EnvType":"string","EnvValue":"admin","EnvDescription":"User name for argocd","Example":"","Deprecated":"false"},{"Env":"GITOPS_SECRET_NAME","EnvType":"string","EnvValue":"devtron-gitops-secret","EnvDescription":"devtron-gitops-secret","Example":"","Deprecated":"false"},{"Env":"RESOURCE_LIST_FOR_REPLICAS","EnvType":"string","EnvValue":"Deployment,Rollout,StatefulSet,ReplicaSet","EnvDescription":"this holds the list of k8s resource names which support replicas key. this list used in hibernate/un hibernate process","Example":"","Deprecated":"false"},{"Env":"RESOURCE_LIST_FOR_REPLICAS_BATCH_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"this the batch size to control no of above resources can be parsed in one go to determine hibernate status","Example":"","Deprecated":"false"}]},{"Category":"INFRA_SETUP","Fields":[{"Env":"DASHBOARD_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"Dashboard micro-service URL","Example":"","Deprecated":"false"},{"Env":"DASHBOARD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"Dashboard micro-service namespace","Example":"","Deprecated":"false"},{"Env":"DASHBOARD_PORT","EnvType":"string","EnvValue":"3000","EnvDescription":"Port for dashboard micro-service","Example":"","Deprecated":"false"},{"Env":"DEX_HOST","EnvType":"string","EnvValue":"http://localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_PORT","EnvType":"string","EnvValue":"5556","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_PROTOCOL","EnvType":"string","EnvValue":"REST","EnvDescription":"Protocol to connect with git-sensor micro-service","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_TIMEOUT","EnvType":"int","EnvValue":"0","EnvDescription":"Timeout for getting response from the git-sensor","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_URL","EnvType":"string","EnvValue":"127.0.0.1:7070","EnvDescription":"git-sensor micro-service url ","Example":"","Deprecated":"false"},{"Env":"HELM_CLIENT_URL","EnvType":"string","EnvValue":"127.0.0.1:50051","EnvDescription":"Kubelink micro-service url ","Example":"","Deprecated":"false"}]},{"Category":"POSTGRES","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"Application name","Example":"","Deprecated":"false"},{"Env":"CASBIN_DATABASE","EnvType":"string","EnvValue":"casbin","EnvDescription":"Database for casbin","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"address of postgres service","Example":"postgresql-postgresql.devtroncd","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"postgres database to be made connection with","Example":"orchestrator, casbin, git_sensor, lens","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"{password}","EnvDescription":"password for postgres, associated with PG_USER","Example":"confidential ;)","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"port of postgresql service","Example":"5432","Deprecated":"false"},{"Env":"PG_READ_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"Time out for read operation in postgres","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"postgres","EnvDescription":"user for postgres","Example":"postgres","Deprecated":"false"},{"Env":"PG_WRITE_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"Time out for write operation in postgres","Example":"","Deprecated":"false"}]},{"Category":"RBAC","Fields":[{"Env":"ENFORCER_CACHE","EnvType":"bool","EnvValue":"false","EnvDescription":"To Enable enforcer cache.","Example":"","Deprecated":"false"},{"Env":"ENFORCER_CACHE_EXPIRATION_IN_SEC","EnvType":"int","EnvValue":"86400","EnvDescription":"Expiration time (in seconds) for enforcer cache. ","Example":"","Deprecated":"false"},{"Env":"ENFORCER_MAX_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"Maximum batch size for the enforcer.","Example":"","Deprecated":"false"},{"Env":"USE_CASBIN_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable casbin V2 API","Example":"","Deprecated":"false"}]}] \ No newline at end of file diff --git a/env_gen.md b/env_gen.md index 63d998590a..dcffdf8783 100644 --- a/env_gen.md +++ b/env_gen.md @@ -3,22 +3,22 @@ ## CD Related Environment Variables | Key | Type | Default Value | Description | Example | Deprecated | |-------|----------|-------------------|-------------------|-----------------------|------------------| - | ARGO_APP_MANUAL_SYNC_TIME | int |3 | | | false | - | CD_HELM_PIPELINE_STATUS_CRON_TIME | string |*/2 * * * * | | | false | - | CD_PIPELINE_STATUS_CRON_TIME | string |*/2 * * * * | | | false | - | CD_PIPELINE_STATUS_TIMEOUT_DURATION | string |20 | | | false | - | DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS | int |12 | | | false | - | DEVTRON_CHART_ARGO_CD_INSTALL_REQUEST_TIMEOUT | int |1 | | | false | - | DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT | int |6 | | | false | + | ARGO_APP_MANUAL_SYNC_TIME | int |3 | retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins) | | false | + | CD_HELM_PIPELINE_STATUS_CRON_TIME | string |*/2 * * * * | Cron time to check the pipeline status | | false | + | CD_PIPELINE_STATUS_CRON_TIME | string |*/2 * * * * | Cron time for CD pipeline status | | false | + | CD_PIPELINE_STATUS_TIMEOUT_DURATION | string |20 | Timeout for CD pipeline to get healthy | | false | + | DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS | int |12 | This flag is used to fetch the deployment status of the application. It retrieves the status of deployments that occurred between 12 hours and 10 minutes prior to the current time. It fetches non-terminal statuses. | | false | + | DEVTRON_CHART_ARGO_CD_INSTALL_REQUEST_TIMEOUT | int |1 | Context timeout for gitops concurrent async deployments | | false | + | DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT | int |6 | Context timeout for no gitops concurrent async deployments | | false | | EXPOSE_CD_METRICS | bool |false | | | false | | FEATURE_MIGRATE_ARGOCD_APPLICATION_ENABLE | bool |false | enable migration of external argocd application to devtron pipeline | | false | - | HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME | string |120 | | | false | - | IS_INTERNAL_USE | bool |true | | | false | + | HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME | string |120 | eligible time for checking helm app status periodically and update in db, value is in seconds., default is 120, if wfr is updated within configured time i.e. HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME then do not include for this cron cycle. | | false | + | IS_INTERNAL_USE | bool |true | If enabled then cd pipeline and helm apps will not need the deployment app type mandatorily. Couple this flag with HIDE_GITOPS_OR_HELM_OPTION (in Dashborad) and if gitops is configured and allowed for the env, pipeline/ helm app will gitops else no-gitops. | | false | | MIGRATE_DEPLOYMENT_CONFIG_DATA | bool |false | migrate deployment config data from charts table to deployment_config table | | false | - | PIPELINE_DEGRADED_TIME | string |10 | | | false | - | REVISION_HISTORY_LIMIT_DEVTRON_APP | int |1 | | | false | - | REVISION_HISTORY_LIMIT_EXTERNAL_HELM_APP | int |0 | | | false | - | REVISION_HISTORY_LIMIT_HELM_APP | int |1 | | | false | + | PIPELINE_DEGRADED_TIME | string |10 | Time to mark a pipeline degraded if not healthy in defined time | | false | + | REVISION_HISTORY_LIMIT_DEVTRON_APP | int |1 | Count for devtron application rivision history | | false | + | REVISION_HISTORY_LIMIT_EXTERNAL_HELM_APP | int |0 | Count for external helm application rivision history | | false | + | REVISION_HISTORY_LIMIT_HELM_APP | int |1 | To set the history limit for the helm app being deployed through devtron | | false | | REVISION_HISTORY_LIMIT_LINKED_HELM_APP | int |15 | | | false | | RUN_HELM_INSTALL_IN_ASYNC_MODE_HELM_APPS | bool |false | | | false | | SHOULD_CHECK_NAMESPACE_ON_CLONE | bool |false | should we check if namespace exists or not while cloning app | | false | @@ -28,97 +28,97 @@ ## CI_RUNNER Related Environment Variables | Key | Type | Default Value | Description | Example | Deprecated | |-------|----------|-------------------|-------------------|-----------------------|------------------| - | AZURE_ACCOUNT_KEY | string | | | | false | - | AZURE_ACCOUNT_NAME | string | | | | false | - | AZURE_BLOB_CONTAINER_CI_CACHE | string | | | | false | - | AZURE_BLOB_CONTAINER_CI_LOG | string | | | | false | - | AZURE_GATEWAY_CONNECTION_INSECURE | bool |true | | | false | - | AZURE_GATEWAY_URL | string |http://devtron-minio.devtroncd:9000 | | | false | - | BASE_LOG_LOCATION_PATH | string |/home/devtron/ | | | false | - | BLOB_STORAGE_GCP_CREDENTIALS_JSON | string | | | | false | - | BLOB_STORAGE_PROVIDER | |S3 | | | false | - | BLOB_STORAGE_S3_ACCESS_KEY | string | | | | false | - | BLOB_STORAGE_S3_BUCKET_VERSIONED | bool |true | | | false | - | BLOB_STORAGE_S3_ENDPOINT | string | | | | false | - | BLOB_STORAGE_S3_ENDPOINT_INSECURE | bool |false | | | false | - | BLOB_STORAGE_S3_SECRET_KEY | string | | | | false | - | BUILDX_CACHE_PATH | string |/var/lib/devtron/buildx | | | false | - | BUILDX_K8S_DRIVER_OPTIONS | string | | | | false | - | BUILDX_PROVENANCE_MODE | string | | | | false | - | BUILD_LOG_TTL_VALUE_IN_SECS | int |3600 | | | false | - | CACHE_LIMIT | int64 |5000000000 | | | false | - | CD_DEFAULT_ADDRESS_POOL_BASE_CIDR | string | | | | false | - | CD_DEFAULT_ADDRESS_POOL_SIZE | int | | | | false | - | CD_LIMIT_CI_CPU | string |0.5 | | | false | - | CD_LIMIT_CI_MEM | string |3G | | | false | - | CD_NODE_LABEL_SELECTOR | | | | | false | - | CD_NODE_TAINTS_KEY | string |dedicated | | | false | - | CD_NODE_TAINTS_VALUE | string |ci | | | false | - | CD_REQ_CI_CPU | string |0.5 | | | false | - | CD_REQ_CI_MEM | string |3G | | | false | - | CD_WORKFLOW_EXECUTOR_TYPE | |AWF | | | false | - | CD_WORKFLOW_SERVICE_ACCOUNT | string |cd-runner | | | false | - | CI_DEFAULT_ADDRESS_POOL_BASE_CIDR | string | | | | false | - | CI_DEFAULT_ADDRESS_POOL_SIZE | int | | | | false | - | CI_IGNORE_DOCKER_CACHE | bool | | | | false | - | CI_LOGS_KEY_PREFIX | string | | | | false | - | CI_NODE_LABEL_SELECTOR | | | | | false | - | CI_NODE_TAINTS_KEY | string | | | | false | - | CI_NODE_TAINTS_VALUE | string | | | | false | - | CI_RUNNER_DOCKER_MTU_VALUE | int |-1 | | | false | - | CI_SUCCESS_AUTO_TRIGGER_BATCH_SIZE | int |1 | | | false | - | CI_VOLUME_MOUNTS_JSON | string | | | | false | - | CI_WORKFLOW_EXECUTOR_TYPE | |AWF | | | false | - | DEFAULT_ARTIFACT_KEY_LOCATION | string |arsenal-v1/ci-artifacts | | | false | + | AZURE_ACCOUNT_KEY | string | | If blob storage is bieng used of azure then pass the secret key to access the bucket | | false | + | AZURE_ACCOUNT_NAME | string | | Account name for azure blob storage | | false | + | AZURE_BLOB_CONTAINER_CI_CACHE | string | | Cache bucket name for azure blob storage | | false | + | AZURE_BLOB_CONTAINER_CI_LOG | string | | Log bucket for azure blob storage | | false | + | AZURE_GATEWAY_CONNECTION_INSECURE | bool |true | Azure gateway connection allows insecure if true | | false | + | AZURE_GATEWAY_URL | string |http://devtron-minio.devtroncd:9000 | Sent to CI runner for blob | | false | + | BASE_LOG_LOCATION_PATH | string |/home/devtron/ | Used to store, download logs of ci workflow, artifact | | false | + | BLOB_STORAGE_GCP_CREDENTIALS_JSON | string | | GCP cred json for GCS blob storage | | false | + | BLOB_STORAGE_PROVIDER | |S3 | Blob storage provider name(AWS/GCP/Azure) | | false | + | BLOB_STORAGE_S3_ACCESS_KEY | string | | S3 access key for s3 blob storage | | false | + | BLOB_STORAGE_S3_BUCKET_VERSIONED | bool |true | To enable buctet versioning for blob storage | | false | + | BLOB_STORAGE_S3_ENDPOINT | string | | S3 endpoint URL for s3 blob storage | | false | + | BLOB_STORAGE_S3_ENDPOINT_INSECURE | bool |false | To use insecure s3 endpoint | | false | + | BLOB_STORAGE_S3_SECRET_KEY | string | | Secret key for s3 blob storage | | false | + | BUILDX_CACHE_PATH | string |/var/lib/devtron/buildx | Path for the buildx cache | | false | + | BUILDX_K8S_DRIVER_OPTIONS | string | | To enable the k8s driver and pass args for k8s driver in buildx | | false | + | BUILDX_PROVENANCE_MODE | string | | provinance is set to true by default by docker. this will add some build related data in generated build manifest.it also adds some unknown:unknown key:value pair which may not be compatible by some container registries. with buildx k8s driver , provinenance=true is causing issue when push manifest to quay registry, so setting it to false | | false | + | BUILD_LOG_TTL_VALUE_IN_SECS | int |3600 | This is the time that the pods of ci/pre-cd/post-cd live after completion state. | | false | + | CACHE_LIMIT | int64 |5000000000 | Cache limit. | | false | + | CD_DEFAULT_ADDRESS_POOL_BASE_CIDR | string | | To pass the IP cidr for Pre/Post cd | | false | + | CD_DEFAULT_ADDRESS_POOL_SIZE | int | | The subnet size to allocate from the base pool for CD | | false | + | CD_LIMIT_CI_CPU | string |0.5 | CPU Resource Limit Pre/Post CD | | false | + | CD_LIMIT_CI_MEM | string |3G | Memory Resource Limit Pre/Post CD | | false | + | CD_NODE_LABEL_SELECTOR | | | Node label selector for Pre/Post CD | | false | + | CD_NODE_TAINTS_KEY | string |dedicated | Toleration key for Pre/Post CD | | false | + | CD_NODE_TAINTS_VALUE | string |ci | Toleration value for Pre/Post CD | | false | + | CD_REQ_CI_CPU | string |0.5 | CPU Resource Rquest Pre/Post CD | | false | + | CD_REQ_CI_MEM | string |3G | Memory Resource Rquest Pre/Post CD | | false | + | CD_WORKFLOW_EXECUTOR_TYPE | |AWF | Executor type for Pre/Post CD(AWF,System) | | false | + | CD_WORKFLOW_SERVICE_ACCOUNT | string |cd-runner | Service account to be used in Pre/Post CD pod | | false | + | CI_DEFAULT_ADDRESS_POOL_BASE_CIDR | string | | To pass the IP cidr for CI | | false | + | CI_DEFAULT_ADDRESS_POOL_SIZE | int | | The subnet size to allocate from the base pool for CI | | false | + | CI_IGNORE_DOCKER_CACHE | bool | | Ignoring docker cache | | false | + | CI_LOGS_KEY_PREFIX | string | | Prefix for build logs | | false | + | CI_NODE_LABEL_SELECTOR | | | Node label selector for CI | | false | + | CI_NODE_TAINTS_KEY | string | | Toleration key for CI | | false | + | CI_NODE_TAINTS_VALUE | string | | Toleration value for CI | | false | + | CI_RUNNER_DOCKER_MTU_VALUE | int |-1 | this is to control the bytes of inofrmation passed in a network packet in ci-runner. default is -1 (defaults to the underlying node mtu value) | | false | + | CI_SUCCESS_AUTO_TRIGGER_BATCH_SIZE | int |1 | this is to control the no of linked pipelines should be hanled in one go when a ci-success event of an parent ci is received | | false | + | CI_VOLUME_MOUNTS_JSON | string | | additional volume mount data for CI and JOB | | false | + | CI_WORKFLOW_EXECUTOR_TYPE | |AWF | Executor type for CI(AWF,System) | | false | + | DEFAULT_ARTIFACT_KEY_LOCATION | string |arsenal-v1/ci-artifacts | Key location for artifacts being created | | false | | DEFAULT_BUILD_LOGS_BUCKET | string |devtron-pro-ci-logs | | | false | - | DEFAULT_BUILD_LOGS_KEY_PREFIX | string |arsenal-v1 | | | false | - | DEFAULT_CACHE_BUCKET | string |ci-caching | | | false | - | DEFAULT_CACHE_BUCKET_REGION | string |us-east-2 | | | false | - | DEFAULT_CD_ARTIFACT_KEY_LOCATION | string | | | | false | + | DEFAULT_BUILD_LOGS_KEY_PREFIX | string |arsenal-v1 | Bucket prefix for build logs | | false | + | DEFAULT_CACHE_BUCKET | string |ci-caching | Bucket name for build cache | | false | + | DEFAULT_CACHE_BUCKET_REGION | string |us-east-2 | Build Cache bucket region | | false | + | DEFAULT_CD_ARTIFACT_KEY_LOCATION | string | | Bucket prefix for build cache | | false | | DEFAULT_CD_LOGS_BUCKET_REGION | string |us-east-2 | | | false | - | DEFAULT_CD_NAMESPACE | string | | | | false | - | DEFAULT_CD_TIMEOUT | int64 |3600 | | | false | - | DEFAULT_CI_IMAGE | string |686244538589.dkr.ecr.us-east-2.amazonaws.com/cirunner:47 | | | false | - | DEFAULT_NAMESPACE | string |devtron-ci | | | false | - | DEFAULT_TARGET_PLATFORM | string | | | | false | - | DOCKER_BUILD_CACHE_PATH | string |/var/lib/docker | | | false | - | ENABLE_BUILD_CONTEXT | bool |false | | | false | + | DEFAULT_CD_NAMESPACE | string | | Namespace for devtron stack | | false | + | DEFAULT_CD_TIMEOUT | int64 |3600 | Timeout for Pre/Post-Cd to be completed | | false | + | DEFAULT_CI_IMAGE | string |686244538589.dkr.ecr.us-east-2.amazonaws.com/cirunner:47 | To pass the ci-runner image | | false | + | DEFAULT_NAMESPACE | string |devtron-ci | Timeout for CI to be completed | | false | + | DEFAULT_TARGET_PLATFORM | string | | Default architecture for buildx | | false | + | DOCKER_BUILD_CACHE_PATH | string |/var/lib/docker | Path to store cache of docker build (/var/lib/docker-> for legacy docker build, /var/lib/devtron-> for buildx) | | false | + | ENABLE_BUILD_CONTEXT | bool |false | To Enable build context in Devtron. | | false | | ENABLE_WORKFLOW_EXECUTION_STAGE | bool |true | if enabled then we will display build stages separately for CI/Job/Pre-Post CD | true | false | - | EXTERNAL_BLOB_STORAGE_CM_NAME | string |blob-storage-cm | | | false | - | EXTERNAL_BLOB_STORAGE_SECRET_NAME | string |blob-storage-secret | | | false | - | EXTERNAL_CD_NODE_LABEL_SELECTOR | | | | | false | + | EXTERNAL_BLOB_STORAGE_CM_NAME | string |blob-storage-cm | name of the config map(contains bucket name, etc.) in external cluster when there is some operation related to external cluster, for example:-downloading cd artifact pushed in external cluster's env and we need to download from there, downloads ci logs pushed in external cluster's blob | | false | + | EXTERNAL_BLOB_STORAGE_SECRET_NAME | string |blob-storage-secret | name of the secret(contains password, accessId,passKeys, etc.) in external cluster when there is some operation related to external cluster, for example:-downloading cd artifact pushed in external cluster's env and we need to download from there, downloads ci logs pushed in external cluster's blob | | false | + | EXTERNAL_CD_NODE_LABEL_SELECTOR | | | This is an array of strings used when submitting a workflow for pre or post-CD execution. If the | | false | | EXTERNAL_CD_NODE_TAINTS_KEY | string |dedicated | | | false | | EXTERNAL_CD_NODE_TAINTS_VALUE | string |ci | | | false | - | EXTERNAL_CI_API_SECRET | string |devtroncd-secret | | | false | - | EXTERNAL_CI_PAYLOAD | string |{"ciProjectDetails":[{"gitRepository":"https://github.com/vikram1601/getting-started-nodejs.git","checkoutPath":"./abc","commitHash":"239077135f8cdeeccb7857e2851348f558cb53d3","commitTime":"2022-10-30T20:00:00","branch":"master","message":"Update README.md","author":"User Name "}],"dockerImage":"445808685819.dkr.ecr.us-east-2.amazonaws.com/orch:23907713-2"} | | | false | - | EXTERNAL_CI_WEB_HOOK_URL | string | | | | false | - | IGNORE_CM_CS_IN_CI_JOB | bool |false | | | false | - | IMAGE_RETRY_COUNT | int |0 | | | false | - | IMAGE_RETRY_INTERVAL | int |5 | | | false | - | IMAGE_SCANNER_ENDPOINT | string |http://image-scanner-new-demo-devtroncd-service.devtroncd:80 | | | false | - | IMAGE_SCAN_MAX_RETRIES | int |3 | | | false | - | IMAGE_SCAN_RETRY_DELAY | int |5 | | | false | - | IN_APP_LOGGING_ENABLED | bool |false | | | false | - | MAX_CD_WORKFLOW_RUNNER_RETRIES | int |0 | | | false | - | MAX_CI_WORKFLOW_RETRIES | int |0 | | | false | + | EXTERNAL_CI_API_SECRET | string |devtroncd-secret | External CI API secret. | | false | + | EXTERNAL_CI_PAYLOAD | string |{"ciProjectDetails":[{"gitRepository":"https://github.com/vikram1601/getting-started-nodejs.git","checkoutPath":"./abc","commitHash":"239077135f8cdeeccb7857e2851348f558cb53d3","commitTime":"2022-10-30T20:00:00","branch":"master","message":"Update README.md","author":"User Name "}],"dockerImage":"445808685819.dkr.ecr.us-east-2.amazonaws.com/orch:23907713-2"} | External CI payload with project details. | | false | + | EXTERNAL_CI_WEB_HOOK_URL | string | | default is {{HOST_URL}}/orchestrator/webhook/ext-ci. It is used for external ci. | | false | + | IGNORE_CM_CS_IN_CI_JOB | bool |false | Ignore CM/CS in CI-pipeline as Job | | false | + | IMAGE_RETRY_COUNT | int |0 | push artifact(image) in ci retry count | | false | + | IMAGE_RETRY_INTERVAL | int |5 | image retry interval takes value in seconds | | false | + | IMAGE_SCANNER_ENDPOINT | string |http://image-scanner-new-demo-devtroncd-service.devtroncd:80 | Image-scanner micro-service URL | | false | + | IMAGE_SCAN_MAX_RETRIES | int |3 | Max retry count for image-scanning | | false | + | IMAGE_SCAN_RETRY_DELAY | int |5 | Delay for the image-scaning to start | | false | + | IN_APP_LOGGING_ENABLED | bool |false | Used in case of argo workflow is enabled. If enabled logs push will be managed by us, else will be managed by argo workflow. | | false | + | MAX_CD_WORKFLOW_RUNNER_RETRIES | int |0 | Maximum time pre/post-cd-workflow create pod if it fails to complete | | false | + | MAX_CI_WORKFLOW_RETRIES | int |0 | Maximum time CI-workflow create pod if it fails to complete | | false | | MODE | string |DEV | | | false | | NATS_SERVER_HOST | string |localhost:4222 | | | false | - | ORCH_HOST | string |http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats | | | false | - | ORCH_TOKEN | string | | | | false | - | PRE_CI_CACHE_PATH | string |/devtroncd-cache | | | false | - | SHOW_DOCKER_BUILD_ARGS | bool |true | | | false | - | SKIP_CI_JOB_BUILD_CACHE_PUSH_PULL | bool |false | | | false | - | SKIP_CREATING_ECR_REPO | bool |false | | | false | - | TERMINATION_GRACE_PERIOD_SECS | int |180 | | | false | - | USE_ARTIFACT_LISTING_QUERY_V2 | bool |true | | | false | - | USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW | bool |true | | | false | - | USE_BLOB_STORAGE_CONFIG_IN_CI_WORKFLOW | bool |true | | | false | - | USE_BUILDX | bool |false | | | false | - | USE_DOCKER_API_TO_GET_DIGEST | bool |false | | | false | - | USE_EXTERNAL_NODE | bool |false | | | false | - | USE_IMAGE_TAG_FROM_GIT_PROVIDER_FOR_TAG_BASED_BUILD | bool |false | | | false | - | WF_CONTROLLER_INSTANCE_ID | string |devtron-runner | | | false | - | WORKFLOW_CACHE_CONFIG | string |{} | | | false | + | ORCH_HOST | string |http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats | Orchestrator micro-service URL | | false | + | ORCH_TOKEN | string | | Orchestrator token | | false | + | PRE_CI_CACHE_PATH | string |/devtroncd-cache | Cache path for Pre CI tasks | | false | + | SHOW_DOCKER_BUILD_ARGS | bool |true | To enable showing the args passed for CI in build logs | | false | + | SKIP_CI_JOB_BUILD_CACHE_PUSH_PULL | bool |false | To skip cache Push/Pull for ci job | | false | + | SKIP_CREATING_ECR_REPO | bool |false | By disabling this ECR repo won't get created if it's not available on ECR from build configuration | | false | + | TERMINATION_GRACE_PERIOD_SECS | int |180 | this is the time given to workflow pods to shutdown. (grace full termination time) | | false | + | USE_ARTIFACT_LISTING_QUERY_V2 | bool |true | To use the V2 query for listing artifacts | | false | + | USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW | bool |true | To enable blob storage in pre and post cd | | false | + | USE_BLOB_STORAGE_CONFIG_IN_CI_WORKFLOW | bool |true | To enable blob storage in pre and post ci | | false | + | USE_BUILDX | bool |false | To enable buildx feature globally | | false | + | USE_DOCKER_API_TO_GET_DIGEST | bool |false | when user do not pass the digest then this flag controls , finding the image digest using docker API or not. if set to true we get the digest from docker API call else use docker pull command. [logic in ci-runner] | | false | + | USE_EXTERNAL_NODE | bool |false | It is used in case of Pre/ Post Cd with run in application mode. If enabled the node lebels are read from EXTERNAL_CD_NODE_LABEL_SELECTOR else from CD_NODE_LABEL_SELECTOR MODE: if the vale is DEV, it will read the local kube config file or else from the cluser location. | | false | + | USE_IMAGE_TAG_FROM_GIT_PROVIDER_FOR_TAG_BASED_BUILD | bool |false | To use the same tag in container image as that of git tag | | false | + | WF_CONTROLLER_INSTANCE_ID | string |devtron-runner | Workflow controller instance ID. | | false | + | WORKFLOW_CACHE_CONFIG | string |{} | flag is used to configure how Docker caches are handled during a CI/CD | | false | | WORKFLOW_SERVICE_ACCOUNT | string |ci-runner | | | false | @@ -127,78 +127,78 @@ |-------|----------|-------------------|-------------------|-----------------------|------------------| | - | | | | | false | | ADDITIONAL_NODE_GROUP_LABELS | | | Add comma separated list of additional node group labels to default labels | karpenter.sh/nodepool,cloud.google.com/gke-nodepool | false | - | APP_SYNC_IMAGE | string |quay.io/devtron/chart-sync:1227622d-132-3775 | | | false | - | APP_SYNC_JOB_RESOURCES_OBJ | string | | | | false | - | APP_SYNC_SERVICE_ACCOUNT | string |chart-sync | | | false | + | APP_SYNC_IMAGE | string |quay.io/devtron/chart-sync:1227622d-132-3775 | For the app sync image, this image will be used in app-manual sync job | | false | + | APP_SYNC_JOB_RESOURCES_OBJ | string | | To pass the resource of app sync | | false | + | APP_SYNC_SERVICE_ACCOUNT | string |chart-sync | Service account to be used in app sync Job | | false | | APP_SYNC_SHUTDOWN_WAIT_DURATION | int |120 | | | false | - | ARGO_AUTO_SYNC_ENABLED | bool |true | | | false | - | ARGO_GIT_COMMIT_RETRY_COUNT_ON_CONFLICT | int |3 | | | false | - | ARGO_GIT_COMMIT_RETRY_DELAY_ON_CONFLICT | int |1 | | | false | - | ARGO_REPO_REGISTER_RETRY_COUNT | int |3 | | | false | - | ARGO_REPO_REGISTER_RETRY_DELAY | int |10 | | | false | - | ASYNC_BUILDX_CACHE_EXPORT | bool |false | | | false | - | BATCH_SIZE | int |5 | | | false | + | ARGO_AUTO_SYNC_ENABLED | bool |true | If enabled all argocd application will have auto sync enabled | | false | + | ARGO_GIT_COMMIT_RETRY_COUNT_ON_CONFLICT | int |3 | retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins) | | false | + | ARGO_GIT_COMMIT_RETRY_DELAY_ON_CONFLICT | int |1 | Delay on retrying the maifest commit the on gitops | | false | + | ARGO_REPO_REGISTER_RETRY_COUNT | int |3 | Argo app registration in argo retries on deployment | | false | + | ARGO_REPO_REGISTER_RETRY_DELAY | int |10 | Argo app registration in argo cd on deployment delay between retry | | false | + | ASYNC_BUILDX_CACHE_EXPORT | bool |false | To enable async container image cache export | | false | + | BATCH_SIZE | int |5 | there is feature to get URL's of services/ingresses. so to extract those, we need to parse all the servcie and ingress objects of the application. this BATCH_SIZE flag controls the no of these objects get parsed in one go. | | false | | BLOB_STORAGE_ENABLED | bool |false | | | false | - | BUILDX_CACHE_MODE_MIN | bool |false | | | false | - | CD_HOST | string |localhost | | | false | + | BUILDX_CACHE_MODE_MIN | bool |false | To set build cache mode to minimum in buildx | | false | + | CD_HOST | string |localhost | Host for the devtron stack | | false | | CD_NAMESPACE | string |devtroncd | | | false | - | CD_PORT | string |8000 | | | false | - | CExpirationTime | int |600 | | | false | - | CI_TRIGGER_CRON_TIME | int |2 | | | false | - | CI_WORKFLOW_STATUS_UPDATE_CRON | string |*/5 * * * * | | | false | - | CLI_CMD_TIMEOUT_GLOBAL_SECONDS | int |0 | | | false | - | CLUSTER_STATUS_CRON_TIME | int |15 | | | false | + | CD_PORT | string |8000 | Port for pre/post-cd | | false | + | CExpirationTime | int |600 | Caching expiration time. | | false | + | CI_TRIGGER_CRON_TIME | int |2 | For image poll plugin | | false | + | CI_WORKFLOW_STATUS_UPDATE_CRON | string |*/5 * * * * | Cron schedule for CI pipeline status | | false | + | CLI_CMD_TIMEOUT_GLOBAL_SECONDS | int |0 | Used in git cli opeartion timeout | | false | + | CLUSTER_STATUS_CRON_TIME | int |15 | Cron schedule for cluster status on resource browser | | false | | CONSUMER_CONFIG_JSON | string | | | | false | | DEFAULT_LOG_TIME_LIMIT | int64 |1 | | | false | - | DEFAULT_TIMEOUT | float64 |3600 | | | false | - | DEVTRON_BOM_URL | string |https://raw.githubusercontent.com/devtron-labs/devtron/%s/charts/devtron/devtron-bom.yaml | | | false | + | DEFAULT_TIMEOUT | float64 |3600 | Timeout for CI to be completed | | false | + | DEVTRON_BOM_URL | string |https://raw.githubusercontent.com/devtron-labs/devtron/%s/charts/devtron/devtron-bom.yaml | Path to devtron-bom.yaml of devtron charts, used for module installation and devtron upgrade | | false | | DEVTRON_DEFAULT_NAMESPACE | string |devtroncd | | | false | - | DEVTRON_DEX_SECRET_NAMESPACE | string |devtroncd | | | false | + | DEVTRON_DEX_SECRET_NAMESPACE | string |devtroncd | Namespace of dex secret | | false | | DEVTRON_HELM_RELEASE_CHART_NAME | string |devtron-operator | | | false | - | DEVTRON_HELM_RELEASE_NAME | string |devtron | | | false | - | DEVTRON_HELM_RELEASE_NAMESPACE | string |devtroncd | | | false | - | DEVTRON_HELM_REPO_NAME | string |devtron | | | false | - | DEVTRON_HELM_REPO_URL | string |https://helm.devtron.ai | | | false | - | DEVTRON_INSTALLATION_TYPE | string | | | | false | + | DEVTRON_HELM_RELEASE_NAME | string |devtron | Name of the Devtron Helm release. | | false | + | DEVTRON_HELM_RELEASE_NAMESPACE | string |devtroncd | Namespace of the Devtron Helm release | | false | + | DEVTRON_HELM_REPO_NAME | string |devtron | Is used to install modules (stack manager) | | false | + | DEVTRON_HELM_REPO_URL | string |https://helm.devtron.ai | Is used to install modules (stack manager) | | false | + | DEVTRON_INSTALLATION_TYPE | string | | Devtron Installation type(EA/Full) | | false | | DEVTRON_MODULES_IDENTIFIER_IN_HELM_VALUES | string |installer.modules | | | false | | DEVTRON_SECRET_NAME | string |devtron-secret | | | false | - | DEVTRON_VERSION_IDENTIFIER_IN_HELM_VALUES | string |installer.release | | | false | - | DEX_CID | string |example-app | | | false | + | DEVTRON_VERSION_IDENTIFIER_IN_HELM_VALUES | string |installer.release | devtron operator version identifier in helm values yaml | | false | + | DEX_CID | string |example-app | dex client id | | false | | DEX_CLIENT_ID | string |argo-cd | | | false | - | DEX_CSTOREKEY | string | | | | false | - | DEX_JWTKEY | string | | | | false | - | DEX_RURL | string |http://127.0.0.1:8080/callback | | | false | + | DEX_CSTOREKEY | string | | DEX CSTOREKEY. | | false | + | DEX_JWTKEY | string | | DEX JWT key. | | false | + | DEX_RURL | string |http://127.0.0.1:8080/callback | Dex redirect URL(http://argocd-dex-server.devtroncd:8080/callback) | | false | | DEX_SCOPES | | | | | false | - | DEX_SECRET | string | | | | false | - | DEX_URL | string | | | | false | - | ECR_REPO_NAME_PREFIX | string |test/ | | | false | - | ENABLE_ASYNC_ARGO_CD_INSTALL_DEVTRON_CHART | bool |false | | | false | - | ENABLE_ASYNC_INSTALL_DEVTRON_CHART | bool |false | | | false | - | EPHEMERAL_SERVER_VERSION_REGEX | string |v[1-9]\.\b(2[3-9]\|[3-9][0-9])\b.* | | | false | - | EVENT_URL | string |http://localhost:3000/notify | | | false | - | EXECUTE_WIRE_NIL_CHECKER | bool |false | | | false | - | EXPOSE_CI_METRICS | bool |false | | | false | - | FEATURE_RESTART_WORKLOAD_BATCH_SIZE | int |1 | | | false | - | FEATURE_RESTART_WORKLOAD_WORKER_POOL_SIZE | int |5 | | | false | - | FORCE_SECURITY_SCANNING | bool |false | | | false | - | GITOPS_REPO_PREFIX | string | | | | false | + | DEX_SECRET | string | | Dex secret | | false | + | DEX_URL | string | | Dex service endpoint with dex path(http://argocd-dex-server.devtroncd:5556/dex) | | false | + | ECR_REPO_NAME_PREFIX | string |test/ | Prefix for ECR repo to be created in does not exist | | false | + | ENABLE_ASYNC_ARGO_CD_INSTALL_DEVTRON_CHART | bool |false | To enable async installation of gitops application | | false | + | ENABLE_ASYNC_INSTALL_DEVTRON_CHART | bool |false | To enable async installation of no-gitops application | | false | + | EPHEMERAL_SERVER_VERSION_REGEX | string |v[1-9]\.\b(2[3-9]\|[3-9][0-9])\b.* | ephemeral containers support version regex that is compared with k8sServerVersion | | false | + | EVENT_URL | string |http://localhost:3000/notify | Notifier service url | | false | + | EXECUTE_WIRE_NIL_CHECKER | bool |false | checks for any nil pointer in wire.go | | false | + | EXPOSE_CI_METRICS | bool |false | To expose CI metrics | | false | + | FEATURE_RESTART_WORKLOAD_BATCH_SIZE | int |1 | restart workload retrieval batch size | | false | + | FEATURE_RESTART_WORKLOAD_WORKER_POOL_SIZE | int |5 | restart workload retrieval pool size | | false | + | FORCE_SECURITY_SCANNING | bool |false | By enabling this no one can disable image scaning on ci-pipeline from UI | | false | + | GITOPS_REPO_PREFIX | string | | Prefix for Gitops repo being creation for argocd application | | false | | GO_RUNTIME_ENV | string |production | | | false | - | GRAFANA_HOST | string |localhost | | | false | - | GRAFANA_NAMESPACE | string |devtroncd | | | false | - | GRAFANA_ORG_ID | int |2 | | | false | - | GRAFANA_PASSWORD | string |prom-operator | | | false | - | GRAFANA_PORT | string |8090 | | | false | - | GRAFANA_URL | string | | | | false | - | GRAFANA_USERNAME | string |admin | | | false | - | HIDE_IMAGE_TAGGING_HARD_DELETE | bool |false | | | false | - | IGNORE_AUTOCOMPLETE_AUTH_CHECK | bool |false | | | false | + | GRAFANA_HOST | string |localhost | Host URL for the grafana dashboard | | false | + | GRAFANA_NAMESPACE | string |devtroncd | Namespace for grafana | | false | + | GRAFANA_ORG_ID | int |2 | Org ID for grafana for application metrics | | false | + | GRAFANA_PASSWORD | string |prom-operator | Password for grafana dashboard | | false | + | GRAFANA_PORT | string |8090 | Port for grafana micro-service | | false | + | GRAFANA_URL | string | | Host URL for the grafana dashboard | | false | + | GRAFANA_USERNAME | string |admin | Username for grafana | | false | + | HIDE_IMAGE_TAGGING_HARD_DELETE | bool |false | Flag to hide the hard delete option in the image tagging service | | false | + | IGNORE_AUTOCOMPLETE_AUTH_CHECK | bool |false | flag for ignoring auth check in autocomplete apis. | | false | | INSTALLED_MODULES | | | List of installed modules given in helm values/yaml are written in cm and used by devtron to know which modules are given | security.trivy,security.clair | false | - | INSTALLER_CRD_NAMESPACE | string |devtroncd | | | false | - | INSTALLER_CRD_OBJECT_GROUP_NAME | string |installer.devtron.ai | | | false | - | INSTALLER_CRD_OBJECT_RESOURCE | string |installers | | | false | - | INSTALLER_CRD_OBJECT_VERSION | string |v1alpha1 | | | false | + | INSTALLER_CRD_NAMESPACE | string |devtroncd | namespace where Custom Resource Definitions get installed | | false | + | INSTALLER_CRD_OBJECT_GROUP_NAME | string |installer.devtron.ai | Devtron installer CRD group name, partially deprecated. | | false | + | INSTALLER_CRD_OBJECT_RESOURCE | string |installers | Devtron installer CRD resource name, partially deprecated | | false | + | INSTALLER_CRD_OBJECT_VERSION | string |v1alpha1 | version of the CRDs. default is v1alpha1 | | false | | IS_AIR_GAP_ENVIRONMENT | bool |false | | | false | - | JwtExpirationTime | int |120 | | | false | + | JwtExpirationTime | int |120 | JWT expiration time. | | false | | K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST | int |25 | | | false | | K8s_TCP_IDLE_CONN_TIMEOUT | int |300 | | | false | | K8s_TCP_KEEPALIVE | int |30 | | | false | @@ -206,46 +206,46 @@ | K8s_TLS_HANDSHAKE_TIMEOUT | int |10 | | | false | | KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE | int |20 | | | false | | KUBELINK_GRPC_MAX_SEND_MSG_SIZE | int |4 | | | false | - | LENS_TIMEOUT | int |0 | | | false | - | LENS_URL | string |http://lens-milandevtron-service:80 | | | false | + | LENS_TIMEOUT | int |0 | Lens microservice timeout. | | false | + | LENS_URL | string |http://lens-milandevtron-service:80 | Lens micro-service URL | | false | | LIMIT_CI_CPU | string |0.5 | | | false | | LIMIT_CI_MEM | string |3G | | | false | - | LOGGER_DEV_MODE | bool |false | | | false | + | LOGGER_DEV_MODE | bool |false | Enables a different logger theme. | | false | | LOG_LEVEL | int |-1 | | | false | - | MAX_SESSION_PER_USER | int |5 | | | false | - | MODULE_METADATA_API_URL | string |https://api.devtron.ai/module?name=%s | | | false | + | MAX_SESSION_PER_USER | int |5 | max no of cluster terminal pods can be created by an user | | false | + | MODULE_METADATA_API_URL | string |https://api.devtron.ai/module?name=%s | Modules list and meta info will be fetched from this server, that is central api server of devtron. | | false | | MODULE_STATUS_HANDLING_CRON_DURATION_MIN | int |3 | | | false | | NATS_MSG_ACK_WAIT_IN_SECS | int |120 | | | false | | NATS_MSG_BUFFER_SIZE | int |-1 | | | false | | NATS_MSG_MAX_AGE | int |86400 | | | false | | NATS_MSG_PROCESSING_BATCH_SIZE | int |1 | | | false | | NATS_MSG_REPLICAS | int |0 | | | false | - | NOTIFICATION_MEDIUM | NotificationMedium |rest | | | false | - | OTEL_COLLECTOR_URL | string | | | | false | - | PARALLELISM_LIMIT_FOR_TAG_PROCESSING | int | | | | false | + | NOTIFICATION_MEDIUM | NotificationMedium |rest | notification medium | | false | + | OTEL_COLLECTOR_URL | string | | Opentelemetry URL | | false | + | PARALLELISM_LIMIT_FOR_TAG_PROCESSING | int | | App manual sync job parallel tag processing count. | | false | | PG_EXPORT_PROM_METRICS | bool |true | | | false | | PG_LOG_ALL_FAILURE_QUERIES | bool |true | | | false | | PG_LOG_ALL_QUERY | bool |false | | | false | | PG_LOG_SLOW_QUERY | bool |true | | | false | | PG_QUERY_DUR_THRESHOLD | int64 |5000 | | | false | - | PLUGIN_NAME | string |Pull images from container repository | | | false | - | PROPAGATE_EXTRA_LABELS | bool |false | | | false | - | PROXY_SERVICE_CONFIG | string |{} | | | false | + | PLUGIN_NAME | string |Pull images from container repository | Handles image retrieval from a container repository and triggers subsequent CI processes upon detecting new images.Current default plugin name: Pull Images from Container Repository. | | false | + | PROPAGATE_EXTRA_LABELS | bool |false | Add additional propagate labels like api.devtron.ai/appName, api.devtron.ai/envName, api.devtron.ai/project along with the user defined ones. | | false | + | PROXY_SERVICE_CONFIG | string |{} | Proxy configuration for micro-service to be accessible on orhcestrator ingress | | false | | REQ_CI_CPU | string |0.5 | | | false | | REQ_CI_MEM | string |3G | | | false | - | RESTRICT_TERMINAL_ACCESS_FOR_NON_SUPER_USER | bool |false | | | false | + | RESTRICT_TERMINAL_ACCESS_FOR_NON_SUPER_USER | bool |false | To restrict the cluster terminal from user having non-super admin acceess | | false | | RUNTIME_CONFIG_LOCAL_DEV | LocalDevMode |true | | | false | - | SCOPED_VARIABLE_ENABLED | bool |false | | | false | - | SCOPED_VARIABLE_FORMAT | string |@{{%s}} | | | false | - | SCOPED_VARIABLE_HANDLE_PRIMITIVES | bool |false | | | false | - | SCOPED_VARIABLE_NAME_REGEX | string |^[a-zA-Z][a-zA-Z0-9_-]{0,62}[a-zA-Z0-9]$ | | | false | - | SOCKET_DISCONNECT_DELAY_SECONDS | int |5 | | | false | - | SOCKET_HEARTBEAT_SECONDS | int |25 | | | false | + | SCOPED_VARIABLE_ENABLED | bool |false | To enable scoped variable option | | false | + | SCOPED_VARIABLE_FORMAT | string |@{{%s}} | Its a scope format for varialbe name. | | false | + | SCOPED_VARIABLE_HANDLE_PRIMITIVES | bool |false | This describe should we handle primitives or not in scoped variable template parsing. | | false | + | SCOPED_VARIABLE_NAME_REGEX | string |^[a-zA-Z][a-zA-Z0-9_-]{0,62}[a-zA-Z0-9]$ | Regex for scoped variable name that must passed this regex. | | false | + | SOCKET_DISCONNECT_DELAY_SECONDS | int |5 | The server closes a session when a client receiving connection have not been seen for a while.This delay is configured by this setting. By default the session is closed when a receiving connection wasn't seen for 5 seconds. | | false | + | SOCKET_HEARTBEAT_SECONDS | int |25 | In order to keep proxies and load balancers from closing long running http requests we need to pretend that the connection is active and send a heartbeat packet once in a while. This setting controls how often this is done. By default a heartbeat packet is sent every 25 seconds. | | false | | STREAM_CONFIG_JSON | string | | | | false | - | SYSTEM_VAR_PREFIX | string |DEVTRON_ | | | false | - | TERMINAL_POD_DEFAULT_NAMESPACE | string |default | | | false | - | TERMINAL_POD_INACTIVE_DURATION_IN_MINS | int |10 | | | false | - | TERMINAL_POD_STATUS_SYNC_In_SECS | int |600 | | | false | + | SYSTEM_VAR_PREFIX | string |DEVTRON_ | Scoped variable prefix, variable name must have this prefix. | | false | + | TERMINAL_POD_DEFAULT_NAMESPACE | string |default | Cluster terminal default namespace | | false | + | TERMINAL_POD_INACTIVE_DURATION_IN_MINS | int |10 | Timeout for cluster terminal to be inactive | | false | + | TERMINAL_POD_STATUS_SYNC_In_SECS | int |600 | this is the time interval at which the status of the cluster terminal pod | | false | | TEST_APP | string |orchestrator | | | false | | TEST_PG_ADDR | string |127.0.0.1 | | | false | | TEST_PG_DATABASE | string |orchestrator | | | false | @@ -253,63 +253,63 @@ | TEST_PG_PASSWORD | string |postgrespw | | | false | | TEST_PG_PORT | string |55000 | | | false | | TEST_PG_USER | string |postgres | | | false | - | TIMEOUT_FOR_FAILED_CI_BUILD | string |15 | | | false | - | TIMEOUT_IN_SECONDS | int |5 | | | false | + | TIMEOUT_FOR_FAILED_CI_BUILD | string |15 | Timeout for Failed CI build | | false | + | TIMEOUT_IN_SECONDS | int |5 | timeout to compute the urls from services and ingress objects of an application | | false | | USER_SESSION_DURATION_SECONDS | int |86400 | | | false | - | USE_ARTIFACT_LISTING_API_V2 | bool |true | | | false | + | USE_ARTIFACT_LISTING_API_V2 | bool |true | To use the V2 API for listing artifacts in Listing the images in pipeline | | false | | USE_CUSTOM_HTTP_TRANSPORT | bool |false | | | false | - | USE_GIT_CLI | bool |false | | | false | - | USE_RBAC_CREATION_V2 | bool |true | | | false | - | VARIABLE_CACHE_ENABLED | bool |true | | | false | - | VARIABLE_EXPRESSION_REGEX | string |@{{([^}]+)}} | | | false | - | WEBHOOK_TOKEN | string | | | | false | + | USE_GIT_CLI | bool |false | To enable git cli | | false | + | USE_RBAC_CREATION_V2 | bool |true | To use the V2 for RBAC creation | | false | + | VARIABLE_CACHE_ENABLED | bool |true | This is used to control caching of all the scope variables defined in the system. | | false | + | VARIABLE_EXPRESSION_REGEX | string |@{{([^}]+)}} | Scoped variable expression regex | | false | + | WEBHOOK_TOKEN | string | | If you want to continue using jenkins for CI then please provide this for authentication of requests | | false | ## GITOPS Related Environment Variables | Key | Type | Default Value | Description | Example | Deprecated | |-------|----------|-------------------|-------------------|-----------------------|------------------| - | ACD_CM | string |argocd-cm | | | false | - | ACD_NAMESPACE | string |devtroncd | | | false | - | ACD_PASSWORD | string | | | | false | - | ACD_USERNAME | string |admin | | | false | - | GITOPS_SECRET_NAME | string |devtron-gitops-secret | | | false | - | RESOURCE_LIST_FOR_REPLICAS | string |Deployment,Rollout,StatefulSet,ReplicaSet | | | false | - | RESOURCE_LIST_FOR_REPLICAS_BATCH_SIZE | int |5 | | | false | + | ACD_CM | string |argocd-cm | Name of the argocd CM | | false | + | ACD_NAMESPACE | string |devtroncd | To pass the argocd namespace | | false | + | ACD_PASSWORD | string | | Password for the Argocd (deprecated) | | false | + | ACD_USERNAME | string |admin | User name for argocd | | false | + | GITOPS_SECRET_NAME | string |devtron-gitops-secret | devtron-gitops-secret | | false | + | RESOURCE_LIST_FOR_REPLICAS | string |Deployment,Rollout,StatefulSet,ReplicaSet | this holds the list of k8s resource names which support replicas key. this list used in hibernate/un hibernate process | | false | + | RESOURCE_LIST_FOR_REPLICAS_BATCH_SIZE | int |5 | this the batch size to control no of above resources can be parsed in one go to determine hibernate status | | false | ## INFRA_SETUP Related Environment Variables | Key | Type | Default Value | Description | Example | Deprecated | |-------|----------|-------------------|-------------------|-----------------------|------------------| - | DASHBOARD_HOST | string |localhost | | | false | - | DASHBOARD_NAMESPACE | string |devtroncd | | | false | - | DASHBOARD_PORT | string |3000 | | | false | + | DASHBOARD_HOST | string |localhost | Dashboard micro-service URL | | false | + | DASHBOARD_NAMESPACE | string |devtroncd | Dashboard micro-service namespace | | false | + | DASHBOARD_PORT | string |3000 | Port for dashboard micro-service | | false | | DEX_HOST | string |http://localhost | | | false | | DEX_PORT | string |5556 | | | false | - | GIT_SENSOR_PROTOCOL | string |REST | | | false | - | GIT_SENSOR_TIMEOUT | int |0 | | | false | - | GIT_SENSOR_URL | string |127.0.0.1:7070 | | | false | - | HELM_CLIENT_URL | string |127.0.0.1:50051 | | | false | + | GIT_SENSOR_PROTOCOL | string |REST | Protocol to connect with git-sensor micro-service | | false | + | GIT_SENSOR_TIMEOUT | int |0 | Timeout for getting response from the git-sensor | | false | + | GIT_SENSOR_URL | string |127.0.0.1:7070 | git-sensor micro-service url | | false | + | HELM_CLIENT_URL | string |127.0.0.1:50051 | Kubelink micro-service url | | false | ## POSTGRES Related Environment Variables | Key | Type | Default Value | Description | Example | Deprecated | |-------|----------|-------------------|-------------------|-----------------------|------------------| | APP | string |orchestrator | Application name | | false | - | CASBIN_DATABASE | string |casbin | | | false | + | CASBIN_DATABASE | string |casbin | Database for casbin | | false | | PG_ADDR | string |127.0.0.1 | address of postgres service | postgresql-postgresql.devtroncd | false | | PG_DATABASE | string |orchestrator | postgres database to be made connection with | orchestrator, casbin, git_sensor, lens | false | | PG_PASSWORD | string |{password} | password for postgres, associated with PG_USER | confidential ;) | false | | PG_PORT | string |5432 | port of postgresql service | 5432 | false | - | PG_READ_TIMEOUT | int64 |30 | | | false | + | PG_READ_TIMEOUT | int64 |30 | Time out for read operation in postgres | | false | | PG_USER | string |postgres | user for postgres | postgres | false | - | PG_WRITE_TIMEOUT | int64 |30 | | | false | + | PG_WRITE_TIMEOUT | int64 |30 | Time out for write operation in postgres | | false | ## RBAC Related Environment Variables | Key | Type | Default Value | Description | Example | Deprecated | |-------|----------|-------------------|-------------------|-----------------------|------------------| - | ENFORCER_CACHE | bool |false | | | false | - | ENFORCER_CACHE_EXPIRATION_IN_SEC | int |86400 | | | false | - | ENFORCER_MAX_BATCH_SIZE | int |1 | | | false | - | USE_CASBIN_V2 | bool |true | | | false | + | ENFORCER_CACHE | bool |false | To Enable enforcer cache. | | false | + | ENFORCER_CACHE_EXPIRATION_IN_SEC | int |86400 | Expiration time (in seconds) for enforcer cache. | | false | + | ENFORCER_MAX_BATCH_SIZE | int |1 | Maximum batch size for the enforcer. | | false | + | USE_CASBIN_V2 | bool |true | To enable casbin V2 API | | false | diff --git a/pkg/build/trigger/HandlerService.go b/pkg/build/trigger/HandlerService.go index 00697d4454..7b5832ffa4 100644 --- a/pkg/build/trigger/HandlerService.go +++ b/pkg/build/trigger/HandlerService.go @@ -84,8 +84,8 @@ type HandlerService interface { } type BuildxCacheFlags struct { - BuildxCacheModeMin bool `env:"BUILDX_CACHE_MODE_MIN" envDefault:"false"` - AsyncBuildxCacheExport bool `env:"ASYNC_BUILDX_CACHE_EXPORT" envDefault:"false"` + BuildxCacheModeMin bool `env:"BUILDX_CACHE_MODE_MIN" envDefault:"false" description:"To set build cache mode to minimum in buildx" ` + AsyncBuildxCacheExport bool `env:"ASYNC_BUILDX_CACHE_EXPORT" envDefault:"false" description:"To enable async container image cache export"` } type HandlerServiceImpl struct { diff --git a/wire_gen.go b/wire_gen.go index 8b81223419..2c1c56d27b 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run -mod=mod github.com/google/wire/cmd/wire +//go:generate go run github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject