From 237d8d0c51a731c4490cc3672bf5d0a890dd1668 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 1 Apr 2025 19:38:15 +0530 Subject: [PATCH 01/17] changes around ciHandler --- api/restHandler/CoreAppRestHandler.go | 3 +- client/cron/CiTriggerCron.go | 4 +- client/events/EventBuilder.go | 13 +- client/events/EventClient.go | 60 ++---- cmd/external-app/wire_gen.go | 2 +- .../pipelineConfig/CdWorfkflowRepository.go | 19 -- pkg/bean/app.go | 101 ++++----- .../pipeline/bean/CIPipelineMaterialBean.go | 30 +++ pkg/build/pipeline/bean/CiBuildConfig.go | 26 --- .../pipeline/bean/common/CommonBuildBean.go | 27 +++ pkg/build/trigger/Service.go | 1 + pkg/pipeline/BuildPipelineConfigService.go | 29 +-- pkg/pipeline/BuildPipelineSwitchService.go | 20 +- pkg/pipeline/CIHandler_ent.go | 31 +++ pkg/pipeline/CdHandler.go | 9 +- pkg/pipeline/CiCdPipelineOrchestrator.go | 5 +- pkg/pipeline/CiHandler.go | 198 +++++++++--------- pkg/pipeline/CiService.go | 30 ++- pkg/pipeline/WorkflowService.go | 4 +- pkg/pipeline/adapter/adapter.go | 7 +- pkg/pipeline/bean/CdHandlerBean.go | 57 ++--- pkg/pipeline/constants/constants.go | 1 - pkg/pipeline/types/CiCdConfig.go | 24 ++- pkg/pipeline/types/Workflow.go | 63 +++--- pkg/workflow/dag/WorkflowDagExecutor.go | 5 +- wire_gen.go | 2 +- 26 files changed, 404 insertions(+), 367 deletions(-) create mode 100644 pkg/build/pipeline/bean/CIPipelineMaterialBean.go create mode 100644 pkg/build/pipeline/bean/common/CommonBuildBean.go create mode 100644 pkg/build/trigger/Service.go create mode 100644 pkg/pipeline/CIHandler_ent.go diff --git a/api/restHandler/CoreAppRestHandler.go b/api/restHandler/CoreAppRestHandler.go index bb8581dcfc..14d3556db5 100644 --- a/api/restHandler/CoreAppRestHandler.go +++ b/api/restHandler/CoreAppRestHandler.go @@ -29,6 +29,7 @@ import ( "github.com/devtron-labs/devtron/pkg/build/git/gitProvider" "github.com/devtron-labs/devtron/pkg/build/git/gitProvider/read" pipelineBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + common2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" bean3 "github.com/devtron-labs/devtron/pkg/chart/bean" read5 "github.com/devtron-labs/devtron/pkg/chart/read" "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" @@ -1688,7 +1689,7 @@ func (handler CoreAppRestHandlerImpl) createCiPipeline(appId int, userId int32, ParentCiPipeline: ciPipelineData.ParentCiPipeline, ParentAppId: ciPipelineData.ParentAppId, LinkedCount: ciPipelineData.LinkedCount, - PipelineType: pipelineBean.PipelineType(ciPipelineData.PipelineType), + PipelineType: common2.PipelineType(ciPipelineData.PipelineType), }, } diff --git a/client/cron/CiTriggerCron.go b/client/cron/CiTriggerCron.go index 3ded91c038..35f1e9876a 100644 --- a/client/cron/CiTriggerCron.go +++ b/client/cron/CiTriggerCron.go @@ -22,7 +22,7 @@ import ( repository2 "github.com/devtron-labs/devtron/internal/sql/repository" bean2 "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/bean" - pipelineConfigBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/repository" repository3 "github.com/devtron-labs/devtron/pkg/plugin/repository" @@ -101,7 +101,7 @@ func (impl *CiTriggerCronImpl) TriggerCiCron() { CiPipelineMaterial: ciPipelineMaterials, TriggeredBy: bean2.SYSTEM_USER_ID, InvalidateCache: false, - PipelineType: string(pipelineConfigBean.CI_JOB), + PipelineType: string(common.CI_JOB), } _, err = impl.ciHandler.HandleCIManual(ciTriggerRequest) if err != nil { diff --git a/client/events/EventBuilder.go b/client/events/EventBuilder.go index 83f7186811..4464d7aa98 100644 --- a/client/events/EventBuilder.go +++ b/client/events/EventBuilder.go @@ -19,6 +19,7 @@ package client import ( "context" "fmt" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" repository4 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "strings" "time" @@ -38,7 +39,7 @@ import ( type EventFactory interface { Build(eventType util.EventType, sourceId *int, appId int, envId *int, pipelineType util.PipelineType) (Event, error) BuildExtraCDData(event Event, wfr *pipelineConfig.CdWorkflowRunner, pipelineOverrideId int, stage bean2.WorkflowType) Event - BuildExtraCIData(event Event, material *MaterialTriggerInfo) Event + BuildExtraCIData(event Event, material *buildBean.MaterialTriggerInfo) Event //BuildFinalData(event Event) *Payload } @@ -173,7 +174,7 @@ func (impl *EventSimpleFactoryImpl) BuildExtraCDData(event Event, wfr *pipelineC return event } -func (impl *EventSimpleFactoryImpl) BuildExtraCIData(event Event, material *MaterialTriggerInfo) Event { +func (impl *EventSimpleFactoryImpl) BuildExtraCIData(event Event, material *buildBean.MaterialTriggerInfo) Event { if material == nil { materialInfo, err := impl.getCiMaterialInfo(event.PipelineId, event.CiArtifactId) if err != nil { @@ -207,8 +208,8 @@ func (impl *EventSimpleFactoryImpl) BuildExtraCIData(event Event, material *Mate return event } -func (impl *EventSimpleFactoryImpl) getCiMaterialInfo(ciPipelineId int, ciArtifactId int) (*MaterialTriggerInfo, error) { - materialTriggerInfo := &MaterialTriggerInfo{} +func (impl *EventSimpleFactoryImpl) getCiMaterialInfo(ciPipelineId int, ciArtifactId int) (*buildBean.MaterialTriggerInfo, error) { + materialTriggerInfo := &buildBean.MaterialTriggerInfo{} if ciPipelineId > 0 { ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(ciPipelineId) if err != nil { @@ -216,13 +217,13 @@ func (impl *EventSimpleFactoryImpl) getCiMaterialInfo(ciPipelineId int, ciArtifa return nil, err } - var ciMaterialsArr []CiPipelineMaterialResponse + var ciMaterialsArr []buildBean.CiPipelineMaterialResponse for _, m := range ciMaterials { if m.GitMaterial == nil { impl.logger.Warnw("git material are empty", "material", m) continue } - res := CiPipelineMaterialResponse{ + res := buildBean.CiPipelineMaterialResponse{ Id: m.Id, GitMaterialId: m.GitMaterialId, GitMaterialName: m.GitMaterial.Name[strings.Index(m.GitMaterial.Name, "-")+1:], diff --git a/client/events/EventClient.go b/client/events/EventClient.go index a58cadea82..e2ddcd4886 100644 --- a/client/events/EventClient.go +++ b/client/events/EventClient.go @@ -21,20 +21,18 @@ import ( "encoding/json" "errors" "fmt" - bean2 "github.com/devtron-labs/devtron/pkg/attributes/bean" - "github.com/devtron-labs/devtron/pkg/module" - bean3 "github.com/devtron-labs/devtron/pkg/module/bean" - "net/http" - "time" - "github.com/caarlos0/env" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + bean2 "github.com/devtron-labs/devtron/pkg/attributes/bean" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/module" + bean3 "github.com/devtron-labs/devtron/pkg/module/bean" util "github.com/devtron-labs/devtron/util/event" "go.uber.org/zap" + "net/http" ) type EventClientConfig struct { @@ -81,41 +79,19 @@ type Event struct { } type Payload struct { - AppName string `json:"appName"` - EnvName string `json:"envName"` - PipelineName string `json:"pipelineName"` - Source string `json:"source"` - DockerImageUrl string `json:"dockerImageUrl"` - TriggeredBy string `json:"triggeredBy"` - Stage string `json:"stage"` - DeploymentHistoryLink string `json:"deploymentHistoryLink"` - AppDetailLink string `json:"appDetailLink"` - DownloadLink string `json:"downloadLink"` - BuildHistoryLink string `json:"buildHistoryLink"` - MaterialTriggerInfo *MaterialTriggerInfo `json:"material"` - FailureReason string `json:"failureReason"` -} - -type CiPipelineMaterialResponse struct { - Id int `json:"id"` - GitMaterialId int `json:"gitMaterialId"` - GitMaterialUrl string `json:"gitMaterialUrl"` - GitMaterialName string `json:"gitMaterialName"` - Type string `json:"type"` - Value string `json:"value"` - Active bool `json:"active"` - History []*gitSensor.GitCommit `json:"history,omitempty"` - LastFetchTime time.Time `json:"lastFetchTime"` - IsRepoError bool `json:"isRepoError"` - RepoErrorMsg string `json:"repoErrorMsg"` - IsBranchError bool `json:"isBranchError"` - BranchErrorMsg string `json:"branchErrorMsg"` - Url string `json:"url"` -} - -type MaterialTriggerInfo struct { - GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` - CiMaterials []CiPipelineMaterialResponse `json:"ciMaterials"` + AppName string `json:"appName"` + EnvName string `json:"envName"` + PipelineName string `json:"pipelineName"` + Source string `json:"source"` + DockerImageUrl string `json:"dockerImageUrl"` + TriggeredBy string `json:"triggeredBy"` + Stage string `json:"stage"` + DeploymentHistoryLink string `json:"deploymentHistoryLink"` + AppDetailLink string `json:"appDetailLink"` + DownloadLink string `json:"downloadLink"` + BuildHistoryLink string `json:"buildHistoryLink"` + MaterialTriggerInfo *buildBean.MaterialTriggerInfo `json:"material"` + FailureReason string `json:"failureReason"` } type EventRESTClientImpl struct { diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index cc31476cbe..e827637c0a 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index a737a7076a..b0708ad99e 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -20,7 +20,6 @@ import ( "context" "errors" apiBean "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" @@ -149,24 +148,6 @@ func (c *CdWorkflowRunner) IsExternalRun() bool { return isExtCluster } -type CiPipelineMaterialResponse struct { - Id int `json:"id"` - GitMaterialId int `json:"gitMaterialId"` - GitMaterialUrl string `json:"gitMaterialUrl"` - GitMaterialName string `json:"gitMaterialName"` - Type string `json:"type"` - Value string `json:"value"` - Active bool `json:"active"` - History []*gitSensor.GitCommit `json:"history,omitempty"` - LastFetchTime time.Time `json:"lastFetchTime"` - IsRepoError bool `json:"isRepoError"` - RepoErrorMsg string `json:"repoErrorMsg"` - IsBranchError bool `json:"isBranchError"` - BranchErrorMsg string `json:"branchErrorMsg"` - Url string `json:"url"` - Regex string `json:"regex"` -} - type TriggerWorkflowStatus struct { CdWorkflowStatus []*CdWorkflowStatus `json:"cdWorkflowStatus"` CiWorkflowStatus []*CiWorkflowStatus `json:"ciWorkflowStatus"` diff --git a/pkg/bean/app.go b/pkg/bean/app.go index 96c581b5f6..83c6717093 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -29,6 +29,7 @@ import ( "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/bean/common" CiPipeline2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + common2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" "github.com/devtron-labs/devtron/pkg/chartRepo/repository" bean3 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/pipeline/bean" @@ -117,39 +118,39 @@ type CiMaterial struct { } type CiPipeline struct { - IsManual bool `json:"isManual"` - DockerArgs map[string]string `json:"dockerArgs"` - IsExternal bool `json:"isExternal"` - ParentCiPipeline int `json:"parentCiPipeline"` - ParentAppId int `json:"parentAppId"` - AppId int `json:"appId"` - AppName string `json:"appName,omitempty"` - AppType helper.AppType `json:"appType,omitempty"` - ExternalCiConfig ExternalCiConfig `json:"externalCiConfig"` - CiMaterial []*CiMaterial `json:"ciMaterial,omitempty" validate:"dive,min=1"` - Name string `json:"name,omitempty" validate:"name-component,max=100"` // name suffix of corresponding pipeline. required, unique, validation corresponding to gocd pipelineName will be applicable - Id int `json:"id,omitempty" ` - Version string `json:"version,omitempty"` // matchIf token version in gocd . used for update request - Active bool `json:"active,omitempty"` // pipeline is active or not - Deleted bool `json:"deleted,omitempty"` - BeforeDockerBuild []*Task `json:"beforeDockerBuild,omitempty" validate:"dive"` - AfterDockerBuild []*Task `json:"afterDockerBuild,omitempty" validate:"dive"` - BeforeDockerBuildScripts []*CiScript `json:"beforeDockerBuildScripts,omitempty" validate:"dive"` - AfterDockerBuildScripts []*CiScript `json:"afterDockerBuildScripts,omitempty" validate:"dive"` - LinkedCount int `json:"linkedCount"` - PipelineType CiPipeline2.PipelineType `json:"pipelineType,omitempty"` - ScanEnabled bool `json:"scanEnabled,notnull"` - AppWorkflowId int `json:"appWorkflowId,omitempty"` - PreBuildStage *bean.PipelineStageDto `json:"preBuildStage,omitempty" validate:"omitempty,dive"` - PostBuildStage *bean.PipelineStageDto `json:"postBuildStage,omitempty" validate:"omitempty,dive"` - TargetPlatform string `json:"targetPlatform,omitempty"` - IsDockerConfigOverridden bool `json:"isDockerConfigOverridden"` - DockerConfigOverride DockerConfigOverride `json:"dockerConfigOverride,omitempty"` - EnvironmentId int `json:"environmentId,omitempty"` - LastTriggeredEnvId int `json:"lastTriggeredEnvId"` - CustomTagObject *CustomTagData `json:"customTag,omitempty"` - DefaultTag []string `json:"defaultTag,omitempty"` - EnableCustomTag bool `json:"enableCustomTag"` + IsManual bool `json:"isManual"` + DockerArgs map[string]string `json:"dockerArgs"` + IsExternal bool `json:"isExternal"` + ParentCiPipeline int `json:"parentCiPipeline"` + ParentAppId int `json:"parentAppId"` + AppId int `json:"appId"` + AppName string `json:"appName,omitempty"` + AppType helper.AppType `json:"appType,omitempty"` + ExternalCiConfig ExternalCiConfig `json:"externalCiConfig"` + CiMaterial []*CiMaterial `json:"ciMaterial,omitempty" validate:"dive,min=1"` + Name string `json:"name,omitempty" validate:"name-component,max=100"` // name suffix of corresponding pipeline. required, unique, validation corresponding to gocd pipelineName will be applicable + Id int `json:"id,omitempty" ` + Version string `json:"version,omitempty"` // matchIf token version in gocd . used for update request + Active bool `json:"active,omitempty"` // pipeline is active or not + Deleted bool `json:"deleted,omitempty"` + BeforeDockerBuild []*Task `json:"beforeDockerBuild,omitempty" validate:"dive"` + AfterDockerBuild []*Task `json:"afterDockerBuild,omitempty" validate:"dive"` + BeforeDockerBuildScripts []*CiScript `json:"beforeDockerBuildScripts,omitempty" validate:"dive"` + AfterDockerBuildScripts []*CiScript `json:"afterDockerBuildScripts,omitempty" validate:"dive"` + LinkedCount int `json:"linkedCount"` + PipelineType common2.PipelineType `json:"pipelineType,omitempty"` + ScanEnabled bool `json:"scanEnabled,notnull"` + AppWorkflowId int `json:"appWorkflowId,omitempty"` + PreBuildStage *bean.PipelineStageDto `json:"preBuildStage,omitempty" validate:"omitempty,dive"` + PostBuildStage *bean.PipelineStageDto `json:"postBuildStage,omitempty" validate:"omitempty,dive"` + TargetPlatform string `json:"targetPlatform,omitempty"` + IsDockerConfigOverridden bool `json:"isDockerConfigOverridden"` + DockerConfigOverride DockerConfigOverride `json:"dockerConfigOverride,omitempty"` + EnvironmentId int `json:"environmentId,omitempty"` + LastTriggeredEnvId int `json:"lastTriggeredEnvId"` + CustomTagObject *CustomTagData `json:"customTag,omitempty"` + DefaultTag []string `json:"defaultTag,omitempty"` + EnableCustomTag bool `json:"enableCustomTag"` } func (ciPipeline *CiPipeline) IsLinkedCi() bool { @@ -164,14 +165,14 @@ type DockerConfigOverride struct { } type CiPipelineMin struct { - Name string `json:"name,omitempty" validate:"name-component,max=100"` //name suffix of corresponding pipeline. required, unique, validation corresponding to gocd pipelineName will be applicable - Id int `json:"id,omitempty" ` - Version string `json:"version,omitempty"` //matchIf token version in gocd . used for update request - IsExternal bool `json:"isExternal,omitempty"` - ParentCiPipeline int `json:"parentCiPipeline"` - ParentAppId int `json:"parentAppId"` - PipelineType CiPipeline2.PipelineType `json:"pipelineType,omitempty"` - ScanEnabled bool `json:"scanEnabled,notnull"` + Name string `json:"name,omitempty" validate:"name-component,max=100"` //name suffix of corresponding pipeline. required, unique, validation corresponding to gocd pipelineName will be applicable + Id int `json:"id,omitempty" ` + Version string `json:"version,omitempty"` //matchIf token version in gocd . used for update request + IsExternal bool `json:"isExternal,omitempty"` + ParentCiPipeline int `json:"parentCiPipeline"` + ParentAppId int `json:"parentAppId"` + PipelineType common2.PipelineType `json:"pipelineType,omitempty"` + ScanEnabled bool `json:"scanEnabled,notnull"` } type CiScript struct { @@ -303,20 +304,20 @@ type CiPatchRequest struct { IsJob bool `json:"-"` IsCloneJob bool `json:"isCloneJob,omitempty"` - ParentCDPipeline int `json:"parentCDPipeline"` - DeployEnvId int `json:"deployEnvId"` - SwitchFromCiPipelineId int `json:"switchFromCiPipelineId"` - SwitchFromExternalCiPipelineId int `json:"switchFromExternalCiPipelineId"` - SwitchFromCiPipelineType CiPipeline2.PipelineType `json:"-"` - SwitchToCiPipelineType CiPipeline2.PipelineType `json:"-"` + ParentCDPipeline int `json:"parentCDPipeline"` + DeployEnvId int `json:"deployEnvId"` + SwitchFromCiPipelineId int `json:"switchFromCiPipelineId"` + SwitchFromExternalCiPipelineId int `json:"switchFromExternalCiPipelineId"` + SwitchFromCiPipelineType common2.PipelineType `json:"-"` + SwitchToCiPipelineType common2.PipelineType `json:"-"` } -func (ciPatchRequest CiPatchRequest) SwitchSourceInfo() (int, CiPipeline2.PipelineType) { +func (ciPatchRequest CiPatchRequest) SwitchSourceInfo() (int, common2.PipelineType) { // get the ciPipeline - var switchFromType CiPipeline2.PipelineType + var switchFromType common2.PipelineType var switchFromPipelineId int if ciPatchRequest.SwitchFromExternalCiPipelineId != 0 { - switchFromType = CiPipeline2.EXTERNAL + switchFromType = common2.EXTERNAL switchFromPipelineId = ciPatchRequest.SwitchFromExternalCiPipelineId } else { switchFromPipelineId = ciPatchRequest.SwitchFromCiPipelineId diff --git a/pkg/build/pipeline/bean/CIPipelineMaterialBean.go b/pkg/build/pipeline/bean/CIPipelineMaterialBean.go new file mode 100644 index 0000000000..8f09c47ebe --- /dev/null +++ b/pkg/build/pipeline/bean/CIPipelineMaterialBean.go @@ -0,0 +1,30 @@ +package bean + +import ( + "github.com/devtron-labs/devtron/client/gitSensor" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "time" +) + +type CiPipelineMaterialResponse struct { + Id int `json:"id"` + GitMaterialId int `json:"gitMaterialId"` + GitMaterialUrl string `json:"gitMaterialUrl"` + GitMaterialName string `json:"gitMaterialName"` + Type string `json:"type"` + Value string `json:"value"` + Active bool `json:"active"` + History []*gitSensor.GitCommit `json:"history,omitempty"` + LastFetchTime time.Time `json:"lastFetchTime"` + IsRepoError bool `json:"isRepoError"` + RepoErrorMsg string `json:"repoErrorMsg"` + IsBranchError bool `json:"isBranchError"` + BranchErrorMsg string `json:"branchErrorMsg"` + Url string `json:"url"` + Regex string `json:"regex"` +} + +type MaterialTriggerInfo struct { + GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` + CiMaterials []CiPipelineMaterialResponse `json:"ciMaterials"` +} diff --git a/pkg/build/pipeline/bean/CiBuildConfig.go b/pkg/build/pipeline/bean/CiBuildConfig.go index 34357f5e68..39d8671a35 100644 --- a/pkg/build/pipeline/bean/CiBuildConfig.go +++ b/pkg/build/pipeline/bean/CiBuildConfig.go @@ -30,23 +30,6 @@ const UniquePlaceHolderForAppName = "$etron" const PIPELINE_NAME_ALREADY_EXISTS_ERROR = "pipeline name already exist" const PIPELINE_TYPE_IS_NOT_VALID = "PipelineType is not valid for pipeline %s" -type PipelineType string - -// default PipelineType -const DefaultPipelineType = CI_BUILD - -const ( - CI_BUILD PipelineType = "CI_BUILD" - LINKED PipelineType = "LINKED" - EXTERNAL PipelineType = "EXTERNAL" - CI_JOB PipelineType = "CI_JOB" - LINKED_CD PipelineType = "LINKED_CD" -) - -func (pType PipelineType) ToString() string { - return string(pType) -} - type CiBuildConfigBean struct { Id int `json:"id"` GitMaterialId int `json:"gitMaterialId,omitempty" validate:"required"` @@ -81,15 +64,6 @@ type BuildPackConfig struct { ProjectPath string `json:"projectPath,omitempty"` } -func (pType PipelineType) IsValidPipelineType() bool { - switch pType { - case CI_BUILD, LINKED, EXTERNAL, CI_JOB, LINKED_CD: - return true - default: - return false - } -} - const ( ExtraEnvVarExternalCiArtifactKey = "externalCiArtifact" ExtraEnvVarImageDigestKey = "imageDigest" diff --git a/pkg/build/pipeline/bean/common/CommonBuildBean.go b/pkg/build/pipeline/bean/common/CommonBuildBean.go new file mode 100644 index 0000000000..af419f962f --- /dev/null +++ b/pkg/build/pipeline/bean/common/CommonBuildBean.go @@ -0,0 +1,27 @@ +package common + +type PipelineType string + +// default PipelineType +const DefaultPipelineType = CI_BUILD + +const ( + CI_BUILD PipelineType = "CI_BUILD" + LINKED PipelineType = "LINKED" + EXTERNAL PipelineType = "EXTERNAL" + CI_JOB PipelineType = "CI_JOB" + LINKED_CD PipelineType = "LINKED_CD" +) + +func (pType PipelineType) ToString() string { + return string(pType) +} + +func (pType PipelineType) IsValidPipelineType() bool { + switch pType { + case CI_BUILD, LINKED, EXTERNAL, CI_JOB, LINKED_CD: + return true + default: + return false + } +} diff --git a/pkg/build/trigger/Service.go b/pkg/build/trigger/Service.go new file mode 100644 index 0000000000..dee4fe7ace --- /dev/null +++ b/pkg/build/trigger/Service.go @@ -0,0 +1 @@ +package trigger diff --git a/pkg/pipeline/BuildPipelineConfigService.go b/pkg/pipeline/BuildPipelineConfigService.go index fca115326e..95cd8775ac 100644 --- a/pkg/pipeline/BuildPipelineConfigService.go +++ b/pkg/pipeline/BuildPipelineConfigService.go @@ -35,6 +35,7 @@ import ( repository3 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/repository" "github.com/devtron-labs/devtron/pkg/build/pipeline" bean3 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" "github.com/devtron-labs/devtron/pkg/build/pipeline/read" pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/history" @@ -302,11 +303,11 @@ func (impl *CiPipelineConfigServiceImpl) patchCiPipelineUpdateSource(baseCiConfi } // updating PipelineType from db if not present in request if modifiedCiPipeline.PipelineType == "" { - if bean3.PipelineType(pipeline.PipelineType) != "" { - modifiedCiPipeline.PipelineType = bean3.PipelineType(pipeline.PipelineType) + if common.PipelineType(pipeline.PipelineType) != "" { + modifiedCiPipeline.PipelineType = common.PipelineType(pipeline.PipelineType) } else { // updating default pipelineType if not present in request - modifiedCiPipeline.PipelineType = bean3.DefaultPipelineType + modifiedCiPipeline.PipelineType = common.DefaultPipelineType } } @@ -617,7 +618,7 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipeline(appId int) (ciConfig *bea AfterDockerBuildScripts: afterDockerBuildScripts, ScanEnabled: pipeline.ScanEnabled, IsDockerConfigOverridden: pipeline.IsDockerConfigOverridden, - PipelineType: bean3.PipelineType(pipeline.PipelineType), + PipelineType: common.PipelineType(pipeline.PipelineType), } ciEnvMapping, err := impl.ciPipelineRepository.FindCiEnvMappingByCiPipelineId(pipeline.Id) if err != nil && err != pg.ErrNoRows { @@ -773,7 +774,7 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipelineById(pipelineId int) (ciPi AfterDockerBuildScripts: afterDockerBuildScripts, ScanEnabled: pipeline.ScanEnabled, IsDockerConfigOverridden: pipeline.IsDockerConfigOverridden, - PipelineType: bean3.PipelineType(pipeline.PipelineType), + PipelineType: common.PipelineType(pipeline.PipelineType), } customTag, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id)) if err != nil && err != pg.ErrNoRows { @@ -897,7 +898,7 @@ func (impl *CiPipelineConfigServiceImpl) GetTriggerViewCiPipeline(appId int) (*b ParentCiPipeline: pipeline.ParentCiPipeline, ScanEnabled: pipeline.ScanEnabled, IsDockerConfigOverridden: pipeline.IsDockerConfigOverridden, - PipelineType: bean3.PipelineType(pipeline.PipelineType), + PipelineType: common.PipelineType(pipeline.PipelineType), } if ciTemplateBean, ok := ciOverrideTemplateMap[pipeline.Id]; ok { templateOverride := ciTemplateBean.CiTemplateOverride @@ -1263,7 +1264,7 @@ func (impl *CiPipelineConfigServiceImpl) UpdateCiTemplate(updateRequest *bean.Ci } for _, ciTemplateOverride := range ciTemplateOverrides { if _, ok := ciPipelineIdsMap[ciTemplateOverride.CiPipelineId]; ok { - if ciPipelineIdsMap[ciTemplateOverride.CiPipelineId].PipelineType == string(bean3.CI_JOB) { + if ciPipelineIdsMap[ciTemplateOverride.CiPipelineId].PipelineType == string(common.CI_JOB) { ciTemplateOverride.DockerRepository = updateRequest.DockerRepository ciTemplateOverride.DockerRegistryId = updateRequest.DockerRegistry _, err = impl.ciTemplateOverrideRepository.Update(ciTemplateOverride) @@ -1319,7 +1320,7 @@ func (impl *CiPipelineConfigServiceImpl) PatchCiPipeline(request *bean.CiPatchRe impl.logger.Errorw("err in fetching template for pipeline patch, ", "err", err, "appId", request.AppId) return nil, err } - if request.CiPipeline.PipelineType == bean3.CI_JOB { + if request.CiPipeline.PipelineType == common.CI_JOB { request.CiPipeline.IsDockerConfigOverridden = true request.CiPipeline.DockerConfigOverride = bean.DockerConfigOverride{ DockerRegistry: ciConfig.DockerRegistry, @@ -1512,15 +1513,15 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipelineMin(appId int, envIds []in var ciPipelineResp []*bean.CiPipelineMin for _, pipeline := range pipelines { parentCiPipeline := pipelineConfig.CiPipeline{} - pipelineType := bean3.CI_BUILD + pipelineType := common.CI_BUILD if pipelineParentCiMap[pipeline.Id] != nil { parentCiPipeline = *pipelineParentCiMap[pipeline.Id] - pipelineType = bean3.LINKED + pipelineType = common.LINKED } else if pipeline.IsExternal == true { - pipelineType = bean3.EXTERNAL - } else if pipeline.PipelineType == string(bean3.CI_JOB) { - pipelineType = bean3.CI_JOB + pipelineType = common.EXTERNAL + } else if pipeline.PipelineType == string(common.CI_JOB) { + pipelineType = common.CI_JOB } ciPipeline := &bean.CiPipelineMin{ @@ -1736,7 +1737,7 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipelineByEnvironment(request reso ExternalCiConfig: externalCiConfig, ScanEnabled: pipeline.ScanEnabled, IsDockerConfigOverridden: pipeline.IsDockerConfigOverridden, - PipelineType: bean3.PipelineType(pipeline.PipelineType), + PipelineType: common.PipelineType(pipeline.PipelineType), } parentPipelineAppId, ok := pipelineIdVsAppId[parentCiPipelineId] if ok { diff --git a/pkg/pipeline/BuildPipelineSwitchService.go b/pkg/pipeline/BuildPipelineSwitchService.go index 076d95a12e..858f03473c 100644 --- a/pkg/pipeline/BuildPipelineSwitchService.go +++ b/pkg/pipeline/BuildPipelineSwitchService.go @@ -21,7 +21,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/bean" - pipelineConfigBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" "github.com/devtron-labs/devtron/pkg/build/pipeline/read" "github.com/devtron-labs/devtron/pkg/pipeline/adapter" "github.com/devtron-labs/devtron/pkg/pipeline/history" @@ -124,10 +124,10 @@ func (impl *BuildPipelineSwitchServiceImpl) SwitchToCiPipelineExceptExternal(req } //get the ciPipeline - var switchFromType pipelineConfigBean.PipelineType + var switchFromType common.PipelineType var switchFromPipelineId int if request.SwitchFromExternalCiPipelineId != 0 { - switchFromType = pipelineConfigBean.EXTERNAL + switchFromType = common.EXTERNAL switchFromPipelineId = request.SwitchFromExternalCiPipelineId } else { switchFromPipelineId = request.SwitchFromCiPipelineId @@ -145,9 +145,9 @@ func (impl *BuildPipelineSwitchServiceImpl) SwitchToCiPipelineExceptExternal(req return impl.createNewPipelineAndReplaceOldPipelineLinks(request.CiPipeline, ciConfig, switchFromPipelineId, switchFromType, request.UserId) } -func (impl *BuildPipelineSwitchServiceImpl) createNewPipelineAndReplaceOldPipelineLinks(ciPipelineReq *bean.CiPipeline, ciConfig *bean.CiConfigRequest, switchFromPipelineId int, switchFromType pipelineConfigBean.PipelineType, userId int32) (*bean.CiConfigRequest, error) { +func (impl *BuildPipelineSwitchServiceImpl) createNewPipelineAndReplaceOldPipelineLinks(ciPipelineReq *bean.CiPipeline, ciConfig *bean.CiConfigRequest, switchFromPipelineId int, switchFromType common.PipelineType, userId int32) (*bean.CiConfigRequest, error) { - isSelfLinkedCiPipeline := switchFromType != pipelineConfigBean.EXTERNAL && ciPipelineReq.IsLinkedCi() && ciPipelineReq.ParentCiPipeline == switchFromPipelineId + isSelfLinkedCiPipeline := switchFromType != common.EXTERNAL && ciPipelineReq.IsLinkedCi() && ciPipelineReq.ParentCiPipeline == switchFromPipelineId if isSelfLinkedCiPipeline { errMsg := "cannot create linked ci pipeline from the same source" return nil, util.DefaultApiError().WithInternalMessage(errMsg).WithUserMessage(errMsg).WithHttpStatusCode(http.StatusBadRequest) @@ -208,7 +208,7 @@ func (impl *BuildPipelineSwitchServiceImpl) createNewPipelineAndReplaceOldPipeli // add switchType and remove other id // make constants for error msgs -func (impl *BuildPipelineSwitchServiceImpl) validateCiPipelineSwitch(switchFromCiPipelineId int, switchToType, switchFromType pipelineConfigBean.PipelineType) error { +func (impl *BuildPipelineSwitchServiceImpl) validateCiPipelineSwitch(switchFromCiPipelineId int, switchToType, switchFromType common.PipelineType) error { // this will only allow below conversions // ext -> {ci_job,direct,linked} // direct -> {ci_job,linked} @@ -220,13 +220,13 @@ func (impl *BuildPipelineSwitchServiceImpl) validateCiPipelineSwitch(switchFromC } // refer SwitchToExternalCi - if switchToType == pipelineConfigBean.EXTERNAL { + if switchToType == common.EXTERNAL { return errors.New(string(cannotConvertToExternalCi)) } // we should not check the below logic for external_ci type as builds are not built in devtron and // linked pipelines won't be there as per current external-ci-pipeline architecture - if switchFromCiPipelineId > 0 && switchFromType != pipelineConfigBean.EXTERNAL { + if switchFromCiPipelineId > 0 && switchFromType != common.EXTERNAL { err := impl.validateSwitchPreConditions(switchFromCiPipelineId) if err != nil { return err @@ -252,13 +252,13 @@ func (impl *BuildPipelineSwitchServiceImpl) deleteCiAndItsWorkflowMappings(tx *p return err } -func (impl *BuildPipelineSwitchServiceImpl) deleteOldCiPipelineAndWorkflowMappingBeforeSwitch(tx *pg.Tx, switchFromPipelineId int, switchFromType pipelineConfigBean.PipelineType, userId int32) (*appWorkflow.AppWorkflowMapping, error) { +func (impl *BuildPipelineSwitchServiceImpl) deleteOldCiPipelineAndWorkflowMappingBeforeSwitch(tx *pg.Tx, switchFromPipelineId int, switchFromType common.PipelineType, userId int32) (*appWorkflow.AppWorkflowMapping, error) { // 1) delete build pipelines // 2) delete app workflowMappings var err error pipelineId := switchFromPipelineId pipelineType := "" - if switchFromType == pipelineConfigBean.EXTERNAL { + if switchFromType == common.EXTERNAL { err = impl.deleteExternalCi(tx, switchFromPipelineId, userId) pipelineType = appWorkflow.WEBHOOK } else { diff --git a/pkg/pipeline/CIHandler_ent.go b/pkg/pipeline/CIHandler_ent.go new file mode 100644 index 0000000000..7e1986b518 --- /dev/null +++ b/pkg/pipeline/CIHandler_ent.go @@ -0,0 +1,31 @@ +package pipeline + +import ( + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/pkg/bean" + "github.com/devtron-labs/devtron/pkg/bean/common" +) + +type CiHandlerEnt interface { +} + +func (impl *CiHandlerImpl) updateRuntimeParamsForAutoCI(ciPipelineId int, runtimeParameters *common.RuntimeParameters) (*common.RuntimeParameters, error) { + return runtimeParameters, nil +} + +func (impl *CiHandlerImpl) updateResourceStatusInCache(ciWorkflowId int, podName string, namespace string, status string) { + //do nothing +} + +func (impl *CiHandlerImpl) getRuntimeParamsForBuildingManualTriggerHashes(ciTriggerRequest bean.CiTriggerRequest) *common.RuntimeParameters { + return common.NewRuntimeParameters() +} + +func (impl *CiHandlerImpl) getPipelineIdForTriggerView(pipeline *pipelineConfig.CiPipeline) (pipelineId int) { + if pipeline.ParentCiPipeline == 0 { + pipelineId = pipeline.Id + } else { + pipelineId = pipeline.ParentCiPipeline + } + return pipelineId +} diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index ba2618db49..230d35c522 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -26,6 +26,7 @@ import ( cdWorkflow2 "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" bean2 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/build/artifacts/imageTagging" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" "github.com/devtron-labs/devtron/pkg/cluster/adapter" bean3 "github.com/devtron-labs/devtron/pkg/cluster/bean" repository3 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" @@ -435,9 +436,9 @@ func (impl *CdHandlerImpl) GetCdBuildHistory(appId int, environmentId int, pipel return cdWorkflowArtifact, err } - var ciMaterialsArr []pipelineConfig.CiPipelineMaterialResponse + var ciMaterialsArr []buildBean.CiPipelineMaterialResponse for _, ciMaterial := range ciMaterials { - res := pipelineConfig.CiPipelineMaterialResponse{ + res := buildBean.CiPipelineMaterialResponse{ Id: ciMaterial.Id, GitMaterialId: ciMaterial.GitMaterialId, GitMaterialName: ciMaterial.GitMaterial.Name[strings.Index(ciMaterial.GitMaterial.Name, "-")+1:], @@ -689,9 +690,9 @@ func (impl *CdHandlerImpl) FetchCdWorkflowDetails(appId int, environmentId int, return types.WorkflowResponse{}, err } - var ciMaterialsArr []pipelineConfig.CiPipelineMaterialResponse + var ciMaterialsArr []buildBean.CiPipelineMaterialResponse for _, m := range ciMaterials { - res := pipelineConfig.CiPipelineMaterialResponse{ + res := buildBean.CiPipelineMaterialResponse{ Id: m.Id, GitMaterialId: m.GitMaterialId, GitMaterialName: m.GitMaterial.Name[strings.Index(m.GitMaterial.Name, "-")+1:], diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index 6be1e14ab0..313611d76c 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -32,6 +32,7 @@ import ( repository6 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/repository" "github.com/devtron-labs/devtron/pkg/build/pipeline" bean2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + common3 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" read2 "github.com/devtron-labs/devtron/pkg/chart/read" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/deployment/common" @@ -329,7 +330,7 @@ func (impl CiCdPipelineOrchestratorImpl) validateCiPipelineMaterial(ciPipelineMa func (impl CiCdPipelineOrchestratorImpl) getSkipMessage(ciPipeline *pipelineConfig.CiPipeline) string { switch ciPipeline.PipelineType { - case string(bean2.LINKED_CD): + case string(common3.LINKED_CD): return "“Sync with Environment”" default: return "“Linked Build Pipeline”" @@ -1010,7 +1011,7 @@ func (impl CiCdPipelineOrchestratorImpl) CreateCiConf(createRequest *bean.CiConf var pipelineMaterials []*pipelineConfig.CiPipelineMaterial for _, r := range ciPipeline.CiMaterial { - if ciPipeline.PipelineType == bean2.LINKED_CD { + if ciPipeline.PipelineType == common3.LINKED_CD { continue } material := &pipelineConfig.CiPipelineMaterial{ diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 5f05b28ef6..aa018f6b96 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -23,17 +23,18 @@ import ( "fmt" "github.com/devtron-labs/common-lib/utils" "github.com/devtron-labs/common-lib/utils/workFlow" - "github.com/devtron-labs/devtron/internal/sql/constants" + constants2 "github.com/devtron-labs/devtron/internal/sql/constants" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" bean6 "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/bean/common" "github.com/devtron-labs/devtron/pkg/build/artifacts/imageTagging" - bean4 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" "github.com/devtron-labs/devtron/pkg/cluster/adapter" - bean5 "github.com/devtron-labs/devtron/pkg/cluster/bean" + clusterBean "github.com/devtron-labs/devtron/pkg/cluster/bean" "github.com/devtron-labs/devtron/pkg/cluster/environment" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" - constants2 "github.com/devtron-labs/devtron/pkg/pipeline/constants" + "github.com/devtron-labs/devtron/pkg/pipeline/constants" util3 "github.com/devtron-labs/devtron/pkg/pipeline/util" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" "io/ioutil" @@ -52,11 +53,11 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" "github.com/devtron-labs/devtron/pkg/auth/user" "github.com/devtron-labs/devtron/pkg/cluster" - k8s2 "github.com/devtron-labs/devtron/pkg/k8s" - bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + k8sPkg "github.com/devtron-labs/devtron/pkg/k8s" + pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/types" - resourceGroup "github.com/devtron-labs/devtron/pkg/resourceGroup" + "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/util/rbac" errors2 "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/rest" @@ -67,7 +68,6 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/bean" - util2 "github.com/devtron-labs/devtron/util/event" "github.com/go-pg/pg" "go.uber.org/zap" ) @@ -76,16 +76,14 @@ type CiHandler interface { HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error - FetchMaterialsByPipelineId(pipelineId int, showAll bool) ([]pipelineConfig.CiPipelineMaterialResponse, error) - FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId int, gitMaterialId int, showAll bool) ([]pipelineConfig.CiPipelineMaterialResponse, error) + FetchMaterialsByPipelineId(pipelineId int, showAll bool) ([]buildBean.CiPipelineMaterialResponse, error) + FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId int, gitMaterialId int, showAll bool) ([]buildBean.CiPipelineMaterialResponse, error) FetchWorkflowDetails(appId int, pipelineId int, buildId int) (types.WorkflowResponse, error) FetchArtifactsForCiJob(buildId int) (*types.ArtifactsForCiJob, error) - //FetchBuildById(appId int, pipelineId int) (WorkflowResponse, error) CancelBuild(workflowId int, forceAbort bool) (int, error) GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, func() error, error) GetHistoricBuildLogs(workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) - //SyncWorkflows() error GetBuildHistory(pipelineId int, appId int, offset int, size int) ([]types.WorkflowResponse, error) DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) @@ -97,6 +95,7 @@ type CiHandler interface { FetchMaterialInfoByArtifactId(ciArtifactId int, envId int) (*types.GitTriggerInfoResponse, error) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error FetchCiStatusForTriggerViewForEnvironment(request resourceGroup.ResourceGroupingRequest, token string) ([]*pipelineConfig.CiWorkflowStatus, error) + CiHandlerEnt } type CiHandlerImpl struct { @@ -122,7 +121,7 @@ type CiHandlerImpl struct { customTagService CustomTagService appWorkflowRepository appWorkflow.AppWorkflowRepository config *types.CiConfig - k8sCommonService k8s2.K8sCommonService + k8sCommonService k8sPkg.K8sCommonService clusterService cluster.ClusterService blobConfigStorageService BlobStorageConfigService envService environment.EnvironmentService @@ -131,10 +130,11 @@ type CiHandlerImpl struct { func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, gitSensorClient gitSensor.Client, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, workflowService WorkflowService, ciLogService CiLogService, ciArtifactRepository repository.CiArtifactRepository, userService user.UserService, eventClient client.EventClient, eventFactory client.EventFactory, ciPipelineRepository pipelineConfig.CiPipelineRepository, - appListingRepository repository.AppListingRepository, K8sUtil *k8s.K8sServiceImpl, cdPipelineRepository pipelineConfig.PipelineRepository, enforcerUtil rbac.EnforcerUtil, resourceGroupService resourceGroup.ResourceGroupService, envRepository repository2.EnvironmentRepository, - imageTaggingService imageTagging.ImageTaggingService, k8sCommonService k8s2.K8sCommonService, clusterService cluster.ClusterService, blobConfigStorageService BlobStorageConfigService, appWorkflowRepository appWorkflow.AppWorkflowRepository, customTagService CustomTagService, - envService environment.EnvironmentService, - workFlowStageStatusService workflowStatus.WorkFlowStageStatusService) *CiHandlerImpl { + appListingRepository repository.AppListingRepository, cdPipelineRepository pipelineConfig.PipelineRepository, enforcerUtil rbac.EnforcerUtil, resourceGroupService resourceGroup.ResourceGroupService, envRepository repository2.EnvironmentRepository, + imageTaggingService imageTagging.ImageTaggingService, k8sCommonService k8sPkg.K8sCommonService, clusterService cluster.ClusterService, blobConfigStorageService BlobStorageConfigService, appWorkflowRepository appWorkflow.AppWorkflowRepository, customTagService CustomTagService, + envService environment.EnvironmentService, workFlowStageStatusService workflowStatus.WorkFlowStageStatusService, + K8sUtil *k8s.K8sServiceImpl, +) *CiHandlerImpl { cih := &CiHandlerImpl{ Logger: Logger, ciService: ciService, @@ -174,7 +174,7 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline func (impl *CiHandlerImpl) CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error { - //return if re-trigger feature is disabled + // return if re-trigger feature is disabled if !impl.config.WorkflowRetriesEnabled() { impl.Logger.Debug("CI re-trigger is disabled") return nil @@ -224,6 +224,14 @@ func (impl *CiHandlerImpl) reTriggerCi(retryCount int, refCiWorkflow *pipelineCo trigger := types.Trigger{} trigger.BuildTriggerObject(refCiWorkflow, ciMaterials, bean6.SYSTEM_USER_ID, true, nil, "") + + // updating runtime params + trigger.RuntimeParameters, err = impl.updateRuntimeParamsForAutoCI(trigger.PipelineId, trigger.RuntimeParameters) + if err != nil { + impl.Logger.Errorw("err, updateRuntimeParamsForAutoCI", "ciPipelineId", trigger.PipelineId, + "runtimeParameters", trigger.RuntimeParameters, "err", err) + return err + } _, err = impl.ciService.TriggerCiPipeline(trigger) if err != nil { @@ -277,7 +285,7 @@ func (impl *CiHandlerImpl) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTrigger impl.Logger.Errorw("err in getting ci_pipeline by ciPipelineMaterialId", "ciPipelineMaterialId", gitCiTriggerRequest.CiPipelineMaterial.Id, "err", err) return 0, err } - if ciPipeline.IsManual || ciPipeline.PipelineType == bean4.LINKED_CD.ToString() { + if ciPipeline.IsManual || ciPipeline.PipelineType == buildCommonBean.LINKED_CD.ToString() { impl.Logger.Debugw("not handling for manual pipeline or in case of linked cd", "pipelineId", ciPipeline.Id) return 0, err } @@ -297,6 +305,12 @@ func (impl *CiHandlerImpl) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTrigger for k, v := range gitCiTriggerRequest.ExtraEnvironmentVariables { runtimeParams = runtimeParams.AddSystemVariable(k, v) } + runtimeParams, err = impl.updateRuntimeParamsForAutoCI(ciPipeline.Id, runtimeParams) + if err != nil { + impl.Logger.Errorw("err, updateRuntimeParamsForAutoCI", "ciPipelineId", ciPipeline.Id, + "runtimeParameters", runtimeParams, "err", err) + return 0, err + } commitHashes, err := impl.buildAutomaticTriggerCommitHashes(ciMaterials, gitCiTriggerRequest) if err != nil { return 0, err @@ -329,7 +343,7 @@ func (impl *CiHandlerImpl) validateBuildSequence(gitCiTriggerRequest bean.GitCiT ciPipelineMaterial := gitCiTriggerRequest.CiPipelineMaterial - if ciPipelineMaterial.Type == string(constants.SOURCE_TYPE_BRANCH_FIXED) { + if ciPipelineMaterial.Type == string(constants2.SOURCE_TYPE_BRANCH_FIXED) { if ciPipelineMaterial.GitCommit.Date.Before(lastTriggeredBuild.GitTriggers[ciPipelineMaterial.Id].Date) { impl.Logger.Warnw("older commit cannot be built for pipeline", "pipelineId", pipelineId, "ciMaterial", gitCiTriggerRequest.CiPipelineMaterial.Id) isValid = false @@ -347,12 +361,12 @@ func (impl *CiHandlerImpl) RefreshMaterialByCiPipelineMaterialId(gitMaterialId i return refreshRes, err } -func (impl *CiHandlerImpl) FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId int, gitMaterialId int, showAll bool) ([]pipelineConfig.CiPipelineMaterialResponse, error) { +func (impl *CiHandlerImpl) FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId int, gitMaterialId int, showAll bool) ([]buildBean.CiPipelineMaterialResponse, error) { ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineIdAndGitMaterialId(pipelineId, gitMaterialId) if err != nil { impl.Logger.Errorw("ciMaterials fetch failed", "err", err) } - var ciPipelineMaterialResponses []pipelineConfig.CiPipelineMaterialResponse + var ciPipelineMaterialResponses []buildBean.CiPipelineMaterialResponse var responseMap = make(map[int]bool) ciMaterialHistoryMap := make(map[*pipelineConfig.CiPipelineMaterial]*gitSensor.MaterialChangeResp) @@ -369,13 +383,13 @@ func (impl *CiHandlerImpl) FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId impl.Logger.Debugw("commits for material ", "m", m, "commits: ", changesResp) if apiErr != nil { impl.Logger.Warnw("git sensor FetchChanges failed for material", "id", m.Id) - return []pipelineConfig.CiPipelineMaterialResponse{}, apiErr + return []buildBean.CiPipelineMaterialResponse{}, apiErr } ciMaterialHistoryMap[m] = changesResp } for k, v := range ciMaterialHistoryMap { - r := pipelineConfig.CiPipelineMaterialResponse{ + r := buildBean.CiPipelineMaterialResponse{ Id: k.Id, GitMaterialId: k.GitMaterialId, GitMaterialName: k.GitMaterial.Name[strings.Index(k.GitMaterial.Name, "-")+1:], @@ -398,10 +412,10 @@ func (impl *CiHandlerImpl) FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId regexMaterials, err := impl.ciPipelineMaterialRepository.GetRegexByPipelineId(pipelineId) if err != nil { impl.Logger.Errorw("regex ciMaterials fetch failed", "err", err) - return []pipelineConfig.CiPipelineMaterialResponse{}, err + return []buildBean.CiPipelineMaterialResponse{}, err } for _, k := range regexMaterials { - r := pipelineConfig.CiPipelineMaterialResponse{ + r := buildBean.CiPipelineMaterialResponse{ Id: k.Id, GitMaterialId: k.GitMaterialId, GitMaterialName: k.GitMaterial.Name[strings.Index(k.GitMaterial.Name, "-")+1:], @@ -424,12 +438,12 @@ func (impl *CiHandlerImpl) FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId return ciPipelineMaterialResponses, nil } -func (impl *CiHandlerImpl) FetchMaterialsByPipelineId(pipelineId int, showAll bool) ([]pipelineConfig.CiPipelineMaterialResponse, error) { +func (impl *CiHandlerImpl) FetchMaterialsByPipelineId(pipelineId int, showAll bool) ([]buildBean.CiPipelineMaterialResponse, error) { ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(pipelineId) if err != nil { impl.Logger.Errorw("ciMaterials fetch failed", "err", err) } - var ciPipelineMaterialResponses []pipelineConfig.CiPipelineMaterialResponse + var ciPipelineMaterialResponses []buildBean.CiPipelineMaterialResponse var responseMap = make(map[int]bool) ciMaterialHistoryMap := make(map[*pipelineConfig.CiPipelineMaterial]*gitSensor.MaterialChangeResp) @@ -452,7 +466,7 @@ func (impl *CiHandlerImpl) FetchMaterialsByPipelineId(pipelineId int, showAll bo } for k, v := range ciMaterialHistoryMap { - r := pipelineConfig.CiPipelineMaterialResponse{ + r := buildBean.CiPipelineMaterialResponse{ Id: k.Id, GitMaterialId: k.GitMaterialId, GitMaterialName: k.GitMaterial.Name[strings.Index(k.GitMaterial.Name, "-")+1:], @@ -478,7 +492,7 @@ func (impl *CiHandlerImpl) FetchMaterialsByPipelineId(pipelineId int, showAll bo return nil, err } for _, k := range regexMaterials { - r := pipelineConfig.CiPipelineMaterialResponse{ + r := buildBean.CiPipelineMaterialResponse{ Id: k.Id, GitMaterialId: k.GitMaterialId, GitMaterialName: k.GitMaterial.Name[strings.Index(k.GitMaterial.Name, "-")+1:], @@ -507,9 +521,9 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int if err != nil { impl.Logger.Errorw("ciMaterials fetch failed", "err", err) } - var ciPipelineMaterialResponses []pipelineConfig.CiPipelineMaterialResponse + var ciPipelineMaterialResponses []buildBean.CiPipelineMaterialResponse for _, m := range ciMaterials { - r := pipelineConfig.CiPipelineMaterialResponse{ + r := buildBean.CiPipelineMaterialResponse{ Id: m.Id, GitMaterialId: m.GitMaterialId, Type: string(m.Type), @@ -520,7 +534,7 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int } ciPipelineMaterialResponses = append(ciPipelineMaterialResponses, r) } - //this map contains artifactId -> array of tags of that artifact + // this map contains artifactId -> array of tags of that artifact imageTagsDataMap, err := impl.imageTaggingService.GetTagsDataMapByAppId(appId) if err != nil { impl.Logger.Errorw("error in fetching image tags with appId", "err", err, "appId", appId) @@ -531,6 +545,7 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int impl.Logger.Errorw("err", "err", err) return nil, err } + var workflowIds []int var artifactIds []int for _, w := range workFlows { @@ -544,7 +559,7 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int return nil, err } - //this map contains artifactId -> imageComment of that artifact + // this map contains artifactId -> imageComment of that artifact imageCommetnsDataMap, err := impl.imageTaggingService.GetImageCommentsDataMapByArtifactIds(artifactIds) if err != nil { impl.Logger.Errorw("error in fetching imageCommetnsDataMap", "err", err, "appId", appId, "artifactIds", artifactIds) @@ -586,25 +601,25 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int WorkflowExecutionStage: impl.workFlowStageStatusService.ConvertDBWorkflowStageToMap(allWfStagesDetail, w.Id, w.Status, w.PodStatus, w.Message, bean2.CI_WORKFLOW_TYPE.String(), w.StartedOn, w.FinishedOn), } - if w.Message == bean3.ImageTagUnavailableMessage { - customTag, err := impl.customTagService.GetCustomTagByEntityKeyAndValue(bean3.EntityTypeCiPipelineId, strconv.Itoa(w.CiPipelineId)) + if w.Message == pipelineConfigBean.ImageTagUnavailableMessage { + customTag, err := impl.customTagService.GetCustomTagByEntityKeyAndValue(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(w.CiPipelineId)) if err != nil && err != pg.ErrNoRows { - //err == pg.ErrNoRows should never happen + // err == pg.ErrNoRows should never happen return nil, err } appWorkflows, err := impl.appWorkflowRepository.FindWFCIMappingByCIPipelineId(w.CiPipelineId) if err != nil && err != pg.ErrNoRows { return nil, err } - wfResponse.AppWorkflowId = appWorkflows[0].AppWorkflowId //it is guaranteed there will always be 1 entry (in case of ci_pipeline_id) + wfResponse.AppWorkflowId = appWorkflows[0].AppWorkflowId // it is guaranteed there will always be 1 entry (in case of ci_pipeline_id) wfResponse.CustomTag = &bean2.CustomTagErrorResponse{ TagPattern: customTag.TagPattern, AutoIncreasingNumber: customTag.AutoIncreasingNumber, - Message: bean3.ImageTagUnavailableMessage, + Message: pipelineConfigBean.ImageTagUnavailableMessage, } } if imageTagsDataMap[w.CiArtifactId] != nil { - wfResponse.ImageReleaseTags = imageTagsDataMap[w.CiArtifactId] //if artifact is not yet created,empty list will be sent + wfResponse.ImageReleaseTags = imageTagsDataMap[w.CiArtifactId] // if artifact is not yet created,empty list will be sent } if imageCommetnsDataMap[w.CiArtifactId] != nil { wfResponse.ImageComment = imageCommetnsDataMap[w.CiArtifactId] @@ -620,7 +635,7 @@ func (impl *CiHandlerImpl) CancelBuild(workflowId int, forceAbort bool) (int, er impl.Logger.Errorw("error in finding ci-workflow by workflow id", "ciWorkflowId", workflowId, "err", err) return 0, err } - isExt := workflow.Namespace != constants2.DefaultCiWorkflowNamespace + isExt := workflow.Namespace != constants.DefaultCiWorkflowNamespace var env *repository2.Environment var restConfig *rest.Config if isExt { @@ -667,7 +682,7 @@ func (impl *CiHandlerImpl) CancelBuild(workflowId int, forceAbort bool) (int, er workflow.Status = cdWorkflow.WorkflowCancel if workflow.ExecutorType == cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM { workflow.PodStatus = "Failed" - workflow.Message = constants2.TERMINATE_MESSAGE + workflow.Message = constants.TERMINATE_MESSAGE } err = impl.ciService.UpdateCiWorkflowWithStage(workflow) if err != nil { @@ -710,7 +725,7 @@ func (impl *CiHandlerImpl) handleForceAbortCaseForCi(workflow *pipelineConfig.Ci func (impl *CiHandlerImpl) updateWorkflowForForceAbort(workflow *pipelineConfig.CiWorkflow) error { workflow.Status = cdWorkflow.WorkflowCancel workflow.PodStatus = string(bean.Failed) - workflow.Message = constants2.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE + workflow.Message = constants.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE err := impl.ciService.UpdateCiWorkflowWithStage(workflow) if err != nil { impl.Logger.Errorw("error in updating workflow status", "err", err) @@ -766,9 +781,9 @@ func (impl *CiHandlerImpl) FetchWorkflowDetails(appId int, pipelineId int, build return types.WorkflowResponse{}, err } - var ciMaterialsArr []pipelineConfig.CiPipelineMaterialResponse + var ciMaterialsArr []buildBean.CiPipelineMaterialResponse for _, m := range ciMaterials { - res := pipelineConfig.CiPipelineMaterialResponse{ + res := buildBean.CiPipelineMaterialResponse{ Id: m.Id, GitMaterialId: m.GitMaterialId, GitMaterialName: m.GitMaterial.Name[strings.Index(m.GitMaterial.Name, "-")+1:], @@ -794,6 +809,7 @@ func (impl *CiHandlerImpl) FetchWorkflowDetails(appId int, pipelineId int, build impl.ciWorkflowRepository.MigrateIsArtifactUploaded(workflow.Id, ciArtifact.IsArtifactUploaded) isArtifactUploaded = ciArtifact.IsArtifactUploaded } + wfStagesDetail, err := impl.workFlowStageStatusService.GetWorkflowStagesByWorkflowIdsAndWfType([]int{workflow.Id}, bean2.CI_WORKFLOW_TYPE.String()) if err != nil { impl.Logger.Errorw("error in fetching allWfStagesDetail", "err", err, "workflowId", workflow.Id) @@ -811,19 +827,19 @@ func (impl *CiHandlerImpl) FetchWorkflowDetails(appId int, pipelineId int, build CiPipelineId: workflow.CiPipelineId, Namespace: workflow.Namespace, LogLocation: workflow.LogLocation, - BlobStorageEnabled: workflow.BlobStorageEnabled, //TODO default value if value not found in db + BlobStorageEnabled: workflow.BlobStorageEnabled, // TODO default value if value not found in db GitTriggers: workflow.GitTriggers, CiMaterials: ciMaterialsArr, TriggeredBy: workflow.TriggeredBy, TriggeredByEmail: triggeredByUserEmailId, Artifact: ciArtifact.Image, - TargetPlatforms: utils.ConvertTargetPlatformStringToObject(ciArtifact.TargetPlatforms), ArtifactId: ciArtifact.Id, IsArtifactUploaded: isArtifactUploaded, EnvironmentId: workflow.EnvironmentId, EnvironmentName: environmentName, PipelineType: workflow.CiPipeline.PipelineType, PodName: workflow.PodName, + TargetPlatforms: utils.ConvertTargetPlatformStringToObject(ciArtifact.TargetPlatforms), WorkflowExecutionStage: impl.workFlowStageStatusService.ConvertDBWorkflowStageToMap(wfStagesDetail, workflow.Id, workflow.Status, workflow.PodStatus, workflow.Message, bean2.CI_WORKFLOW_TYPE.String(), workflow.StartedOn, workflow.FinishedOn), } return workflowResponse, nil @@ -864,7 +880,7 @@ func (impl *CiHandlerImpl) getWorkflowLogs(ciWorkflow *pipelineConfig.CiWorkflow if err != nil { return nil, nil, err } - var clusterBean bean5.ClusterBean + var clusterBean clusterBean.ClusterBean if env != nil && env.Cluster != nil { clusterBean = adapter.GetClusterBean(*env.Cluster) } @@ -927,8 +943,8 @@ func (impl *CiHandlerImpl) getLogsFromRepository(ciWorkflow *pipelineConfig.CiWo } useExternalBlobStorage := isExternalBlobStorageEnabled(isExt, impl.config.UseBlobStorageConfigInCiWorkflow) if useExternalBlobStorage { - //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - //from them else return. + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) if err != nil { impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) @@ -1014,8 +1030,8 @@ func (impl *CiHandlerImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId i impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig by clusterId", "err", err, "clusterId", envBean.ClusterId) return nil, err } - //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - //from them else return. + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) if err != nil { impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) @@ -1088,8 +1104,8 @@ func (impl *CiHandlerImpl) GetHistoricBuildLogs(workflowId int, ciWorkflow *pipe impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig by clusterId", "err", err, "clusterId", envBean.ClusterId) return nil, err } - //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - //from them else return. + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) if err != nil { impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) @@ -1119,7 +1135,7 @@ func ExtractWorkflowStatus(workflowStatus v1alpha1.WorkflowStatus) (string, stri podName := "" logLocation := "" for k, v := range workflowStatus.Nodes { - if v.TemplateName == bean3.CI_WORKFLOW_NAME { + if v.TemplateName == pipelineConfigBean.CI_WORKFLOW_NAME { if v.BoundaryID == "" { workflowName = k } else { @@ -1194,7 +1210,7 @@ func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus impl.Logger.Errorw("cannot get saved wf", "err", err) return 0, err } - + impl.updateResourceStatusInCache(workflowId, podName, savedWorkflow.Namespace, status) ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() ciArtifactLocation := fmt.Sprintf(ciArtifactLocationFormat, savedWorkflow.Id, savedWorkflow.Id) @@ -1210,12 +1226,12 @@ func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus } if savedWorkflow.ExecutorType == cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM && savedWorkflow.Status == cdWorkflow.WorkflowCancel { savedWorkflow.PodStatus = "Failed" - savedWorkflow.Message = constants2.TERMINATE_MESSAGE + savedWorkflow.Message = constants.TERMINATE_MESSAGE } savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time savedWorkflow.Name = workflowName - //savedWorkflow.LogLocation = "/ci-pipeline/" + strconv.Itoa(savedWorkflow.CiPipelineId) + "/workflow/" + strconv.Itoa(savedWorkflow.Id) + "/logs" //TODO need to fetch from workflow object - //savedWorkflow.LogLocation = logLocation // removed because we are saving log location at trigger + // savedWorkflow.LogLocation = "/ci-pipeline/" + strconv.Itoa(savedWorkflow.CiPipelineId) + "/workflow/" + strconv.Itoa(savedWorkflow.Id) + "/logs" //TODO need to fetch from workflow object + // savedWorkflow.LogLocation = logLocation // removed because we are saving log location at trigger savedWorkflow.CiArtifactLocation = ciArtifactLocation savedWorkflow.PodName = podName impl.Logger.Debugw("updating workflow ", "workflow", savedWorkflow) @@ -1238,7 +1254,7 @@ func (impl *CiHandlerImpl) sendCIFailEvent(savedWorkflow *pipelineConfig.CiWorkf impl.Logger.Warnw("ci failed for workflow: ", "wfId", savedWorkflow.Id) if extractErrorCode(savedWorkflow.Message) != workFlow.CiStageFailErrorCode { - go impl.WriteCIFailEvent(savedWorkflow) + impl.ciService.WriteCIFailEvent(savedWorkflow) } else { impl.Logger.Infof("Step failed notification received for wfID %d with message %s", savedWorkflow.Id, savedWorkflow.Message) } @@ -1257,27 +1273,10 @@ func extractErrorCode(msg string) int { return -1 } -func (impl *CiHandlerImpl) WriteCIFailEvent(ciWorkflow *pipelineConfig.CiWorkflow) { - event, _ := impl.eventFactory.Build(util2.Fail, &ciWorkflow.CiPipelineId, ciWorkflow.CiPipeline.AppId, nil, util2.CI) - material := &client.MaterialTriggerInfo{} - material.GitTriggers = ciWorkflow.GitTriggers - event.CiWorkflowRunnerId = ciWorkflow.Id - event.UserId = int(ciWorkflow.TriggeredBy) - event = impl.eventFactory.BuildExtraCIData(event, material) - event.CiArtifactId = 0 - _, evtErr := impl.eventClient.WriteNotificationEvent(event) - if evtErr != nil { - impl.Logger.Errorw("error in writing event", "err", evtErr) - } -} - func (impl *CiHandlerImpl) BuildPayload(ciWorkflow *pipelineConfig.CiWorkflow) *client.Payload { payload := &client.Payload{} payload.AppName = ciWorkflow.CiPipeline.App.AppName payload.PipelineName = ciWorkflow.CiPipeline.Name - //payload["buildName"] = ciWorkflow.Name - //payload["podStatus"] = ciWorkflow.PodStatus - //payload["message"] = ciWorkflow.Message return payload } @@ -1325,7 +1324,7 @@ func SetGitCommitValuesForBuildingCommitHash(ciMaterial *pipelineConfig.CiPipeli func (impl *CiHandlerImpl) buildManualTriggerCommitHashes(ciTriggerRequest bean.CiTriggerRequest) (map[int]pipelineConfig.GitCommit, *common.RuntimeParameters, error) { commitHashes := map[int]pipelineConfig.GitCommit{} - runtimeParams := common.NewRuntimeParameters() + runtimeParams := impl.getRuntimeParamsForBuildingManualTriggerHashes(ciTriggerRequest) for _, ciPipelineMaterial := range ciTriggerRequest.CiPipelineMaterial { pipeLineMaterialFromDb, err := impl.ciPipelineMaterialRepository.GetById(ciPipelineMaterial.Id) @@ -1335,7 +1334,7 @@ func (impl *CiHandlerImpl) buildManualTriggerCommitHashes(ciTriggerRequest bean. } pipelineType := pipeLineMaterialFromDb.Type - if pipelineType == constants.SOURCE_TYPE_BRANCH_FIXED { + if pipelineType == constants2.SOURCE_TYPE_BRANCH_FIXED { gitCommit, err := impl.BuildManualTriggerCommitHashesForSourceTypeBranchFix(ciPipelineMaterial, pipeLineMaterialFromDb) if err != nil { impl.Logger.Errorw("err", "err", err) @@ -1343,7 +1342,7 @@ func (impl *CiHandlerImpl) buildManualTriggerCommitHashes(ciTriggerRequest bean. } commitHashes[ciPipelineMaterial.Id] = gitCommit - } else if pipelineType == constants.SOURCE_TYPE_WEBHOOK { + } else if pipelineType == constants2.SOURCE_TYPE_WEBHOOK { gitCommit, extraEnvVariables, err := impl.BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial, pipeLineMaterialFromDb) if err != nil { impl.Logger.Errorw("err", "err", err) @@ -1490,12 +1489,7 @@ func (impl *CiHandlerImpl) FetchCiStatusForTriggerView(appId int) ([]*pipelineCo return ciWorkflowStatuses, err } for _, pipeline := range pipelines { - pipelineId := 0 - if pipeline.ParentCiPipeline == 0 { - pipelineId = pipeline.Id - } else { - pipelineId = pipeline.ParentCiPipeline - } + pipelineId := impl.getPipelineIdForTriggerView(pipeline) workflow, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflow(pipelineId) if err != nil && !util.IsErrNoRows(err) { impl.Logger.Errorw("err", "pipelineId", pipelineId, "err", err) @@ -1540,7 +1534,7 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId return &types.GitTriggerInfoResponse{}, err } - ciMaterialsArr := make([]pipelineConfig.CiPipelineMaterialResponse, 0) + ciMaterialsArr := make([]buildBean.CiPipelineMaterialResponse, 0) var triggeredByUserEmailId string //check workflow data only for non external builds if !ciPipeline.IsExternal { @@ -1596,7 +1590,7 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId history = append(history, _gitCommit) - res := pipelineConfig.CiPipelineMaterialResponse{ + res := buildBean.CiPipelineMaterialResponse{ Id: m.Id, GitMaterialId: m.GitMaterialId, GitMaterialName: m.GitMaterial.Name[strings.Index(m.GitMaterial.Name, "-")+1:], @@ -1618,6 +1612,7 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId //GitTriggers: workflow.GitTriggers, CiMaterials: ciMaterialsArr, TriggeredByEmail: triggeredByUserEmailId, + CiPipelineId: ciPipeline.Id, AppId: ciPipeline.AppId, AppName: deployDetail.AppName, EnvironmentId: deployDetail.EnvironmentId, @@ -1632,7 +1627,7 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId } func (impl *CiHandlerImpl) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error { - ciWorkflows, err := impl.ciWorkflowRepository.FindByStatusesIn([]string{constants2.Starting, constants2.Running}) + ciWorkflows, err := impl.ciWorkflowRepository.FindByStatusesIn([]string{constants.Starting, constants.Running}) if err != nil { impl.Logger.Errorw("error on fetching ci workflows", "err", err) return err @@ -1647,7 +1642,7 @@ func (impl *CiHandlerImpl) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuil var isExt bool var env *repository2.Environment var restConfig *rest.Config - if ciWorkflow.Namespace != constants2.DefaultCiWorkflowNamespace { + if ciWorkflow.Namespace != constants.DefaultCiWorkflowNamespace { isExt = true env, err = impl.envRepository.FindById(ciWorkflow.EnvironmentId) if err != nil { @@ -1680,7 +1675,7 @@ func (impl *CiHandlerImpl) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuil //if ci workflow is exists, check its pod if !isEligibleToMarkFailed { - ns := constants2.DefaultCiWorkflowNamespace + ns := constants.DefaultCiWorkflowNamespace if isExt { _, client, err = impl.k8sCommonService.GetCoreClientByClusterId(env.ClusterId) if err != nil { @@ -1706,8 +1701,8 @@ func (impl *CiHandlerImpl) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuil isPodDeleted = true } } else { - //check workflow status,get the status - if wf.Status == string(v1alpha1.WorkflowFailed) && wf.Message == constants2.POD_DELETED_MESSAGE { + // check workflow status,get the status + if wf.Status == string(v1alpha1.WorkflowFailed) && wf.Message == cdWorkflow.POD_DELETED_MESSAGE { isPodDeleted = true } } @@ -1718,7 +1713,7 @@ func (impl *CiHandlerImpl) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuil ciWorkflow.PodStatus = "Failed" if isPodDeleted { ciWorkflow.Message = cdWorkflow.POD_DELETED_MESSAGE - //error logging handled inside handlePodDeleted + // error logging handled inside handlePodDeleted impl.handlePodDeleted(ciWorkflow) } else { ciWorkflow.Message = "marked failed by job" @@ -1761,7 +1756,7 @@ func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewForEnvironment(request res if err != nil { return nil, err } - //override appIds if already provided app group id in request. + // override appIds if already provided app group id in request. request.ResourceIds = appIds } if len(request.ResourceIds) > 0 { @@ -1794,7 +1789,7 @@ func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewForEnvironment(request res if len(ciPipelineIds) == 0 { return ciWorkflowStatuses, nil } - //authorization block starts here + // authorization block starts here var appObjectArr []string objects := impl.enforcerUtil.GetAppObjectByCiPipelineIds(ciPipelineIds) ciPipelineIds = []int{} @@ -1803,17 +1798,12 @@ func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewForEnvironment(request res } appResults, _ := request.CheckAuthBatch(token, appObjectArr, []string{}) for _, ciPipeline := range ciPipelines { - appObject := objects[ciPipeline.Id] //here only app permission have to check + appObject := objects[ciPipeline.Id] // here only app permission have to check if !appResults[appObject] { - //if user unauthorized, skip items + // if user unauthorized, skip items continue } - ciPipelineId := 0 - if ciPipeline.ParentCiPipeline == 0 { - ciPipelineId = ciPipeline.Id - } else { - ciPipelineId = ciPipeline.ParentCiPipeline - } + ciPipelineId := impl.getPipelineIdForTriggerView(ciPipeline) ciPipelineIds = append(ciPipelineIds, ciPipelineId) } if len(ciPipelineIds) == 0 { diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index d5282add0b..da1883339f 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -32,7 +32,8 @@ import ( bean4 "github.com/devtron-labs/devtron/pkg/attributes/bean" "github.com/devtron-labs/devtron/pkg/bean/common" "github.com/devtron-labs/devtron/pkg/build/pipeline" - bean6 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + common2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" repository6 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/pipeline/adapter" pipelineConst "github.com/devtron-labs/devtron/pkg/pipeline/constants" @@ -79,6 +80,7 @@ import ( type CiService interface { TriggerCiPipeline(trigger types.Trigger) (int, error) GetCiMaterials(pipelineId int, ciMaterials []*pipelineConfig.CiPipelineMaterial) ([]*pipelineConfig.CiPipelineMaterial, error) + WriteCIFailEvent(ciWorkflow *pipelineConfig.CiWorkflow) SaveCiWorkflowWithStage(wf *pipelineConfig.CiWorkflow) error UpdateCiWorkflowWithStage(wf *pipelineConfig.CiWorkflow) error } @@ -188,7 +190,7 @@ func (impl *CiServiceImpl) GetCiMaterials(pipelineId int, ciMaterials []*pipelin func (impl *CiServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, workflowRequest *types.WorkflowRequest) error { // externalCi artifact is meant only for CI_JOB - if trigger.PipelineType != string(bean6.CI_JOB) { + if trigger.PipelineType != string(common2.CI_JOB) { return nil } @@ -247,7 +249,7 @@ func (impl *CiServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, } } - if trigger.PipelineType == string(bean6.CI_JOB) && len(ciMaterials) != 0 && !exists && externalCiArtifact == "" { + if trigger.PipelineType == string(common2.CI_JOB) && len(ciMaterials) != 0 && !exists && externalCiArtifact == "" { ciMaterials[0].GitMaterial = nil ciMaterials[0].GitMaterialId = 0 } @@ -490,7 +492,7 @@ func (impl *CiServiceImpl) getEnvironmentForJob(pipeline *pipelineConfig.CiPipel func (impl *CiServiceImpl) WriteCITriggerEvent(trigger types.Trigger, pipeline *pipelineConfig.CiPipeline, workflowRequest *types.WorkflowRequest) { event, _ := impl.eventFactory.Build(util2.Trigger, &pipeline.Id, pipeline.AppId, nil, util2.CI) - material := &client.MaterialTriggerInfo{} + material := &buildBean.MaterialTriggerInfo{} material.GitTriggers = trigger.CommitHashes @@ -663,7 +665,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. var dockerfilePath string var dockerRepository string var checkoutPath string - var ciBuildConfigBean *bean6.CiBuildConfigBean + var ciBuildConfigBean *buildBean.CiBuildConfigBean dockerRegistry := &repository3.DockerArtifactStore{} ciBaseBuildConfigEntity := ciTemplate.CiBuildConfig ciBaseBuildConfigBean, err := adapter.ConvertDbBuildConfigToBean(ciBaseBuildConfigEntity) @@ -774,13 +776,13 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. ciBuildConfigBean.PipelineType = trigger.PipelineType - if ciBuildConfigBean.CiBuildType == bean6.SELF_DOCKERFILE_BUILD_TYPE || ciBuildConfigBean.CiBuildType == bean6.MANAGED_DOCKERFILE_BUILD_TYPE { + if ciBuildConfigBean.CiBuildType == buildBean.SELF_DOCKERFILE_BUILD_TYPE || ciBuildConfigBean.CiBuildType == buildBean.MANAGED_DOCKERFILE_BUILD_TYPE { ciBuildConfigBean.DockerBuildConfig.BuildContext = filepath.Join(buildContextCheckoutPath, ciBuildConfigBean.DockerBuildConfig.BuildContext) dockerBuildConfig := ciBuildConfigBean.DockerBuildConfig dockerfilePath = filepath.Join(checkoutPath, dockerBuildConfig.DockerfilePath) dockerBuildConfig.DockerfilePath = dockerfilePath checkoutPath = dockerfilePath[:strings.LastIndex(dockerfilePath, "/")+1] - } else if ciBuildConfigBean.CiBuildType == bean6.BUILDPACK_BUILD_TYPE { + } else if ciBuildConfigBean.CiBuildType == buildBean.BUILDPACK_BUILD_TYPE { buildPackConfig := ciBuildConfigBean.BuildPackConfig checkoutPath = filepath.Join(checkoutPath, buildPackConfig.ProjectPath) } @@ -1135,6 +1137,20 @@ func _getTruncatedImageTag(imageTag string) string { } } +func (impl *CiServiceImpl) WriteCIFailEvent(ciWorkflow *pipelineConfig.CiWorkflow) { + event, _ := impl.eventFactory.Build(util2.Fail, &ciWorkflow.CiPipelineId, ciWorkflow.CiPipeline.AppId, nil, util2.CI) + material := &buildBean.MaterialTriggerInfo{} + material.GitTriggers = ciWorkflow.GitTriggers + event.CiWorkflowRunnerId = ciWorkflow.Id + event.UserId = int(ciWorkflow.TriggeredBy) + event = impl.eventFactory.BuildExtraCIData(event, material) + event.CiArtifactId = 0 + _, evtErr := impl.eventClient.WriteNotificationEvent(event) + if evtErr != nil { + impl.Logger.Errorw("error in writing event", "err", evtErr) + } +} + func (impl *CiServiceImpl) SaveCiWorkflowWithStage(wf *pipelineConfig.CiWorkflow) error { // implementation tx, err := impl.transactionManager.StartTx() diff --git a/pkg/pipeline/WorkflowService.go b/pkg/pipeline/WorkflowService.go index 3cc2215867..191b973c75 100644 --- a/pkg/pipeline/WorkflowService.go +++ b/pkg/pipeline/WorkflowService.go @@ -28,7 +28,7 @@ import ( "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" - bean2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/config/read" v1 "github.com/devtron-labs/devtron/pkg/infraConfig/bean/v1" @@ -218,7 +218,7 @@ func (impl *WorkflowServiceImpl) createWorkflowTemplate(workflowRequest *types.W func (impl *WorkflowServiceImpl) shouldAddExistingCmCsInWorkflow(workflowRequest *types.WorkflowRequest) bool { // CmCs are not added for CI_JOB if IgnoreCmCsInCiJob is true - if workflowRequest.CiPipelineType == string(bean2.CI_JOB) && impl.ciCdConfig.IgnoreCmCsInCiJob { + if workflowRequest.CiPipelineType == string(common.CI_JOB) && impl.ciCdConfig.IgnoreCmCsInCiJob { return false } return true diff --git a/pkg/pipeline/adapter/adapter.go b/pkg/pipeline/adapter/adapter.go index badb5c0f3e..d480731a2a 100644 --- a/pkg/pipeline/adapter/adapter.go +++ b/pkg/pipeline/adapter/adapter.go @@ -23,6 +23,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/ciPipeline" "github.com/devtron-labs/devtron/pkg/bean" bean2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" bean3 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" @@ -190,7 +191,7 @@ func mergeMap(oldDockerArgs map[string]string, ciLevelDockerArgs map[string]stri // IsLinkedCD will return if the pipelineConfig.CiPipeline is a Linked CD func IsLinkedCD(ci pipelineConfig.CiPipeline) bool { - return ci.ParentCiPipeline != 0 && ci.PipelineType == string(bean2.LINKED_CD) + return ci.ParentCiPipeline != 0 && ci.PipelineType == string(common.LINKED_CD) } // IsLinkedCI will return if the pipelineConfig.CiPipeline is a Linked CI @@ -199,7 +200,7 @@ func IsLinkedCI(ci *pipelineConfig.CiPipeline) bool { return false } return ci.ParentCiPipeline != 0 && - ci.PipelineType == string(bean2.LINKED) + ci.PipelineType == string(common.LINKED) } // IsCIJob will return if the pipelineConfig.CiPipeline is a CI JOB @@ -207,7 +208,7 @@ func IsCIJob(ci *pipelineConfig.CiPipeline) bool { if ci == nil { return false } - return ci.PipelineType == string(bean2.CI_JOB) + return ci.PipelineType == string(common.CI_JOB) } // GetSourceCiDownStreamResponse will take the models []bean.LinkedCIDetails and []pipelineConfig.CdWorkflowRunner (for last deployment status) and generate the []CiPipeline.SourceCiDownStreamResponse diff --git a/pkg/pipeline/bean/CdHandlerBean.go b/pkg/pipeline/bean/CdHandlerBean.go index 4b6480e365..5dc408c226 100644 --- a/pkg/pipeline/bean/CdHandlerBean.go +++ b/pkg/pipeline/bean/CdHandlerBean.go @@ -4,37 +4,38 @@ import ( "github.com/devtron-labs/common-lib/utils/bean" "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" bean2 "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus/bean" "time" ) type CdWorkflowWithArtifact struct { - Id int `json:"id"` - CdWorkflowId int `json:"cd_workflow_id"` - Name string `json:"name"` - Status string `json:"status"` - PodStatus string `json:"pod_status"` - Message string `json:"message"` - StartedOn time.Time `json:"started_on"` - FinishedOn time.Time `json:"finished_on"` - PipelineId int `json:"pipeline_id"` - Namespace string `json:"namespace"` - LogFilePath string `json:"log_file_path"` - TriggeredBy int32 `json:"triggered_by"` - EmailId string `json:"email_id"` - Image string `json:"image"` - TargetPlatforms []*bean.TargetPlatform `json:"targetPlatforms"` - MaterialInfo string `json:"material_info,omitempty"` - DataSource string `json:"data_source,omitempty"` - CiArtifactId int `json:"ci_artifact_id,omitempty"` - IsArtifactUploaded bool `json:"isArtifactUploaded"` - WorkflowType string `json:"workflow_type,omitempty"` - ExecutorType string `json:"executor_type,omitempty"` - BlobStorageEnabled bool `json:"blobStorageEnabled"` - GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` - CiMaterials []pipelineConfig.CiPipelineMaterialResponse `json:"ciMaterials"` - ImageReleaseTags []*repository.ImageTag `json:"imageReleaseTags"` - ImageComment *repository.ImageComment `json:"imageComment"` - RefCdWorkflowRunnerId int `json:"referenceCdWorkflowRunnerId"` - WorkflowExecutionStage map[string][]*bean2.WorkflowStageDto `json:"workflowExecutionStages"` + Id int `json:"id"` + CdWorkflowId int `json:"cd_workflow_id"` + Name string `json:"name"` + Status string `json:"status"` + PodStatus string `json:"pod_status"` + Message string `json:"message"` + StartedOn time.Time `json:"started_on"` + FinishedOn time.Time `json:"finished_on"` + PipelineId int `json:"pipeline_id"` + Namespace string `json:"namespace"` + LogFilePath string `json:"log_file_path"` + TriggeredBy int32 `json:"triggered_by"` + EmailId string `json:"email_id"` + Image string `json:"image"` + TargetPlatforms []*bean.TargetPlatform `json:"targetPlatforms"` + MaterialInfo string `json:"material_info,omitempty"` + DataSource string `json:"data_source,omitempty"` + CiArtifactId int `json:"ci_artifact_id,omitempty"` + IsArtifactUploaded bool `json:"isArtifactUploaded"` + WorkflowType string `json:"workflow_type,omitempty"` + ExecutorType string `json:"executor_type,omitempty"` + BlobStorageEnabled bool `json:"blobStorageEnabled"` + GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` + CiMaterials []buildBean.CiPipelineMaterialResponse `json:"ciMaterials"` + ImageReleaseTags []*repository.ImageTag `json:"imageReleaseTags"` + ImageComment *repository.ImageComment `json:"imageComment"` + RefCdWorkflowRunnerId int `json:"referenceCdWorkflowRunnerId"` + WorkflowExecutionStage map[string][]*bean2.WorkflowStageDto `json:"workflowExecutionStages"` } diff --git a/pkg/pipeline/constants/constants.go b/pkg/pipeline/constants/constants.go index 0d5cd5bcab..243fd6679e 100644 --- a/pkg/pipeline/constants/constants.go +++ b/pkg/pipeline/constants/constants.go @@ -68,7 +68,6 @@ const ( const DefaultCiWorkflowNamespace = "devtron-ci" const Running = "Running" const Starting = "Starting" -const POD_DELETED_MESSAGE = "pod deleted" const TERMINATE_MESSAGE = "workflow shutdown with strategy: Terminate" const FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE = "workflow shutdown with strategy: Force Abort" const POD_TIMEOUT_MESSAGE = "Pod was active on the node longer than the specified deadline" diff --git a/pkg/pipeline/types/CiCdConfig.go b/pkg/pipeline/types/CiCdConfig.go index 961bd9f26f..d6e98efc5c 100644 --- a/pkg/pipeline/types/CiCdConfig.go +++ b/pkg/pipeline/types/CiCdConfig.go @@ -27,6 +27,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/pkg/bean/common" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/pipeline/bean" v12 "k8s.io/api/core/v1" @@ -550,17 +551,18 @@ type ArtifactsForCiJob struct { } type GitTriggerInfoResponse struct { - CiMaterials []pipelineConfig.CiPipelineMaterialResponse `json:"ciMaterials"` - TriggeredByEmail string `json:"triggeredByEmail"` - LastDeployedTime string `json:"lastDeployedTime,omitempty"` - AppId int `json:"appId"` - AppName string `json:"appName"` - EnvironmentId int `json:"environmentId"` - EnvironmentName string `json:"environmentName"` - Default bool `json:"default,omitempty"` - ImageTaggingData ImageTaggingResponseDTO `json:"imageTaggingData"` - Image string `json:"image"` - TargetPlatforms []*bean2.TargetPlatform `json:"targetPlatforms"` + CiMaterials []buildBean.CiPipelineMaterialResponse `json:"ciMaterials"` + TriggeredByEmail string `json:"triggeredByEmail"` + LastDeployedTime string `json:"lastDeployedTime,omitempty"` + CiPipelineId int `json:"ciPipelineId"` + AppId int `json:"appId"` + AppName string `json:"appName"` + EnvironmentId int `json:"environmentId"` + EnvironmentName string `json:"environmentName"` + Default bool `json:"default,omitempty"` + ImageTaggingData ImageTaggingResponseDTO `json:"imageTaggingData"` + Image string `json:"image"` + TargetPlatforms []*bean2.TargetPlatform `json:"targetPlatforms"` } type Trigger struct { diff --git a/pkg/pipeline/types/Workflow.go b/pkg/pipeline/types/Workflow.go index dca45ab8fc..dceb9a03b0 100644 --- a/pkg/pipeline/types/Workflow.go +++ b/pkg/pipeline/types/Workflow.go @@ -31,6 +31,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" bean2 "github.com/devtron-labs/devtron/pkg/bean" bean5 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" repository4 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" infraBean "github.com/devtron-labs/devtron/pkg/infraConfig/bean/v1" "github.com/devtron-labs/devtron/pkg/pipeline/bean" @@ -649,37 +650,37 @@ type ChildCdMetadata struct { } type WorkflowResponse struct { - Id int `json:"id"` - Name string `json:"name"` - Status string `json:"status"` - PodStatus string `json:"podStatus"` - Message string `json:"message"` - StartedOn time.Time `json:"startedOn"` - FinishedOn time.Time `json:"finishedOn"` - CiPipelineId int `json:"ciPipelineId"` - Namespace string `json:"namespace"` - LogLocation string `json:"logLocation"` - BlobStorageEnabled bool `json:"blobStorageEnabled"` - GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` - CiMaterials []pipelineConfig.CiPipelineMaterialResponse `json:"ciMaterials"` - TriggeredBy int32 `json:"triggeredBy"` - Artifact string `json:"artifact"` - TriggeredByEmail string `json:"triggeredByEmail"` - Stage string `json:"stage"` - ArtifactId int `json:"artifactId"` - IsArtifactUploaded bool `json:"isArtifactUploaded"` - IsVirtualEnvironment bool `json:"isVirtualEnvironment"` - PodName string `json:"podName"` - EnvironmentId int `json:"environmentId"` - EnvironmentName string `json:"environmentName"` - ImageReleaseTags []*repository3.ImageTag `json:"imageReleaseTags"` - ImageComment *repository3.ImageComment `json:"imageComment"` - AppWorkflowId int `json:"appWorkflowId"` - CustomTag *bean3.CustomTagErrorResponse `json:"customTag,omitempty"` - PipelineType string `json:"pipelineType"` - ReferenceWorkflowId int `json:"referenceWorkflowId"` - TargetPlatforms []*bean7.TargetPlatform `json:"targetPlatforms"` - WorkflowExecutionStage map[string][]*bean6.WorkflowStageDto `json:"workflowExecutionStages"` + Id int `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + PodStatus string `json:"podStatus"` + Message string `json:"message"` + StartedOn time.Time `json:"startedOn"` + FinishedOn time.Time `json:"finishedOn"` + CiPipelineId int `json:"ciPipelineId"` + Namespace string `json:"namespace"` + LogLocation string `json:"logLocation"` + BlobStorageEnabled bool `json:"blobStorageEnabled"` + GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` + CiMaterials []buildBean.CiPipelineMaterialResponse `json:"ciMaterials"` + TriggeredBy int32 `json:"triggeredBy"` + Artifact string `json:"artifact"` + TriggeredByEmail string `json:"triggeredByEmail"` + Stage string `json:"stage"` + ArtifactId int `json:"artifactId"` + IsArtifactUploaded bool `json:"isArtifactUploaded"` + IsVirtualEnvironment bool `json:"isVirtualEnvironment"` + PodName string `json:"podName"` + EnvironmentId int `json:"environmentId"` + EnvironmentName string `json:"environmentName"` + ImageReleaseTags []*repository3.ImageTag `json:"imageReleaseTags"` + ImageComment *repository3.ImageComment `json:"imageComment"` + AppWorkflowId int `json:"appWorkflowId"` + CustomTag *bean3.CustomTagErrorResponse `json:"customTag,omitempty"` + PipelineType string `json:"pipelineType"` + ReferenceWorkflowId int `json:"referenceWorkflowId"` + TargetPlatforms []*bean7.TargetPlatform `json:"targetPlatforms"` + WorkflowExecutionStage map[string][]*bean6.WorkflowStageDto `json:"workflowExecutionStages"` } type ConfigMapSecretDto struct { diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index d87f6bfd97..515ab4cafc 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -38,6 +38,7 @@ import ( bean7 "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/build/artifacts" bean5 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/manifest" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" @@ -830,7 +831,7 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext trigger var pluginArtifacts []*repository.CiArtifact for registry, artifacts := range request.PluginRegistryArtifactDetails { for _, image := range artifacts { - if pipelineModal.PipelineType == string(bean5.CI_JOB) && image == "" { + if pipelineModal.PipelineType == string(common.CI_JOB) && image == "" { continue } pluginArtifact := &repository.CiArtifact{ @@ -994,7 +995,7 @@ func (impl *WorkflowDagExecutorImpl) HandleCiStepFailedEvent(ciPipelineId int, r func (impl *WorkflowDagExecutorImpl) WriteCiStepFailedEvent(pipeline *pipelineConfig.CiPipeline, request *bean2.CiArtifactWebhookRequest, ciWorkflow *pipelineConfig.CiWorkflow) { event, _ := impl.eventFactory.Build(util2.Fail, &pipeline.Id, pipeline.AppId, nil, util2.CI) - material := &client.MaterialTriggerInfo{} + material := &bean5.MaterialTriggerInfo{} material.GitTriggers = ciWorkflow.GitTriggers event.CiWorkflowRunnerId = ciWorkflow.Id event.UserId = int(ciWorkflow.TriggeredBy) diff --git a/wire_gen.go b/wire_gen.go index 679957d5ec..ff2d593a71 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -685,7 +685,7 @@ func InitializeApp() (*App, error) { } imageTaggingServiceImpl := imageTagging.NewImageTaggingServiceImpl(imageTaggingRepositoryImpl, imageTaggingReadServiceImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, sugaredLogger) blobStorageConfigServiceImpl := pipeline.NewBlobStorageConfigServiceImpl(sugaredLogger, k8sServiceImpl, ciCdConfig) - ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, workflowServiceImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, k8sServiceImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, environmentServiceImpl, workFlowStageStatusServiceImpl) + ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, workflowServiceImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, environmentServiceImpl, workFlowStageStatusServiceImpl, k8sServiceImpl) gitWebhookRepositoryImpl := repository23.NewGitWebhookRepositoryImpl(db) gitWebhookServiceImpl := gitWebhook.NewGitWebhookServiceImpl(sugaredLogger, ciHandlerImpl, gitWebhookRepositoryImpl) gitWebhookRestHandlerImpl := restHandler.NewGitWebhookRestHandlerImpl(sugaredLogger, gitWebhookServiceImpl) From 7e7928d41bed9784cdbae7ef48d8e9f4dac50ffc Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 1 Apr 2025 19:48:13 +0530 Subject: [PATCH 02/17] common constants create --- .../pipelineConfig/CiPipelineRepository.go | 8 ++--- pkg/app/AppListingService.go | 4 +-- .../devtronApps/PreStageTriggerService.go | 4 +-- pkg/pipeline/CiCdPipelineOrchestrator.go | 9 +++--- pkg/pipeline/CiService.go | 7 ++--- pkg/pipeline/constants/constants.go | 31 ------------------- pkg/workflow/dag/WorkflowDagExecutor.go | 7 ++--- 7 files changed, 18 insertions(+), 52 deletions(-) diff --git a/internal/sql/repository/pipelineConfig/CiPipelineRepository.go b/internal/sql/repository/pipelineConfig/CiPipelineRepository.go index fb90f43255..c6adb8b7a8 100644 --- a/internal/sql/repository/pipelineConfig/CiPipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/CiPipelineRepository.go @@ -21,8 +21,8 @@ import ( "fmt" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/ciPipeline" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" - "github.com/devtron-labs/devtron/pkg/pipeline/constants" "github.com/devtron-labs/devtron/pkg/sql" "github.com/devtron-labs/devtron/util/response/pagination" "github.com/go-pg/pg" @@ -641,7 +641,7 @@ func (impl *CiPipelineRepositoryImpl) FindLinkedCiCount(ciPipelineId int) (int, pipeline := &CiPipeline{} cnt, err := impl.dbConnection.Model(pipeline). Where("parent_ci_pipeline = ?", ciPipelineId). - Where("ci_pipeline_type != ?", constants.LINKED_CD). + Where("ci_pipeline_type != ?", buildCommonBean.LINKED_CD). Where("deleted = ?", false). Count() if err == pg.ErrNoRows { @@ -656,7 +656,7 @@ func (impl *CiPipelineRepositoryImpl) GetLinkedCiPipelines(ctx context.Context, var linkedCIPipelines []*CiPipeline err := impl.dbConnection.Model(&linkedCIPipelines). Where("parent_ci_pipeline = ?", ciPipelineId). - Where("ci_pipeline_type != ?", constants.LINKED_CD). + Where("ci_pipeline_type != ?", buildCommonBean.LINKED_CD). Where("deleted = ?", false). Select() if err != nil { @@ -694,7 +694,7 @@ func (impl *CiPipelineRepositoryImpl) GetDownStreamInfo(ctx context.Context, sou JoinOn("e.active = ?", true). // constrains Where("ci_pipeline.parent_ci_pipeline = ?", sourceCiPipelineId). - Where("ci_pipeline.ci_pipeline_type != ?", constants.LINKED_CD). + Where("ci_pipeline.ci_pipeline_type != ?", buildCommonBean.LINKED_CD). Where("ci_pipeline.deleted = ?", false) // app name filtering with lower case if len(appNameMatch) != 0 { diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index ad2659551b..67bf66bc28 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -25,13 +25,13 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" read4 "github.com/devtron-labs/devtron/pkg/app/appDetails/read" userrepository "github.com/devtron-labs/devtron/pkg/auth/user/repository" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" ciConfig "github.com/devtron-labs/devtron/pkg/build/pipeline/read" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/read" "github.com/devtron-labs/devtron/pkg/dockerRegistry" - "github.com/devtron-labs/devtron/pkg/pipeline/constants" errors2 "github.com/juju/errors" "go.opentelemetry.io/otel" "golang.org/x/exp/slices" @@ -607,7 +607,7 @@ func (impl AppListingServiceImpl) setIpAccessProvidedData(ctx context.Context, a } if ciPipeline != nil && ciPipeline.CiTemplate != nil && len(*ciPipeline.CiTemplate.DockerRegistryId) > 0 { - if !ciPipeline.IsExternal || ciPipeline.ParentCiPipeline != 0 && ciPipeline.PipelineType != string(constants.LINKED_CD) { + if !ciPipeline.IsExternal || ciPipeline.ParentCiPipeline != 0 && ciPipeline.PipelineType != string(buildCommonBean.LINKED_CD) { appDetailContainer.IsExternalCi = false } // get dockerRegistryId starts diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go index 580bbf4da9..ec80a7cf19 100644 --- a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go @@ -35,6 +35,7 @@ import ( bean7 "github.com/devtron-labs/devtron/pkg/auth/user/bean" bean4 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/bean/common" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" repository4 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" bean5 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" adapter2 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/adapter" @@ -43,7 +44,6 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/adapter" pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" - "github.com/devtron-labs/devtron/pkg/pipeline/constants" repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/plugin" @@ -532,7 +532,7 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow return nil, err } ciProjectDetail.CommitTime = commitTime.Format(bean4.LayoutRFC3339) - } else if ciPipeline.PipelineType == string(constants.CI_JOB) { + } else if ciPipeline.PipelineType == string(buildCommonBean.CI_JOB) { // This has been done to resolve unmarshalling issue in ci-runner, in case of no commit time(eg- polling container images) ciProjectDetail.CommitTime = time.Time{}.Format(bean4.LayoutRFC3339) } else { diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index 313611d76c..46a08da9d5 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -32,13 +32,12 @@ import ( repository6 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/repository" "github.com/devtron-labs/devtron/pkg/build/pipeline" bean2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" - common3 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" read2 "github.com/devtron-labs/devtron/pkg/chart/read" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/common/read" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" - constants3 "github.com/devtron-labs/devtron/pkg/pipeline/constants" util4 "github.com/devtron-labs/devtron/pkg/pipeline/util" "github.com/devtron-labs/devtron/pkg/plugin" "github.com/devtron-labs/devtron/util/beHelper" @@ -330,7 +329,7 @@ func (impl CiCdPipelineOrchestratorImpl) validateCiPipelineMaterial(ciPipelineMa func (impl CiCdPipelineOrchestratorImpl) getSkipMessage(ciPipeline *pipelineConfig.CiPipeline) string { switch ciPipeline.PipelineType { - case string(common3.LINKED_CD): + case string(buildCommonBean.LINKED_CD): return "“Sync with Environment”" default: return "“Linked Build Pipeline”" @@ -1011,7 +1010,7 @@ func (impl CiCdPipelineOrchestratorImpl) CreateCiConf(createRequest *bean.CiConf var pipelineMaterials []*pipelineConfig.CiPipelineMaterial for _, r := range ciPipeline.CiMaterial { - if ciPipeline.PipelineType == common3.LINKED_CD { + if ciPipeline.PipelineType == buildCommonBean.LINKED_CD { continue } material := &pipelineConfig.CiPipelineMaterial{ @@ -2504,7 +2503,7 @@ func (impl *CiCdPipelineOrchestratorImpl) GetWorkflowCacheConfig(appType helper. if appType == helper.Job { return util4.GetWorkflowCacheConfig(pipelineWorkflowCacheConfig, impl.workflowCacheConfig.IgnoreJob) } else { - if pipelineType == string(constants3.CI_JOB) { + if pipelineType == string(buildCommonBean.CI_JOB) { return util4.GetWorkflowCacheConfigWithBackwardCompatibility(pipelineWorkflowCacheConfig, impl.ciConfig.WorkflowCacheConfig, impl.workflowCacheConfig.IgnoreCIJob, impl.ciConfig.SkipCiJobBuildCachePushPull) } else { return util4.GetWorkflowCacheConfigWithBackwardCompatibility(pipelineWorkflowCacheConfig, impl.ciConfig.WorkflowCacheConfig, impl.workflowCacheConfig.IgnoreCI, impl.ciConfig.IgnoreDockerCacheForCI) diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index da1883339f..a2bdd0a58e 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -36,7 +36,6 @@ import ( common2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" repository6 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/pipeline/adapter" - pipelineConst "github.com/devtron-labs/devtron/pkg/pipeline/constants" "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" bean2 "github.com/devtron-labs/devtron/pkg/plugin/bean" @@ -195,7 +194,7 @@ func (impl *CiServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, } // checking if user has given run time parameters for externalCiArtifact, if given then sending git material to Ci-Runner - externalCiArtifact, exists := trigger.RuntimeParameters.GetGlobalRuntimeVariables()[pipelineConst.ExtraEnvVarExternalCiArtifactKey] + externalCiArtifact, exists := trigger.RuntimeParameters.GetGlobalRuntimeVariables()[buildBean.ExtraEnvVarExternalCiArtifactKey] // validate externalCiArtifact as docker image if exists { externalCiArtifact = strings.TrimSpace(externalCiArtifact) @@ -220,11 +219,11 @@ func (impl *CiServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, } // This will overwrite the existing runtime parameters value for constants.externalCiArtifact - trigger.RuntimeParameters = trigger.RuntimeParameters.AddRuntimeGlobalVariable(pipelineConst.ExtraEnvVarExternalCiArtifactKey, externalCiArtifact) + trigger.RuntimeParameters = trigger.RuntimeParameters.AddRuntimeGlobalVariable(buildBean.ExtraEnvVarExternalCiArtifactKey, externalCiArtifact) var artifactExists bool var err error - imageDigest, ok := trigger.RuntimeParameters.GetGlobalRuntimeVariables()[pipelineConst.ExtraEnvVarImageDigestKey] + imageDigest, ok := trigger.RuntimeParameters.GetGlobalRuntimeVariables()[buildBean.ExtraEnvVarImageDigestKey] if !ok || len(imageDigest) == 0 { artifactExists, err = impl.ciArtifactRepository.IfArtifactExistByImage(externalCiArtifact, trigger.PipelineId) if err != nil { diff --git a/pkg/pipeline/constants/constants.go b/pkg/pipeline/constants/constants.go index 243fd6679e..0028c37d45 100644 --- a/pkg/pipeline/constants/constants.go +++ b/pkg/pipeline/constants/constants.go @@ -19,19 +19,6 @@ package constants const CDPipelineNotFoundErr = "cd pipeline not found" const CiPipelineNotFoundErr = "ci pipeline not found" -type PipelineType string - -// default PipelineType -const DefaultPipelineType = CI_BUILD - -const ( - CI_BUILD PipelineType = "CI_BUILD" - LINKED PipelineType = "LINKED" - EXTERNAL PipelineType = "EXTERNAL" - CI_JOB PipelineType = "CI_JOB" - LINKED_CD PipelineType = "LINKED_CD" -) - type PatchPipelineActionResponse string const ( @@ -43,28 +30,10 @@ const ( PATCH_PIPELINE_ACTION_NOT_APPLICABLE PatchPipelineActionResponse = "N/A" ) -func (r PipelineType) ToString() string { - return string(r) -} - -func (pType PipelineType) IsValidPipelineType() bool { - switch pType { - case CI_BUILD, LINKED, EXTERNAL, CI_JOB, LINKED_CD: - return true - default: - return false - } -} - const ( UNIQUE_DEPLOYMENT_APP_NAME = "unique_deployment_app_name" ) -const ( - ExtraEnvVarExternalCiArtifactKey = "externalCiArtifact" - ExtraEnvVarImageDigestKey = "imageDigest" -) - const DefaultCiWorkflowNamespace = "devtron-ci" const Running = "Running" const Starting = "Starting" diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index 515ab4cafc..ce94737490 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -38,7 +38,7 @@ import ( bean7 "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/build/artifacts" bean5 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" - "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/manifest" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" @@ -47,7 +47,6 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/userDeploymentRequest/service" eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" "github.com/devtron-labs/devtron/pkg/pipeline" - constants2 "github.com/devtron-labs/devtron/pkg/pipeline/constants" repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning" repository3 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/repository" @@ -807,7 +806,7 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext trigger // image scanning plugin can only be applied in Post-ci, scanning in pre-ci doesn't make sense pipelineStage := repository4.PIPELINE_STAGE_TYPE_POST_CI - if pipelineModal.PipelineType == constants2.CI_JOB.ToString() { + if pipelineModal.PipelineType == buildCommonBean.CI_JOB.ToString() { pipelineStage = repository4.PIPELINE_STAGE_TYPE_PRE_CI } // this flag comes from ci-runner when scanning is enabled from ciPipeline modal @@ -831,7 +830,7 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext trigger var pluginArtifacts []*repository.CiArtifact for registry, artifacts := range request.PluginRegistryArtifactDetails { for _, image := range artifacts { - if pipelineModal.PipelineType == string(common.CI_JOB) && image == "" { + if pipelineModal.PipelineType == string(buildCommonBean.CI_JOB) && image == "" { continue } pluginArtifact := &repository.CiArtifact{ From d9dcd092470fcee623b528e53c58d8c91dfd052b Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Wed, 2 Apr 2025 08:36:13 +0530 Subject: [PATCH 03/17] ci service and trigger service creation --- .../pipeline/bean/common/CommonBuildBean.go | 13 + pkg/build/trigger/Service.go | 1086 ++++++++++++++++ .../trigger/Service_ent.go} | 6 +- pkg/build/wire_build.go | 2 + .../devtronApps/PreStageTriggerService.go | 5 +- pkg/pipeline/BuildPipelineConfigService.go | 8 +- pkg/pipeline/CiCdPipelineOrchestrator.go | 3 - pkg/pipeline/CiLogService.go | 4 +- pkg/pipeline/CiService.go | 1087 +---------------- pkg/pipeline/pipelineStageVariableParser.go | 6 +- 10 files changed, 1130 insertions(+), 1090 deletions(-) rename pkg/{pipeline/CiService_ent.go => build/trigger/Service_ent.go} (51%) diff --git a/pkg/build/pipeline/bean/common/CommonBuildBean.go b/pkg/build/pipeline/bean/common/CommonBuildBean.go index af419f962f..81692c377a 100644 --- a/pkg/build/pipeline/bean/common/CommonBuildBean.go +++ b/pkg/build/pipeline/bean/common/CommonBuildBean.go @@ -25,3 +25,16 @@ func (pType PipelineType) IsValidPipelineType() bool { return false } } + +const ( + BEFORE_DOCKER_BUILD = "BEFORE_DOCKER_BUILD" + AFTER_DOCKER_BUILD = "AFTER_DOCKER_BUILD" +) + +type RefPluginName = string + +const ( + COPY_CONTAINER_IMAGE RefPluginName = "Copy container image" + COPY_CONTAINER_IMAGE_VERSION_V1 = "1.0.0" + COPY_CONTAINER_IMAGE_VERSION_V2 = "2.0.0" +) diff --git a/pkg/build/trigger/Service.go b/pkg/build/trigger/Service.go index dee4fe7ace..67cff996ab 100644 --- a/pkg/build/trigger/Service.go +++ b/pkg/build/trigger/Service.go @@ -1 +1,1087 @@ package trigger + +import ( + "encoding/json" + "errors" + "fmt" + "github.com/caarlos0/env" + blob_storage "github.com/devtron-labs/common-lib/blob-storage" + "github.com/devtron-labs/common-lib/utils" + bean4 "github.com/devtron-labs/common-lib/utils/bean" + commonBean "github.com/devtron-labs/common-lib/workflow" + client "github.com/devtron-labs/devtron/client/events" + "github.com/devtron-labs/devtron/internal/middleware" + "github.com/devtron-labs/devtron/internal/sql/constants" + repository5 "github.com/devtron-labs/devtron/internal/sql/repository" + appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" + repository3 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" + "github.com/devtron-labs/devtron/internal/sql/repository/helper" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" + "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/app" + "github.com/devtron-labs/devtron/pkg/attributes" + bean3 "github.com/devtron-labs/devtron/pkg/attributes/bean" + "github.com/devtron-labs/devtron/pkg/auth/user" + "github.com/devtron-labs/devtron/pkg/bean" + "github.com/devtron-labs/devtron/pkg/bean/common" + "github.com/devtron-labs/devtron/pkg/build/pipeline" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" + repository6 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" + pipeline2 "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/pipeline/adapter" + pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/repository" + "github.com/devtron-labs/devtron/pkg/pipeline/types" + "github.com/devtron-labs/devtron/pkg/plugin" + bean2 "github.com/devtron-labs/devtron/pkg/plugin/bean" + repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository" + "github.com/devtron-labs/devtron/pkg/resourceQualifiers" + "github.com/devtron-labs/devtron/pkg/variables" + repository4 "github.com/devtron-labs/devtron/pkg/variables/repository" + "github.com/devtron-labs/devtron/util/sliceUtil" + "github.com/go-pg/pg" + "go.uber.org/zap" + "net/http" + "path" + "path/filepath" + "slices" + "strconv" + "strings" + "time" +) + +type Service interface { + TriggerCiPipeline(trigger types.Trigger) (int, error) +} + +type BuildxCacheFlags struct { + BuildxCacheModeMin bool `env:"BUILDX_CACHE_MODE_MIN" envDefault:"false"` + AsyncBuildxCacheExport bool `env:"ASYNC_BUILDX_CACHE_EXPORT" envDefault:"false"` +} + +type ServiceImpl struct { + Logger *zap.SugaredLogger + workflowService pipeline2.WorkflowService + ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository + ciPipelineRepository pipelineConfig.CiPipelineRepository + ciArtifactRepository repository5.CiArtifactRepository + pipelineStageService pipeline2.PipelineStageService + userService user.UserService + ciTemplateService pipeline.CiTemplateReadService + appCrudOperationService app.AppCrudOperationService + envRepository repository6.EnvironmentRepository + appRepository appRepository.AppRepository + customTagService pipeline2.CustomTagService + config *types.CiConfig + scopedVariableManager variables.ScopedVariableManager + ciCdPipelineOrchestrator pipeline2.CiCdPipelineOrchestrator + buildxCacheFlags *BuildxCacheFlags + attributeService attributes.AttributesService + pluginInputVariableParser pipeline2.PluginInputVariableParser + globalPluginService plugin.GlobalPluginService + ciService pipeline2.CiService +} + +func NewServiceImpl(Logger *zap.SugaredLogger, workflowService pipeline2.WorkflowService, + ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, + ciPipelineRepository pipelineConfig.CiPipelineRepository, + ciArtifactRepository repository5.CiArtifactRepository, + pipelineStageService pipeline2.PipelineStageService, + userService user.UserService, + ciTemplateService pipeline.CiTemplateReadService, + appCrudOperationService app.AppCrudOperationService, + envRepository repository6.EnvironmentRepository, + appRepository appRepository.AppRepository, + scopedVariableManager variables.ScopedVariableManager, + customTagService pipeline2.CustomTagService, + ciCdPipelineOrchestrator pipeline2.CiCdPipelineOrchestrator, attributeService attributes.AttributesService, + pluginInputVariableParser pipeline2.PluginInputVariableParser, + globalPluginService plugin.GlobalPluginService, + ciService pipeline2.CiService, +) *ServiceImpl { + buildxCacheFlags := &BuildxCacheFlags{} + err := env.Parse(buildxCacheFlags) + if err != nil { + Logger.Infow("error occurred while parsing BuildxCacheFlags env,so setting BuildxCacheModeMin and AsyncBuildxCacheExport to default value", "err", err) + } + cis := &ServiceImpl{ + Logger: Logger, + workflowService: workflowService, + ciPipelineMaterialRepository: ciPipelineMaterialRepository, + ciPipelineRepository: ciPipelineRepository, + ciArtifactRepository: ciArtifactRepository, + pipelineStageService: pipelineStageService, + userService: userService, + ciTemplateService: ciTemplateService, + appCrudOperationService: appCrudOperationService, + envRepository: envRepository, + appRepository: appRepository, + scopedVariableManager: scopedVariableManager, + customTagService: customTagService, + ciCdPipelineOrchestrator: ciCdPipelineOrchestrator, + buildxCacheFlags: buildxCacheFlags, + attributeService: attributeService, + pluginInputVariableParser: pluginInputVariableParser, + globalPluginService: globalPluginService, + ciService: ciService, + } + config, err := types.GetCiConfig() + if err != nil { + return nil + } + cis.config = config + return cis +} + +func (impl *ServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { + impl.Logger.Debug("ci pipeline manual trigger") + ciMaterials, err := impl.GetCiMaterials(trigger.PipelineId, trigger.CiMaterials) + if err != nil { + return 0, err + } + + ciPipelineScripts, err := impl.ciPipelineRepository.FindCiScriptsByCiPipelineId(trigger.PipelineId) + if err != nil && !util.IsErrNoRows(err) { + return 0, err + } + + var pipeline *pipelineConfig.CiPipeline + for _, m := range ciMaterials { + pipeline = m.CiPipeline + break + } + + scope := resourceQualifiers.Scope{ + AppId: pipeline.App.Id, + } + ciWorkflowConfigNamespace := impl.config.GetDefaultNamespace() + envModal, isJob, err := impl.getEnvironmentForJob(pipeline, trigger) + if err != nil { + return 0, err + } + if isJob && envModal != nil { + ciWorkflowConfigNamespace = envModal.Namespace + + // This will be populated for jobs running in selected environment + scope.EnvId = envModal.Id + scope.ClusterId = envModal.ClusterId + + scope.SystemMetadata = &resourceQualifiers.SystemMetadata{ + EnvironmentName: envModal.Name, + ClusterName: envModal.Cluster.ClusterName, + Namespace: envModal.Namespace, + } + } + if scope.SystemMetadata == nil { + scope.SystemMetadata = &resourceQualifiers.SystemMetadata{ + Namespace: ciWorkflowConfigNamespace, + AppName: pipeline.App.AppName, + } + } + + savedCiWf, err := impl.saveNewWorkflow(pipeline, ciWorkflowConfigNamespace, trigger.CommitHashes, trigger.TriggeredBy, trigger.EnvironmentId, isJob, trigger.ReferenceCiWorkflowId) + if err != nil { + impl.Logger.Errorw("could not save new workflow", "err", err) + return 0, err + } + + // preCiSteps, postCiSteps, refPluginsData, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(pipeline.Id, ciEvent) + request := pipelineConfigBean.NewBuildPrePostStepDataReq(pipeline.Id, pipelineConfigBean.CiStage, scope) + prePostAndRefPluginResponse, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(request) + if err != nil { + impl.Logger.Errorw("error in getting pre steps data for wf request", "err", err, "ciPipelineId", pipeline.Id) + dbErr := impl.markCurrentCiWorkflowFailed(savedCiWf, err) + if dbErr != nil { + impl.Logger.Errorw("saving workflow error", "err", dbErr) + } + return 0, err + } + preCiSteps := prePostAndRefPluginResponse.PreStageSteps + postCiSteps := prePostAndRefPluginResponse.PostStageSteps + refPluginsData := prePostAndRefPluginResponse.RefPluginData + variableSnapshot := prePostAndRefPluginResponse.VariableSnapshot + + if len(preCiSteps) == 0 && isJob { + errMsg := fmt.Sprintf("No tasks are configured in this job pipeline") + validationErr := util.NewApiError(http.StatusNotFound, errMsg, errMsg) + + return 0, validationErr + } + + // get env variables of git trigger data and add it in the extraEnvVariables + gitTriggerEnvVariables, _, err := impl.ciCdPipelineOrchestrator.GetGitCommitEnvVarDataForCICDStage(savedCiWf.GitTriggers) + if err != nil { + impl.Logger.Errorw("error in getting gitTrigger env data for stage", "gitTriggers", savedCiWf.GitTriggers, "err", err) + return 0, err + } + + for k, v := range gitTriggerEnvVariables { + trigger.RuntimeParameters = trigger.RuntimeParameters.AddSystemVariable(k, v) + } + + workflowRequest, err := impl.buildWfRequestForCiPipeline(pipeline, trigger, ciMaterials, savedCiWf, ciWorkflowConfigNamespace, ciPipelineScripts, preCiSteps, postCiSteps, refPluginsData, isJob) + if err != nil { + impl.Logger.Errorw("make workflow req", "err", err) + return 0, err + } + err = impl.handleRuntimeParamsValidations(trigger, ciMaterials, workflowRequest) + if err != nil { + savedCiWf.Status = cdWorkflow.WorkflowAborted + savedCiWf.Message = err.Error() + err1 := impl.ciService.UpdateCiWorkflowWithStage(savedCiWf) + if err1 != nil { + impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag") + } + return 0, err + } + + workflowRequest.Scope = scope + workflowRequest.AppId = pipeline.AppId + workflowRequest.BuildxCacheModeMin = impl.buildxCacheFlags.BuildxCacheModeMin + workflowRequest.AsyncBuildxCacheExport = impl.buildxCacheFlags.AsyncBuildxCacheExport + if impl.config != nil && impl.config.BuildxK8sDriverOptions != "" { + err = impl.setBuildxK8sDriverData(workflowRequest) + if err != nil { + impl.Logger.Errorw("error in setBuildxK8sDriverData", "BUILDX_K8S_DRIVER_OPTIONS", impl.config.BuildxK8sDriverOptions, "err", err) + return 0, err + } + } + + // savedCiWf.LogLocation = impl.ciCdConfig.CiDefaultBuildLogsKeyPrefix + "/" + workflowRequest.WorkflowNamePrefix + "/main.log" + savedCiWf.LogLocation = fmt.Sprintf("%s/%s/main.log", impl.config.GetDefaultBuildLogsKeyPrefix(), workflowRequest.WorkflowNamePrefix) + err = impl.updateCiWorkflow(workflowRequest, savedCiWf) + + appLabels, err := impl.appCrudOperationService.GetLabelsByAppId(pipeline.AppId) + if err != nil { + return 0, err + } + workflowRequest.AppLabels = appLabels + workflowRequest.Env = envModal + if isJob { + workflowRequest.Type = pipelineConfigBean.JOB_WORKFLOW_PIPELINE_TYPE + } else { + workflowRequest.Type = pipelineConfigBean.CI_WORKFLOW_PIPELINE_TYPE + } + + workflowRequest.CiPipelineType = trigger.PipelineType + err = impl.executeCiPipeline(workflowRequest) + if err != nil { + impl.Logger.Errorw("error in executing ci pipeline", "err", err) + dbErr := impl.markCurrentCiWorkflowFailed(savedCiWf, err) + if dbErr != nil { + impl.Logger.Errorw("update ci workflow error", "err", dbErr) + } + return 0, err + } + impl.Logger.Debugw("ci triggered", " pipeline ", trigger.PipelineId) + + var variableSnapshotHistories = sliceUtil.GetBeansPtr( + repository4.GetSnapshotBean(savedCiWf.Id, repository4.HistoryReferenceTypeCIWORKFLOW, variableSnapshot)) + if len(variableSnapshotHistories) > 0 { + err = impl.scopedVariableManager.SaveVariableHistoriesForTrigger(variableSnapshotHistories, trigger.TriggeredBy) + if err != nil { + impl.Logger.Errorf("Not able to save variable snapshot for CI trigger %s", err) + } + } + + middleware.CiTriggerCounter.WithLabelValues(pipeline.App.AppName, pipeline.Name).Inc() + go impl.ciService.WriteCITriggerEvent(trigger, pipeline, workflowRequest) + return savedCiWf.Id, err +} + +func (impl *ServiceImpl) GetCiMaterials(pipelineId int, ciMaterials []*pipelineConfig.CiPipelineMaterial) ([]*pipelineConfig.CiPipelineMaterial, error) { + if !(len(ciMaterials) == 0) { + return ciMaterials, nil + } else { + ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(pipelineId) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return nil, err + } + impl.Logger.Debug("ciMaterials for pipeline trigger ", ciMaterials) + return ciMaterials, nil + } +} + +func (impl *ServiceImpl) setBuildxK8sDriverData(workflowRequest *types.WorkflowRequest) error { + ciBuildConfig := workflowRequest.CiBuildConfig + if ciBuildConfig != nil { + if dockerBuildConfig := ciBuildConfig.DockerBuildConfig; dockerBuildConfig != nil { + k8sDriverOptions, err := impl.getK8sDriverOptions() + if err != nil { + errMsg := "error in parsing BUILDX_K8S_DRIVER_OPTIONS from the devtron-cm, " + err = errors.New(errMsg + "error : " + err.Error()) + impl.Logger.Errorw(errMsg, "err", err) + } + dockerBuildConfig.BuildxK8sDriverOptions = k8sDriverOptions + + } + } + return nil +} + +func (impl *ServiceImpl) getK8sDriverOptions() ([]map[string]string, error) { + buildxK8sDriverOptions := make([]map[string]string, 0) + err := json.Unmarshal([]byte(impl.config.BuildxK8sDriverOptions), &buildxK8sDriverOptions) + if err != nil { + return nil, err + } else { + return buildxK8sDriverOptions, nil + } +} + +func (impl *ServiceImpl) getEnvironmentForJob(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger) (*repository6.Environment, bool, error) { + app, err := impl.appRepository.FindById(pipeline.AppId) + if err != nil { + impl.Logger.Errorw("could not find app", "err", err) + return nil, false, err + } + + var env *repository6.Environment + isJob := false + if app.AppType == helper.Job { + isJob = true + if trigger.EnvironmentId != 0 { + env, err = impl.envRepository.FindById(trigger.EnvironmentId) + if err != nil { + impl.Logger.Errorw("could not find environment", "err", err) + return nil, isJob, err + } + return env, isJob, nil + } + } + return nil, isJob, nil +} + +// TODO: Send all trigger data +func (impl *ServiceImpl) BuildPayload(trigger types.Trigger, pipeline *pipelineConfig.CiPipeline) *client.Payload { + payload := &client.Payload{} + payload.AppName = pipeline.App.AppName + payload.PipelineName = pipeline.Name + return payload +} + +func (impl *ServiceImpl) saveNewWorkflow(pipeline *pipelineConfig.CiPipeline, ciWorkflowConfigNamespace string, + commitHashes map[int]pipelineConfig.GitCommit, userId int32, EnvironmentId int, isJob bool, refCiWorkflowId int) (wf *pipelineConfig.CiWorkflow, error error) { + + ciWorkflow := &pipelineConfig.CiWorkflow{ + Name: pipeline.Name + "-" + strconv.Itoa(pipeline.Id), + Status: cdWorkflow.WorkflowStarting, // starting CIStage + Message: "", + StartedOn: time.Now(), + CiPipelineId: pipeline.Id, + Namespace: impl.config.GetDefaultNamespace(), + BlobStorageEnabled: impl.config.BlobStorageEnabled, + GitTriggers: commitHashes, + LogLocation: "", + TriggeredBy: userId, + ReferenceCiWorkflowId: refCiWorkflowId, + ExecutorType: impl.config.GetWorkflowExecutorType(), + } + if isJob { + ciWorkflow.Namespace = ciWorkflowConfigNamespace + ciWorkflow.EnvironmentId = EnvironmentId + } + err := impl.ciService.SaveCiWorkflowWithStage(ciWorkflow) + if err != nil { + impl.Logger.Errorw("saving workflow error", "err", err) + return &pipelineConfig.CiWorkflow{}, err + } + impl.Logger.Debugw("workflow saved ", "id", ciWorkflow.Id) + return ciWorkflow, nil +} + +func (impl *ServiceImpl) executeCiPipeline(workflowRequest *types.WorkflowRequest) error { + _, _, err := impl.workflowService.SubmitWorkflow(workflowRequest) + if err != nil { + impl.Logger.Errorw("workflow error", "err", err) + return err + } + return nil +} + +func (impl *ServiceImpl) buildS3ArtifactLocation(ciWorkflowConfigLogsBucket string, savedWf *pipelineConfig.CiWorkflow) (string, string, string) { + ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() + ArtifactLocation := fmt.Sprintf("s3://"+path.Join(ciWorkflowConfigLogsBucket, ciArtifactLocationFormat), savedWf.Id, savedWf.Id) + artifactFileName := fmt.Sprintf(ciArtifactLocationFormat, savedWf.Id, savedWf.Id) + return ArtifactLocation, ciWorkflowConfigLogsBucket, artifactFileName +} + +func (impl *ServiceImpl) buildDefaultArtifactLocation(savedWf *pipelineConfig.CiWorkflow) string { + ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() + ArtifactLocation := fmt.Sprintf(ciArtifactLocationFormat, savedWf.Id, savedWf.Id) + return ArtifactLocation +} + +func (impl *ServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, savedWf *pipelineConfig.CiWorkflow, ciWorkflowConfigNamespace string, ciPipelineScripts []*pipelineConfig.CiPipelineScript, preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, refPluginsData []*pipelineConfigBean.RefPluginObject, isJob bool) (*types.WorkflowRequest, error) { + var ciProjectDetails []pipelineConfigBean.CiProjectDetails + commitHashes := trigger.CommitHashes + for _, ciMaterial := range ciMaterials { + // ignore those materials which have inactive git material + if ciMaterial == nil || ciMaterial.GitMaterial == nil || !ciMaterial.GitMaterial.Active { + continue + } + commitHashForPipelineId := commitHashes[ciMaterial.Id] + ciProjectDetail := pipelineConfigBean.CiProjectDetails{ + GitRepository: ciMaterial.GitMaterial.Url, + MaterialName: ciMaterial.GitMaterial.Name, + CheckoutPath: ciMaterial.GitMaterial.CheckoutPath, + FetchSubmodules: ciMaterial.GitMaterial.FetchSubmodules, + CommitHash: commitHashForPipelineId.Commit, + Author: commitHashForPipelineId.Author, + SourceType: ciMaterial.Type, + SourceValue: ciMaterial.Value, + GitTag: ciMaterial.GitTag, + Message: commitHashForPipelineId.Message, + Type: string(ciMaterial.Type), + CommitTime: commitHashForPipelineId.Date.Format(bean.LayoutRFC3339), + GitOptions: pipelineConfigBean.GitOptions{ + UserName: ciMaterial.GitMaterial.GitProvider.UserName, + Password: ciMaterial.GitMaterial.GitProvider.Password, + SshPrivateKey: ciMaterial.GitMaterial.GitProvider.SshPrivateKey, + AccessToken: ciMaterial.GitMaterial.GitProvider.AccessToken, + AuthMode: ciMaterial.GitMaterial.GitProvider.AuthMode, + EnableTLSVerification: ciMaterial.GitMaterial.GitProvider.EnableTLSVerification, + TlsKey: ciMaterial.GitMaterial.GitProvider.TlsKey, + TlsCert: ciMaterial.GitMaterial.GitProvider.TlsCert, + CaCert: ciMaterial.GitMaterial.GitProvider.CaCert, + }, + } + + if ciMaterial.Type == constants.SOURCE_TYPE_WEBHOOK { + webhookData := commitHashForPipelineId.WebhookData + ciProjectDetail.WebhookData = pipelineConfig.WebhookData{ + Id: webhookData.Id, + EventActionType: webhookData.EventActionType, + Data: webhookData.Data, + } + } + + ciProjectDetails = append(ciProjectDetails, ciProjectDetail) + } + + var beforeDockerBuildScripts []*bean.CiScript + var afterDockerBuildScripts []*bean.CiScript + for _, ciPipelineScript := range ciPipelineScripts { + ciTask := &bean.CiScript{ + Id: ciPipelineScript.Id, + Index: ciPipelineScript.Index, + Name: ciPipelineScript.Name, + Script: ciPipelineScript.Script, + OutputLocation: ciPipelineScript.OutputLocation, + } + + if ciPipelineScript.Stage == buildCommonBean.BEFORE_DOCKER_BUILD { + beforeDockerBuildScripts = append(beforeDockerBuildScripts, ciTask) + } else if ciPipelineScript.Stage == buildCommonBean.AFTER_DOCKER_BUILD { + afterDockerBuildScripts = append(afterDockerBuildScripts, ciTask) + } + } + + if !(len(beforeDockerBuildScripts) == 0 && len(afterDockerBuildScripts) == 0) { + // found beforeDockerBuildScripts/afterDockerBuildScripts + // building preCiSteps & postCiSteps from them, refPluginsData not needed + preCiSteps = buildCiStepsDataFromDockerBuildScripts(beforeDockerBuildScripts) + postCiSteps = buildCiStepsDataFromDockerBuildScripts(afterDockerBuildScripts) + refPluginsData = []*pipelineConfigBean.RefPluginObject{} + } + + host, err := impl.attributeService.GetByKey(bean3.HostUrlKey) + if err != nil { + impl.Logger.Errorw("error in getting host url", "err", err, "hostUrl", host.Value) + return nil, err + } + ciWorkflowConfigCiCacheBucket := impl.config.DefaultCacheBucket + + ciWorkflowConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion + + ciWorkflowConfigCiImage := impl.config.GetDefaultImage() + + ciTemplate := pipeline.CiTemplate + ciLevelArgs := pipeline.DockerArgs + + if ciLevelArgs == "" { + ciLevelArgs = "{}" + } + + if pipeline.CiTemplate.DockerBuildOptions == "" { + pipeline.CiTemplate.DockerBuildOptions = "{}" + } + userEmailId, err := impl.userService.GetActiveEmailById(trigger.TriggeredBy) + if err != nil { + impl.Logger.Errorw("unable to find user email by id", "err", err, "id", trigger.TriggeredBy) + return nil, err + } + var dockerfilePath string + var dockerRepository string + var checkoutPath string + var ciBuildConfigBean *buildBean.CiBuildConfigBean + dockerRegistry := &repository3.DockerArtifactStore{} + ciBaseBuildConfigEntity := ciTemplate.CiBuildConfig + ciBaseBuildConfigBean, err := adapter.ConvertDbBuildConfigToBean(ciBaseBuildConfigEntity) + if err != nil { + impl.Logger.Errorw("error occurred while converting buildconfig dbEntity to configBean", "ciBuildConfigEntity", ciBaseBuildConfigEntity, "err", err) + return nil, errors.New("error while parsing ci build config") + } + if !pipeline.IsExternal && pipeline.IsDockerConfigOverridden { + templateOverrideBean, err := impl.ciTemplateService.FindTemplateOverrideByCiPipelineId(pipeline.Id) + if err != nil { + return nil, err + } + ciBuildConfigBean = templateOverrideBean.CiBuildConfig + // updating args coming from ciBaseBuildConfigEntity because it is not part of Ci override + if ciBuildConfigBean != nil && ciBuildConfigBean.DockerBuildConfig != nil && ciBaseBuildConfigBean != nil && ciBaseBuildConfigBean.DockerBuildConfig != nil { + ciBuildConfigBean.DockerBuildConfig.Args = ciBaseBuildConfigBean.DockerBuildConfig.Args + } + templateOverride := templateOverrideBean.CiTemplateOverride + checkoutPath = templateOverride.GitMaterial.CheckoutPath + dockerfilePath = templateOverride.DockerfilePath + dockerRepository = templateOverride.DockerRepository + dockerRegistry = templateOverride.DockerRegistry + } else { + checkoutPath = ciTemplate.GitMaterial.CheckoutPath + dockerfilePath = ciTemplate.DockerfilePath + dockerRegistry = ciTemplate.DockerRegistry + dockerRepository = ciTemplate.DockerRepository + ciBuildConfigBean = ciBaseBuildConfigBean + if ciBuildConfigBean != nil { + ciBuildConfigBean.BuildContextGitMaterialId = ciTemplate.BuildContextGitMaterialId + } + + } + if checkoutPath == "" { + checkoutPath = "./" + } + var dockerImageTag string + customTag, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id)) + if err != nil && err != pg.ErrNoRows { + return nil, err + } + if customTag.Id != 0 && customTag.Enabled == true { + imagePathReservation, err := impl.customTagService.GenerateImagePath(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id), dockerRegistry.RegistryURL, dockerRepository) + if err != nil { + if errors.Is(err, pipelineConfigBean.ErrImagePathInUse) { + errMsg := pipelineConfigBean.ImageTagUnavailableMessage + validationErr := util.NewApiError(http.StatusConflict, errMsg, errMsg) + dbErr := impl.markCurrentCiWorkflowFailed(savedWf, validationErr) + if dbErr != nil { + impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag", "err", dbErr, "savedWf", savedWf.Id) + } + return nil, err + } + return nil, err + } + savedWf.ImagePathReservationIds = []int{imagePathReservation.Id} + // imagePath = docker.io/avd0/dashboard:fd23414b + imagePathSplit := strings.Split(imagePathReservation.ImagePath, ":") + if len(imagePathSplit) >= 1 { + dockerImageTag = imagePathSplit[len(imagePathSplit)-1] + } + } else { + dockerImageTag = impl.buildImageTag(commitHashes, pipeline.Id, savedWf.Id) + } + + // copyContainerImage plugin specific logic + var registryCredentialMap map[string]bean2.RegistryCredentials + var pluginArtifactStage string + var imageReservationIds []int + var registryDestinationImageMap map[string][]string + if !isJob { + registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imageReservationIds, err = impl.GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps, postCiSteps, dockerImageTag, customTag.Id, + fmt.Sprintf(pipelineConfigBean.ImagePathPattern, + dockerRegistry.RegistryURL, + dockerRepository, + dockerImageTag), + dockerRegistry.Id) + if err != nil { + impl.Logger.Errorw("error in getting env variables for copyContainerImage plugin") + dbErr := impl.markCurrentCiWorkflowFailed(savedWf, err) + if dbErr != nil { + impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag", "err", dbErr, "savedWf", savedWf.Id) + } + return nil, err + } + + savedWf.ImagePathReservationIds = append(savedWf.ImagePathReservationIds, imageReservationIds...) + } + // mergedArgs := string(merged) + oldArgs := ciTemplate.Args + ciBuildConfigBean, err = adapter.OverrideCiBuildConfig(dockerfilePath, oldArgs, ciLevelArgs, ciTemplate.DockerBuildOptions, ciTemplate.TargetPlatform, ciBuildConfigBean) + if err != nil { + impl.Logger.Errorw("error occurred while overriding ci build config", "oldArgs", oldArgs, "ciLevelArgs", ciLevelArgs, "error", err) + return nil, errors.New("error while parsing ci build config") + } + buildContextCheckoutPath, err := impl.ciPipelineMaterialRepository.GetCheckoutPath(ciBuildConfigBean.BuildContextGitMaterialId) + if err != nil && err != pg.ErrNoRows { + impl.Logger.Errorw("error occurred while getting checkout path from git material", "gitMaterialId", ciBuildConfigBean.BuildContextGitMaterialId, "error", err) + return nil, err + } + if buildContextCheckoutPath == "" { + buildContextCheckoutPath = checkoutPath + } + if ciBuildConfigBean.UseRootBuildContext { + // use root build context i.e '.' + buildContextCheckoutPath = "." + } + + ciBuildConfigBean.PipelineType = trigger.PipelineType + + if ciBuildConfigBean.CiBuildType == buildBean.SELF_DOCKERFILE_BUILD_TYPE || ciBuildConfigBean.CiBuildType == buildBean.MANAGED_DOCKERFILE_BUILD_TYPE { + ciBuildConfigBean.DockerBuildConfig.BuildContext = filepath.Join(buildContextCheckoutPath, ciBuildConfigBean.DockerBuildConfig.BuildContext) + dockerBuildConfig := ciBuildConfigBean.DockerBuildConfig + dockerfilePath = filepath.Join(checkoutPath, dockerBuildConfig.DockerfilePath) + dockerBuildConfig.DockerfilePath = dockerfilePath + checkoutPath = dockerfilePath[:strings.LastIndex(dockerfilePath, "/")+1] + } else if ciBuildConfigBean.CiBuildType == buildBean.BUILDPACK_BUILD_TYPE { + buildPackConfig := ciBuildConfigBean.BuildPackConfig + checkoutPath = filepath.Join(checkoutPath, buildPackConfig.ProjectPath) + } + + defaultTargetPlatform := impl.config.DefaultTargetPlatform + useBuildx := impl.config.UseBuildx + + if ciBuildConfigBean.DockerBuildConfig != nil { + if ciBuildConfigBean.DockerBuildConfig.TargetPlatform == "" && useBuildx { + ciBuildConfigBean.DockerBuildConfig.TargetPlatform = defaultTargetPlatform + ciBuildConfigBean.DockerBuildConfig.UseBuildx = useBuildx + } + ciBuildConfigBean.DockerBuildConfig.BuildxProvenanceMode = impl.config.BuildxProvenanceMode + } + + workflowRequest := &types.WorkflowRequest{ + WorkflowNamePrefix: strconv.Itoa(savedWf.Id) + "-" + savedWf.Name, + PipelineName: pipeline.Name, + PipelineId: pipeline.Id, + CiCacheFileName: pipeline.Name + "-" + strconv.Itoa(pipeline.Id) + ".tar.gz", + CiProjectDetails: ciProjectDetails, + Namespace: ciWorkflowConfigNamespace, + BlobStorageConfigured: savedWf.BlobStorageEnabled, + CiImage: ciWorkflowConfigCiImage, + WorkflowId: savedWf.Id, + TriggeredBy: savedWf.TriggeredBy, + CacheLimit: impl.config.CacheLimit, + ScanEnabled: pipeline.ScanEnabled, + CloudProvider: impl.config.CloudProvider, + DefaultAddressPoolBaseCidr: impl.config.GetDefaultAddressPoolBaseCidr(), + DefaultAddressPoolSize: impl.config.GetDefaultAddressPoolSize(), + PreCiSteps: preCiSteps, + PostCiSteps: postCiSteps, + RefPlugins: refPluginsData, + AppName: pipeline.App.AppName, + TriggerByAuthor: userEmailId, + CiBuildConfig: ciBuildConfigBean, + CiBuildDockerMtuValue: impl.config.CiRunnerDockerMTUValue, + CacheInvalidate: trigger.InvalidateCache, + SystemEnvironmentVariables: trigger.RuntimeParameters.GetSystemVariables(), + EnableBuildContext: impl.config.EnableBuildContext, + OrchestratorHost: impl.config.OrchestratorHost, + HostUrl: host.Value, + OrchestratorToken: impl.config.OrchestratorToken, + ImageRetryCount: impl.config.ImageRetryCount, + ImageRetryInterval: impl.config.ImageRetryInterval, + WorkflowExecutor: impl.config.GetWorkflowExecutorType(), + Type: pipelineConfigBean.CI_WORKFLOW_PIPELINE_TYPE, + CiArtifactLastFetch: trigger.CiArtifactLastFetch, + RegistryDestinationImageMap: registryDestinationImageMap, + RegistryCredentialMap: registryCredentialMap, + PluginArtifactStage: pluginArtifactStage, + ImageScanMaxRetries: impl.config.ImageScanMaxRetries, + ImageScanRetryDelay: impl.config.ImageScanRetryDelay, + UseDockerApiToGetDigest: impl.config.UseDockerApiToGetDigest, + } + workflowRequest.SetAwsInspectorConfig("") + //in oss, there is no pipeline level workflow cache config, so we pass inherit to get the app level config + workflowCacheConfig := impl.ciCdPipelineOrchestrator.GetWorkflowCacheConfig(pipeline.App.AppType, trigger.PipelineType, common.WorkflowCacheConfigInherit) + workflowRequest.IgnoreDockerCachePush = !workflowCacheConfig.Value + workflowRequest.IgnoreDockerCachePull = !workflowCacheConfig.Value + impl.Logger.Debugw("Ignore Cache values", "IgnoreDockerCachePush", workflowRequest.IgnoreDockerCachePush, "IgnoreDockerCachePull", workflowRequest.IgnoreDockerCachePull) + if pipeline.App.AppType == helper.Job { + workflowRequest.AppName = pipeline.App.DisplayName + } + if pipeline.ScanEnabled { + scanToolMetadata, scanVia, err := impl.fetchImageScanExecutionMedium() + if err != nil { + impl.Logger.Errorw("error occurred getting scanned via", "err", err) + return nil, err + } + workflowRequest.SetExecuteImageScanningVia(scanVia) + if scanVia.IsScanMediumExternal() { + imageScanExecutionSteps, refPlugins, err := impl.fetchImageScanExecutionStepsForWfRequest(scanToolMetadata) + if err != nil { + impl.Logger.Errorw("error occurred, fetchImageScanExecutionStepsForWfRequest", "scanToolMetadata", scanToolMetadata, "err", err) + return nil, err + } + workflowRequest.SetImageScanningSteps(imageScanExecutionSteps) + workflowRequest.RefPlugins = append(workflowRequest.RefPlugins, refPlugins...) + } + } + if dockerRegistry != nil { + workflowRequest.DockerRegistryId = dockerRegistry.Id + workflowRequest.DockerRegistryType = string(dockerRegistry.RegistryType) + workflowRequest.DockerImageTag = dockerImageTag + workflowRequest.DockerRegistryURL = dockerRegistry.RegistryURL + workflowRequest.DockerRepository = dockerRepository + workflowRequest.CheckoutPath = checkoutPath + workflowRequest.DockerUsername = dockerRegistry.Username + workflowRequest.DockerPassword = dockerRegistry.Password + workflowRequest.AwsRegion = dockerRegistry.AWSRegion + workflowRequest.AccessKey = dockerRegistry.AWSAccessKeyId + workflowRequest.SecretKey = dockerRegistry.AWSSecretAccessKey + workflowRequest.DockerConnection = dockerRegistry.Connection + workflowRequest.DockerCert = dockerRegistry.Cert + + } + ciWorkflowConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() + + switch workflowRequest.CloudProvider { + case types.BLOB_STORAGE_S3: + // No AccessKey is used for uploading artifacts, instead IAM based auth is used + workflowRequest.CiCacheRegion = ciWorkflowConfigCiCacheRegion + workflowRequest.CiCacheLocation = ciWorkflowConfigCiCacheBucket + workflowRequest.CiArtifactLocation, workflowRequest.CiArtifactBucket, workflowRequest.CiArtifactFileName = impl.buildS3ArtifactLocation(ciWorkflowConfigLogsBucket, savedWf) + workflowRequest.BlobStorageS3Config = &blob_storage.BlobStorageS3Config{ + AccessKey: impl.config.BlobStorageS3AccessKey, + Passkey: impl.config.BlobStorageS3SecretKey, + EndpointUrl: impl.config.BlobStorageS3Endpoint, + IsInSecure: impl.config.BlobStorageS3EndpointInsecure, + CiCacheBucketName: ciWorkflowConfigCiCacheBucket, + CiCacheRegion: ciWorkflowConfigCiCacheRegion, + CiCacheBucketVersioning: impl.config.BlobStorageS3BucketVersioned, + CiArtifactBucketName: workflowRequest.CiArtifactBucket, + CiArtifactRegion: impl.config.GetDefaultCdLogsBucketRegion(), + CiArtifactBucketVersioning: impl.config.BlobStorageS3BucketVersioned, + CiLogBucketName: impl.config.GetDefaultBuildLogsBucket(), + CiLogRegion: impl.config.GetDefaultCdLogsBucketRegion(), + CiLogBucketVersioning: impl.config.BlobStorageS3BucketVersioned, + } + case types.BLOB_STORAGE_GCP: + workflowRequest.GcpBlobConfig = &blob_storage.GcpBlobConfig{ + CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, + CacheBucketName: ciWorkflowConfigCiCacheBucket, + LogBucketName: ciWorkflowConfigLogsBucket, + ArtifactBucketName: ciWorkflowConfigLogsBucket, + } + workflowRequest.CiArtifactLocation = impl.buildDefaultArtifactLocation(savedWf) + workflowRequest.CiArtifactFileName = workflowRequest.CiArtifactLocation + case types.BLOB_STORAGE_AZURE: + workflowRequest.AzureBlobConfig = &blob_storage.AzureBlobConfig{ + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, + AccountName: impl.config.AzureAccountName, + BlobContainerCiCache: impl.config.AzureBlobContainerCiCache, + AccountKey: impl.config.AzureAccountKey, + BlobContainerCiLog: impl.config.AzureBlobContainerCiLog, + BlobContainerArtifact: impl.config.AzureBlobContainerCiLog, + } + workflowRequest.BlobStorageS3Config = &blob_storage.BlobStorageS3Config{ + EndpointUrl: impl.config.AzureGatewayUrl, + IsInSecure: impl.config.AzureGatewayConnectionInsecure, + CiLogBucketName: impl.config.AzureBlobContainerCiLog, + CiLogRegion: impl.config.DefaultCacheBucketRegion, + CiLogBucketVersioning: impl.config.BlobStorageS3BucketVersioned, + AccessKey: impl.config.AzureAccountName, + } + workflowRequest.CiArtifactLocation = impl.buildDefaultArtifactLocation(savedWf) + workflowRequest.CiArtifactFileName = workflowRequest.CiArtifactLocation + default: + if impl.config.BlobStorageEnabled { + return nil, fmt.Errorf("blob storage %s not supported", workflowRequest.CloudProvider) + } + } + return workflowRequest, nil +} + +func (impl *ServiceImpl) GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, customTag string, customTagId int, buildImagePath string, buildImagedockerRegistryId string) (map[string][]string, map[string]bean2.RegistryCredentials, string, []int, error) { + + copyContainerImagePluginDetail, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(buildCommonBean.COPY_CONTAINER_IMAGE) + if err != nil && err != pg.ErrNoRows { + impl.Logger.Errorw("error in getting copyContainerImage plugin id", "err", err) + return nil, nil, "", nil, err + } + + pluginIdToVersionMap := make(map[int]string) + for _, p := range copyContainerImagePluginDetail { + pluginIdToVersionMap[p.Id] = p.Version + } + + for _, step := range preCiSteps { + if _, ok := pluginIdToVersionMap[step.RefPluginId]; ok { + // for copyContainerImage plugin parse destination images and save its data in image path reservation table + return nil, nil, "", nil, errors.New("copyContainerImage plugin not allowed in pre-ci step, please remove it and try again") + } + } + + registryCredentialMap := make(map[string]bean2.RegistryCredentials) + registryDestinationImageMap := make(map[string][]string) + var allDestinationImages []string //saving all images to be reserved in this array + + for _, step := range postCiSteps { + if version, ok := pluginIdToVersionMap[step.RefPluginId]; ok { + destinationImageMap, credentialMap, err := impl.pluginInputVariableParser.HandleCopyContainerImagePluginInputVariables(step.InputVars, customTag, buildImagePath, buildImagedockerRegistryId) + if err != nil { + impl.Logger.Errorw("error in parsing copyContainerImage input variable", "err", err) + return nil, nil, "", nil, err + } + if version == buildCommonBean.COPY_CONTAINER_IMAGE_VERSION_V1 { + // this is needed in ci runner only for v1 + registryDestinationImageMap = destinationImageMap + } + for _, images := range destinationImageMap { + allDestinationImages = append(allDestinationImages, images...) + } + for k, v := range credentialMap { + registryCredentialMap[k] = v + } + } + } + + pluginArtifactStage := repository5.POST_CI + for _, image := range allDestinationImages { + if image == buildImagePath { + return nil, registryCredentialMap, pluginArtifactStage, nil, + pipelineConfigBean.ErrImagePathInUse + } + } + + var imagePathReservationIds []int + if len(allDestinationImages) > 0 { + savedCIArtifacts, err := impl.ciArtifactRepository.FindCiArtifactByImagePaths(allDestinationImages) + if err != nil { + impl.Logger.Errorw("error in fetching artifacts by image path", "err", err) + return nil, nil, pluginArtifactStage, nil, err + } + if len(savedCIArtifacts) > 0 { + // if already present in ci artifact, return "image path already in use error" + return nil, nil, pluginArtifactStage, nil, pipelineConfigBean.ErrImagePathInUse + } + imagePathReservationIds, err = impl.ReserveImagesGeneratedAtPlugin(customTagId, allDestinationImages) + if err != nil { + return nil, nil, pluginArtifactStage, imagePathReservationIds, err + } + } + return registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imagePathReservationIds, nil +} + +func (impl *ServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, destinationImages []string) ([]int, error) { + var imagePathReservationIds []int + for _, image := range destinationImages { + imagePathReservationData, err := impl.customTagService.ReserveImagePath(image, customTagId) + if err != nil { + impl.Logger.Errorw("Error in marking custom tag reserved", "err", err) + return imagePathReservationIds, err + } + imagePathReservationIds = append(imagePathReservationIds, imagePathReservationData.Id) + } + return imagePathReservationIds, nil +} + +func buildCiStepsDataFromDockerBuildScripts(dockerBuildScripts []*bean.CiScript) []*pipelineConfigBean.StepObject { + // before plugin support, few variables were set as env vars in ci-runner + // these variables are now moved to global vars in plugin steps, but to avoid error in old scripts adding those variables in payload + inputVars := []*commonBean.VariableObject{ + { + Name: "DOCKER_IMAGE_TAG", + Format: "STRING", + VariableType: commonBean.VariableTypeRefGlobal, + ReferenceVariableName: "DOCKER_IMAGE_TAG", + }, + { + Name: "DOCKER_REPOSITORY", + Format: "STRING", + VariableType: commonBean.VariableTypeRefGlobal, + ReferenceVariableName: "DOCKER_REPOSITORY", + }, + { + Name: "DOCKER_REGISTRY_URL", + Format: "STRING", + VariableType: commonBean.VariableTypeRefGlobal, + ReferenceVariableName: "DOCKER_REGISTRY_URL", + }, + { + Name: "DOCKER_IMAGE", + Format: "STRING", + VariableType: commonBean.VariableTypeRefGlobal, + ReferenceVariableName: "DOCKER_IMAGE", + }, + } + var ciSteps []*pipelineConfigBean.StepObject + for _, dockerBuildScript := range dockerBuildScripts { + ciStep := &pipelineConfigBean.StepObject{ + Name: dockerBuildScript.Name, + Index: dockerBuildScript.Index, + Script: dockerBuildScript.Script, + ArtifactPaths: []string{dockerBuildScript.OutputLocation}, + StepType: string(repository.PIPELINE_STEP_TYPE_INLINE), + ExecutorType: string(repository2.SCRIPT_TYPE_SHELL), + InputVars: inputVars, + } + ciSteps = append(ciSteps, ciStep) + } + return ciSteps +} + +func (impl *ServiceImpl) buildImageTag(commitHashes map[int]pipelineConfig.GitCommit, id int, wfId int) string { + dockerImageTag := "" + toAppendDevtronParamInTag := true + for _, v := range commitHashes { + if v.WebhookData.Id == 0 { + if v.Commit == "" { + continue + } + dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _getTruncatedImageTag(v.Commit)) + } else { + _targetCheckout := v.WebhookData.Data[bean.WEBHOOK_SELECTOR_TARGET_CHECKOUT_NAME] + if _targetCheckout == "" { + continue + } + // if not PR based then meaning tag based + isPRBasedEvent := v.WebhookData.EventActionType == bean.WEBHOOK_EVENT_MERGED_ACTION_TYPE + if !isPRBasedEvent && impl.config.CiCdConfig.UseImageTagFromGitProviderForTagBasedBuild { + dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _targetCheckout) + } else { + dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _getTruncatedImageTag(_targetCheckout)) + } + if isPRBasedEvent { + _sourceCheckout := v.WebhookData.Data[bean.WEBHOOK_SELECTOR_SOURCE_CHECKOUT_NAME] + dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _getTruncatedImageTag(_sourceCheckout)) + } else { + toAppendDevtronParamInTag = !impl.config.CiCdConfig.UseImageTagFromGitProviderForTagBasedBuild + } + } + } + toAppendDevtronParamInTag = toAppendDevtronParamInTag && dockerImageTag != "" + if toAppendDevtronParamInTag { + dockerImageTag = fmt.Sprintf("%s-%d-%d", dockerImageTag, id, wfId) + } + // replace / with underscore, as docker image tag doesn't support slash. it gives error + dockerImageTag = strings.ReplaceAll(dockerImageTag, "/", "_") + return dockerImageTag +} + +func getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, commitOrCheckOutData string) string { + if dockerImageTag == "" { + dockerImageTag = commitOrCheckOutData + } else { + if commitOrCheckOutData != "" { + dockerImageTag = fmt.Sprintf("%s-%s", dockerImageTag, commitOrCheckOutData) + } + } + return dockerImageTag +} + +func (impl *ServiceImpl) updateCiWorkflow(request *types.WorkflowRequest, savedWf *pipelineConfig.CiWorkflow) error { + ciBuildConfig := request.CiBuildConfig + ciBuildType := string(ciBuildConfig.CiBuildType) + savedWf.CiBuildType = ciBuildType + return impl.ciService.UpdateCiWorkflowWithStage(savedWf) +} + +func (impl *ServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, workflowRequest *types.WorkflowRequest) error { + // externalCi artifact is meant only for CI_JOB + if trigger.PipelineType != string(buildCommonBean.CI_JOB) { + return nil + } + + // checking if user has given run time parameters for externalCiArtifact, if given then sending git material to Ci-Runner + externalCiArtifact, exists := trigger.RuntimeParameters.GetGlobalRuntimeVariables()[buildBean.ExtraEnvVarExternalCiArtifactKey] + // validate externalCiArtifact as docker image + if exists { + externalCiArtifact = strings.TrimSpace(externalCiArtifact) + if !strings.Contains(externalCiArtifact, ":") { + if utils.IsValidDockerTagName(externalCiArtifact) { + fullImageUrl, err := utils.BuildDockerImagePath(bean4.DockerRegistryInfo{ + DockerImageTag: externalCiArtifact, + DockerRegistryId: workflowRequest.DockerRegistryId, + DockerRegistryType: workflowRequest.DockerRegistryType, + DockerRegistryURL: workflowRequest.DockerRegistryURL, + DockerRepository: workflowRequest.DockerRepository, + }) + if err != nil { + impl.Logger.Errorw("Error in building docker image", "err", err) + return err + } + externalCiArtifact = fullImageUrl + } else { + impl.Logger.Errorw("validation error", "externalCiArtifact", externalCiArtifact) + return fmt.Errorf("invalid image name or url given in externalCiArtifact") + } + + } + // This will overwrite the existing runtime parameters value for constants.externalCiArtifact + trigger.RuntimeParameters = trigger.RuntimeParameters.AddRuntimeGlobalVariable(buildBean.ExtraEnvVarExternalCiArtifactKey, externalCiArtifact) + var artifactExists bool + var err error + + imageDigest, ok := trigger.RuntimeParameters.GetGlobalRuntimeVariables()[buildBean.ExtraEnvVarImageDigestKey] + if !ok || len(imageDigest) == 0 { + artifactExists, err = impl.ciArtifactRepository.IfArtifactExistByImage(externalCiArtifact, trigger.PipelineId) + if err != nil { + impl.Logger.Errorw("error in fetching ci artifact", "err", err) + return err + } + if artifactExists { + impl.Logger.Errorw("ci artifact already exists with same image name", "artifact", externalCiArtifact) + return fmt.Errorf("ci artifact already exists with same image name") + } + } else { + artifactExists, err = impl.ciArtifactRepository.IfArtifactExistByImageDigest(imageDigest, externalCiArtifact, trigger.PipelineId) + if err != nil { + impl.Logger.Errorw("error in fetching ci artifact", "err", err, "imageDigest", imageDigest) + return err + } + if artifactExists { + impl.Logger.Errorw("ci artifact already exists with same digest", "artifact", externalCiArtifact) + return fmt.Errorf("ci artifact already exists with same digest") + } + } + } + if trigger.PipelineType == string(buildCommonBean.CI_JOB) && len(ciMaterials) != 0 && !exists && externalCiArtifact == "" { + ciMaterials[0].GitMaterial = nil + ciMaterials[0].GitMaterialId = 0 + } + return nil +} + +func _getTruncatedImageTag(imageTag string) string { + _length := len(imageTag) + if _length == 0 { + return imageTag + } + + _truncatedLength := 8 + + if _length < _truncatedLength { + return imageTag + } else { + return imageTag[:_truncatedLength] + } +} + +func (impl *ServiceImpl) markCurrentCiWorkflowFailed(savedCiWf *pipelineConfig.CiWorkflow, validationErr error) error { + // currently such requirement is not there + if savedCiWf == nil { + return nil + } + if savedCiWf.Id != 0 && slices.Contains(cdWorkflow.WfrTerminalStatusList, savedCiWf.Status) { + impl.Logger.Debug("workflow is already in terminal state", "status", savedCiWf.Status, "workflowId", savedCiWf.Id, "message", savedCiWf.Message) + return nil + } + + savedCiWf.Status = cdWorkflow.WorkflowFailed + savedCiWf.Message = validationErr.Error() + savedCiWf.FinishedOn = time.Now() + + var dbErr error + if savedCiWf.Id == 0 { + dbErr = impl.ciService.SaveCiWorkflowWithStage(savedCiWf) + } else { + dbErr = impl.ciService.UpdateCiWorkflowWithStage(savedCiWf) + } + + if dbErr != nil { + impl.Logger.Errorw("save/update workflow error", "err", dbErr) + return dbErr + } + return nil +} diff --git a/pkg/pipeline/CiService_ent.go b/pkg/build/trigger/Service_ent.go similarity index 51% rename from pkg/pipeline/CiService_ent.go rename to pkg/build/trigger/Service_ent.go index 342429d052..b20094eca7 100644 --- a/pkg/pipeline/CiService_ent.go +++ b/pkg/build/trigger/Service_ent.go @@ -1,4 +1,4 @@ -package pipeline +package trigger import ( "github.com/devtron-labs/common-lib/imageScan/bean" @@ -7,10 +7,10 @@ import ( "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" ) -func (impl *CiServiceImpl) fetchImageScanExecutionMedium() (*repository.ScanToolMetadata, bean.ScanExecutionMedium, error) { +func (impl *ServiceImpl) fetchImageScanExecutionMedium() (*repository.ScanToolMetadata, bean.ScanExecutionMedium, error) { return &repository.ScanToolMetadata{}, "", nil } -func (impl *CiServiceImpl) fetchImageScanExecutionStepsForWfRequest(scanToolMetadata *repository.ScanToolMetadata) ([]*types.ImageScanningSteps, []*bean2.RefPluginObject, error) { +func (impl *ServiceImpl) fetchImageScanExecutionStepsForWfRequest(scanToolMetadata *repository.ScanToolMetadata) ([]*types.ImageScanningSteps, []*bean2.RefPluginObject, error) { return nil, nil, nil } diff --git a/pkg/build/wire_build.go b/pkg/build/wire_build.go index 6610dd24cc..32c2fc0b0b 100644 --- a/pkg/build/wire_build.go +++ b/pkg/build/wire_build.go @@ -20,6 +20,7 @@ import ( "github.com/devtron-labs/devtron/pkg/build/artifacts" "github.com/devtron-labs/devtron/pkg/build/git" "github.com/devtron-labs/devtron/pkg/build/pipeline" + "github.com/devtron-labs/devtron/pkg/build/trigger" "github.com/google/wire" ) @@ -27,4 +28,5 @@ var WireSet = wire.NewSet( artifacts.WireSet, pipeline.WireSet, git.GitWireSet, + trigger.WireSet, ) diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go index ec80a7cf19..ef11bedf2c 100644 --- a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go @@ -41,7 +41,6 @@ import ( adapter2 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/adapter" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" - "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/adapter" pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" @@ -327,7 +326,7 @@ func (impl *TriggerServiceImpl) checkVulnerabilityStatusAndFailWfIfNeeded(ctx co // setCopyContainerImagePluginDataAndReserveImages sets required fields in cdStageWorkflowRequest and reserve images generated by plugin func (impl *TriggerServiceImpl) setCopyContainerImagePluginDataAndReserveImages(cdStageWorkflowRequest *types.WorkflowRequest, pipelineId int, pipelineStage string, artifact *repository.CiArtifact) ([]int, error) { - copyContainerImagePluginDetail, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(pipeline.COPY_CONTAINER_IMAGE) + copyContainerImagePluginDetail, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(buildCommonBean.COPY_CONTAINER_IMAGE) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in getting copyContainerImage plugin id", "err", err) return nil, err @@ -363,7 +362,7 @@ func (impl *TriggerServiceImpl) setCopyContainerImagePluginDataAndReserveImages( impl.logger.Errorw("error in parsing copyContainerImage input variable", "err", err) return nil, err } - if version == pipeline.COPY_CONTAINER_IMAGE_VERSION_V1 { + if version == buildCommonBean.COPY_CONTAINER_IMAGE_VERSION_V1 { // this is needed in ci runner only for v1 cdStageWorkflowRequest.RegistryDestinationImageMap = registryDestinationImageMap } diff --git a/pkg/pipeline/BuildPipelineConfigService.go b/pkg/pipeline/BuildPipelineConfigService.go index 95cd8775ac..f49e04ba53 100644 --- a/pkg/pipeline/BuildPipelineConfigService.go +++ b/pkg/pipeline/BuildPipelineConfigService.go @@ -591,9 +591,9 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipeline(appId int) (ciConfig *bea Script: ciScript.Script, OutputLocation: ciScript.OutputLocation, } - if ciScript.Stage == BEFORE_DOCKER_BUILD { + if ciScript.Stage == common.BEFORE_DOCKER_BUILD { beforeDockerBuildScripts = append(beforeDockerBuildScripts, ciScriptResp) - } else if ciScript.Stage == AFTER_DOCKER_BUILD { + } else if ciScript.Stage == common.AFTER_DOCKER_BUILD { afterDockerBuildScripts = append(afterDockerBuildScripts, ciScriptResp) } } @@ -746,9 +746,9 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipelineById(pipelineId int) (ciPi Script: ciScript.Script, OutputLocation: ciScript.OutputLocation, } - if ciScript.Stage == BEFORE_DOCKER_BUILD { + if ciScript.Stage == common.BEFORE_DOCKER_BUILD { beforeDockerBuildScripts = append(beforeDockerBuildScripts, ciScriptResp) - } else if ciScript.Stage == AFTER_DOCKER_BUILD { + } else if ciScript.Stage == common.AFTER_DOCKER_BUILD { afterDockerBuildScripts = append(afterDockerBuildScripts, ciScriptResp) } } diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index 46a08da9d5..0f2e5c7f2e 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -226,9 +226,6 @@ func NewCiCdPipelineOrchestrator( } } -const BEFORE_DOCKER_BUILD string = "BEFORE_DOCKER_BUILD" -const AFTER_DOCKER_BUILD string = "AFTER_DOCKER_BUILD" - func (impl CiCdPipelineOrchestratorImpl) PatchCiMaterialSource(patchRequest *bean.CiMaterialPatchRequest, userId int32) (*bean.CiMaterialPatchRequest, error) { pipeline, err := impl.findUniquePipelineForAppIdAndEnvironmentId(patchRequest.AppId, patchRequest.EnvironmentId) if err != nil { diff --git a/pkg/pipeline/CiLogService.go b/pkg/pipeline/CiLogService.go index 6ad0752f5c..d554917831 100644 --- a/pkg/pipeline/CiLogService.go +++ b/pkg/pipeline/CiLogService.go @@ -36,12 +36,11 @@ type CiLogService interface { type CiLogServiceImpl struct { logger *zap.SugaredLogger - ciService CiService kubeClient *kubernetes.Clientset k8sUtil *k8s.K8sServiceImpl } -func NewCiLogServiceImpl(logger *zap.SugaredLogger, ciService CiService, k8sUtil *k8s.K8sServiceImpl) (*CiLogServiceImpl, error) { +func NewCiLogServiceImpl(logger *zap.SugaredLogger, k8sUtil *k8s.K8sServiceImpl) (*CiLogServiceImpl, error) { _, _, clientSet, err := k8sUtil.GetK8sInClusterConfigAndClients() if err != nil { logger.Errorw("error in getting k8s in cluster client set", "err", err) @@ -49,7 +48,6 @@ func NewCiLogServiceImpl(logger *zap.SugaredLogger, ciService CiService, k8sUtil } return &CiLogServiceImpl{ logger: logger, - ciService: ciService, kubeClient: clientSet, k8sUtil: k8sUtil, }, nil diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index a2bdd0a58e..c88ae5ca11 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -17,153 +17,50 @@ package pipeline import ( - "encoding/json" - "errors" - "fmt" "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" - "github.com/caarlos0/env" - "github.com/devtron-labs/common-lib/utils" - bean3 "github.com/devtron-labs/common-lib/utils/bean" - commonBean "github.com/devtron-labs/common-lib/workflow" bean5 "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/internal/sql/constants" + client "github.com/devtron-labs/devtron/client/events" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" - "github.com/devtron-labs/devtron/pkg/attributes" - bean4 "github.com/devtron-labs/devtron/pkg/attributes/bean" - "github.com/devtron-labs/devtron/pkg/bean/common" - "github.com/devtron-labs/devtron/pkg/build/pipeline" buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" - common2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" - repository6 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" - "github.com/devtron-labs/devtron/pkg/pipeline/adapter" - "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" - bean2 "github.com/devtron-labs/devtron/pkg/plugin/bean" "github.com/devtron-labs/devtron/pkg/sql" util3 "github.com/devtron-labs/devtron/util" - "github.com/devtron-labs/devtron/util/sliceUtil" - "path" - "path/filepath" - "slices" - "strconv" - "strings" - "time" - - repository5 "github.com/devtron-labs/devtron/internal/sql/repository" - appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" - repository3 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" - "github.com/devtron-labs/devtron/internal/sql/repository/helper" - "github.com/devtron-labs/devtron/pkg/app" - "github.com/devtron-labs/devtron/pkg/auth/user" - pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" - "github.com/devtron-labs/devtron/pkg/pipeline/repository" - "github.com/devtron-labs/devtron/pkg/pipeline/types" - "github.com/devtron-labs/devtron/pkg/plugin" - repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository" - "github.com/devtron-labs/devtron/pkg/resourceQualifiers" - "github.com/devtron-labs/devtron/pkg/variables" - repository4 "github.com/devtron-labs/devtron/pkg/variables/repository" - "github.com/go-pg/pg" - "net/http" - - "github.com/devtron-labs/common-lib/blob-storage" - client "github.com/devtron-labs/devtron/client/events" - "github.com/devtron-labs/devtron/internal/middleware" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/internal/util" - "github.com/devtron-labs/devtron/pkg/bean" util2 "github.com/devtron-labs/devtron/util/event" "go.uber.org/zap" ) type CiService interface { - TriggerCiPipeline(trigger types.Trigger) (int, error) - GetCiMaterials(pipelineId int, ciMaterials []*pipelineConfig.CiPipelineMaterial) ([]*pipelineConfig.CiPipelineMaterial, error) + WriteCITriggerEvent(trigger types.Trigger, pipeline *pipelineConfig.CiPipeline, workflowRequest *types.WorkflowRequest) WriteCIFailEvent(ciWorkflow *pipelineConfig.CiWorkflow) SaveCiWorkflowWithStage(wf *pipelineConfig.CiWorkflow) error UpdateCiWorkflowWithStage(wf *pipelineConfig.CiWorkflow) error } -type BuildxCacheFlags struct { - BuildxCacheModeMin bool `env:"BUILDX_CACHE_MODE_MIN" envDefault:"false"` - AsyncBuildxCacheExport bool `env:"ASYNC_BUILDX_CACHE_EXPORT" envDefault:"false"` -} type CiServiceImpl struct { - Logger *zap.SugaredLogger - workflowService WorkflowService - ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository - workflowStageStatusService workflowStatus.WorkFlowStageStatusService - eventClient client.EventClient - eventFactory client.EventFactory - ciPipelineRepository pipelineConfig.CiPipelineRepository - ciArtifactRepository repository5.CiArtifactRepository - pipelineStageService PipelineStageService - userService user.UserService - ciTemplateService pipeline.CiTemplateReadService - appCrudOperationService app.AppCrudOperationService - envRepository repository6.EnvironmentRepository - appRepository appRepository.AppRepository - customTagService CustomTagService - config *types.CiConfig - scopedVariableManager variables.ScopedVariableManager - pluginInputVariableParser PluginInputVariableParser - globalPluginService plugin.GlobalPluginService - infraProvider infraProviders.InfraProvider - ciCdPipelineOrchestrator CiCdPipelineOrchestrator - buildxCacheFlags *BuildxCacheFlags - attributeService attributes.AttributesService - ciWorkflowRepository pipelineConfig.CiWorkflowRepository - transactionManager sql.TransactionWrapper + Logger *zap.SugaredLogger + workflowStageStatusService workflowStatus.WorkFlowStageStatusService + eventClient client.EventClient + eventFactory client.EventFactory + config *types.CiConfig + ciWorkflowRepository pipelineConfig.CiWorkflowRepository + transactionManager sql.TransactionWrapper } -func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService, - ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, +func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowStageStatusService workflowStatus.WorkFlowStageStatusService, eventClient client.EventClient, eventFactory client.EventFactory, - ciPipelineRepository pipelineConfig.CiPipelineRepository, - ciArtifactRepository repository5.CiArtifactRepository, - pipelineStageService PipelineStageService, - userService user.UserService, - ciTemplateService pipeline.CiTemplateReadService, appCrudOperationService app.AppCrudOperationService, envRepository repository6.EnvironmentRepository, appRepository appRepository.AppRepository, - scopedVariableManager variables.ScopedVariableManager, - customTagService CustomTagService, - pluginInputVariableParser PluginInputVariableParser, - globalPluginService plugin.GlobalPluginService, - infraProvider infraProviders.InfraProvider, - ciCdPipelineOrchestrator CiCdPipelineOrchestrator, attributeService attributes.AttributesService, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, transactionManager sql.TransactionWrapper, ) *CiServiceImpl { - buildxCacheFlags := &BuildxCacheFlags{} - err := env.Parse(buildxCacheFlags) - if err != nil { - Logger.Infow("error occurred while parsing BuildxCacheFlags env,so setting BuildxCacheModeMin and AsyncBuildxCacheExport to default value", "err", err) - } cis := &CiServiceImpl{ - Logger: Logger, - workflowService: workflowService, - ciPipelineMaterialRepository: ciPipelineMaterialRepository, - workflowStageStatusService: workflowStageStatusService, - eventClient: eventClient, - eventFactory: eventFactory, - ciPipelineRepository: ciPipelineRepository, - ciArtifactRepository: ciArtifactRepository, - pipelineStageService: pipelineStageService, - userService: userService, - ciTemplateService: ciTemplateService, - appCrudOperationService: appCrudOperationService, - envRepository: envRepository, - appRepository: appRepository, - scopedVariableManager: scopedVariableManager, - customTagService: customTagService, - pluginInputVariableParser: pluginInputVariableParser, - globalPluginService: globalPluginService, - infraProvider: infraProvider, - ciCdPipelineOrchestrator: ciCdPipelineOrchestrator, - buildxCacheFlags: buildxCacheFlags, - attributeService: attributeService, - ciWorkflowRepository: ciWorkflowRepository, - transactionManager: transactionManager, + Logger: Logger, + workflowStageStatusService: workflowStageStatusService, + eventClient: eventClient, + eventFactory: eventFactory, + ciWorkflowRepository: ciWorkflowRepository, + transactionManager: transactionManager, } config, err := types.GetCiConfig() if err != nil { @@ -173,322 +70,6 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService return cis } -func (impl *CiServiceImpl) GetCiMaterials(pipelineId int, ciMaterials []*pipelineConfig.CiPipelineMaterial) ([]*pipelineConfig.CiPipelineMaterial, error) { - if !(len(ciMaterials) == 0) { - return ciMaterials, nil - } else { - ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(pipelineId) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return nil, err - } - impl.Logger.Debug("ciMaterials for pipeline trigger ", ciMaterials) - return ciMaterials, nil - } -} - -func (impl *CiServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, workflowRequest *types.WorkflowRequest) error { - // externalCi artifact is meant only for CI_JOB - if trigger.PipelineType != string(common2.CI_JOB) { - return nil - } - - // checking if user has given run time parameters for externalCiArtifact, if given then sending git material to Ci-Runner - externalCiArtifact, exists := trigger.RuntimeParameters.GetGlobalRuntimeVariables()[buildBean.ExtraEnvVarExternalCiArtifactKey] - // validate externalCiArtifact as docker image - if exists { - externalCiArtifact = strings.TrimSpace(externalCiArtifact) - if !strings.Contains(externalCiArtifact, ":") { - if utils.IsValidDockerTagName(externalCiArtifact) { - fullImageUrl, err := utils.BuildDockerImagePath(bean3.DockerRegistryInfo{ - DockerImageTag: externalCiArtifact, - DockerRegistryId: workflowRequest.DockerRegistryId, - DockerRegistryType: workflowRequest.DockerRegistryType, - DockerRegistryURL: workflowRequest.DockerRegistryURL, - DockerRepository: workflowRequest.DockerRepository, - }) - if err != nil { - impl.Logger.Errorw("Error in building docker image", "err", err) - return err - } - externalCiArtifact = fullImageUrl - } else { - impl.Logger.Errorw("validation error", "externalCiArtifact", externalCiArtifact) - return fmt.Errorf("invalid image name or url given in externalCiArtifact") - } - - } - // This will overwrite the existing runtime parameters value for constants.externalCiArtifact - trigger.RuntimeParameters = trigger.RuntimeParameters.AddRuntimeGlobalVariable(buildBean.ExtraEnvVarExternalCiArtifactKey, externalCiArtifact) - var artifactExists bool - var err error - - imageDigest, ok := trigger.RuntimeParameters.GetGlobalRuntimeVariables()[buildBean.ExtraEnvVarImageDigestKey] - if !ok || len(imageDigest) == 0 { - artifactExists, err = impl.ciArtifactRepository.IfArtifactExistByImage(externalCiArtifact, trigger.PipelineId) - if err != nil { - impl.Logger.Errorw("error in fetching ci artifact", "err", err) - return err - } - if artifactExists { - impl.Logger.Errorw("ci artifact already exists with same image name", "artifact", externalCiArtifact) - return fmt.Errorf("ci artifact already exists with same image name") - } - } else { - artifactExists, err = impl.ciArtifactRepository.IfArtifactExistByImageDigest(imageDigest, externalCiArtifact, trigger.PipelineId) - if err != nil { - impl.Logger.Errorw("error in fetching ci artifact", "err", err, "imageDigest", imageDigest) - return err - } - if artifactExists { - impl.Logger.Errorw("ci artifact already exists with same digest", "artifact", externalCiArtifact) - return fmt.Errorf("ci artifact already exists with same digest") - } - - } - - } - if trigger.PipelineType == string(common2.CI_JOB) && len(ciMaterials) != 0 && !exists && externalCiArtifact == "" { - ciMaterials[0].GitMaterial = nil - ciMaterials[0].GitMaterialId = 0 - } - return nil -} - -func (impl *CiServiceImpl) markCurrentCiWorkflowFailed(savedCiWf *pipelineConfig.CiWorkflow, validationErr error) error { - // currently such requirement is not there - if savedCiWf == nil { - return nil - } - if savedCiWf.Id != 0 && slices.Contains(cdWorkflow.WfrTerminalStatusList, savedCiWf.Status) { - impl.Logger.Debug("workflow is already in terminal state", "status", savedCiWf.Status, "workflowId", savedCiWf.Id, "message", savedCiWf.Message) - return nil - } - - savedCiWf.Status = cdWorkflow.WorkflowFailed - savedCiWf.Message = validationErr.Error() - savedCiWf.FinishedOn = time.Now() - - var dbErr error - if savedCiWf.Id == 0 { - dbErr = impl.SaveCiWorkflowWithStage(savedCiWf) - } else { - dbErr = impl.UpdateCiWorkflowWithStage(savedCiWf) - } - - if dbErr != nil { - impl.Logger.Errorw("save/update workflow error", "err", dbErr) - return dbErr - } - return nil -} - -func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { - impl.Logger.Debug("ci pipeline manual trigger") - ciMaterials, err := impl.GetCiMaterials(trigger.PipelineId, trigger.CiMaterials) - if err != nil { - return 0, err - } - - ciPipelineScripts, err := impl.ciPipelineRepository.FindCiScriptsByCiPipelineId(trigger.PipelineId) - if err != nil && !util.IsErrNoRows(err) { - return 0, err - } - - var pipeline *pipelineConfig.CiPipeline - for _, m := range ciMaterials { - pipeline = m.CiPipeline - break - } - - scope := resourceQualifiers.Scope{ - AppId: pipeline.App.Id, - } - ciWorkflowConfigNamespace := impl.config.GetDefaultNamespace() - envModal, isJob, err := impl.getEnvironmentForJob(pipeline, trigger) - if err != nil { - return 0, err - } - if isJob && envModal != nil { - ciWorkflowConfigNamespace = envModal.Namespace - - // This will be populated for jobs running in selected environment - scope.EnvId = envModal.Id - scope.ClusterId = envModal.ClusterId - - scope.SystemMetadata = &resourceQualifiers.SystemMetadata{ - EnvironmentName: envModal.Name, - ClusterName: envModal.Cluster.ClusterName, - Namespace: envModal.Namespace, - } - } - if scope.SystemMetadata == nil { - scope.SystemMetadata = &resourceQualifiers.SystemMetadata{ - Namespace: ciWorkflowConfigNamespace, - AppName: pipeline.App.AppName, - } - } - - savedCiWf, err := impl.saveNewWorkflow(pipeline, ciWorkflowConfigNamespace, trigger.CommitHashes, trigger.TriggeredBy, trigger.EnvironmentId, isJob, trigger.ReferenceCiWorkflowId) - if err != nil { - impl.Logger.Errorw("could not save new workflow", "err", err) - return 0, err - } - - // preCiSteps, postCiSteps, refPluginsData, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(pipeline.Id, ciEvent) - request := pipelineConfigBean.NewBuildPrePostStepDataReq(pipeline.Id, pipelineConfigBean.CiStage, scope) - prePostAndRefPluginResponse, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(request) - if err != nil { - impl.Logger.Errorw("error in getting pre steps data for wf request", "err", err, "ciPipelineId", pipeline.Id) - dbErr := impl.markCurrentCiWorkflowFailed(savedCiWf, err) - if dbErr != nil { - impl.Logger.Errorw("saving workflow error", "err", dbErr) - } - return 0, err - } - preCiSteps := prePostAndRefPluginResponse.PreStageSteps - postCiSteps := prePostAndRefPluginResponse.PostStageSteps - refPluginsData := prePostAndRefPluginResponse.RefPluginData - variableSnapshot := prePostAndRefPluginResponse.VariableSnapshot - - if len(preCiSteps) == 0 && isJob { - errMsg := fmt.Sprintf("No tasks are configured in this job pipeline") - validationErr := util.NewApiError(http.StatusNotFound, errMsg, errMsg) - - return 0, validationErr - } - - // get env variables of git trigger data and add it in the extraEnvVariables - gitTriggerEnvVariables, _, err := impl.ciCdPipelineOrchestrator.GetGitCommitEnvVarDataForCICDStage(savedCiWf.GitTriggers) - if err != nil { - impl.Logger.Errorw("error in getting gitTrigger env data for stage", "gitTriggers", savedCiWf.GitTriggers, "err", err) - return 0, err - } - - for k, v := range gitTriggerEnvVariables { - trigger.RuntimeParameters = trigger.RuntimeParameters.AddSystemVariable(k, v) - } - - workflowRequest, err := impl.buildWfRequestForCiPipeline(pipeline, trigger, ciMaterials, savedCiWf, ciWorkflowConfigNamespace, ciPipelineScripts, preCiSteps, postCiSteps, refPluginsData, isJob) - if err != nil { - impl.Logger.Errorw("make workflow req", "err", err) - return 0, err - } - err = impl.handleRuntimeParamsValidations(trigger, ciMaterials, workflowRequest) - if err != nil { - savedCiWf.Status = cdWorkflow.WorkflowAborted - savedCiWf.Message = err.Error() - err1 := impl.UpdateCiWorkflowWithStage(savedCiWf) - if err1 != nil { - impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag") - } - return 0, err - } - - workflowRequest.Scope = scope - workflowRequest.AppId = pipeline.AppId - workflowRequest.BuildxCacheModeMin = impl.buildxCacheFlags.BuildxCacheModeMin - workflowRequest.AsyncBuildxCacheExport = impl.buildxCacheFlags.AsyncBuildxCacheExport - if impl.config != nil && impl.config.BuildxK8sDriverOptions != "" { - err = impl.setBuildxK8sDriverData(workflowRequest) - if err != nil { - impl.Logger.Errorw("error in setBuildxK8sDriverData", "BUILDX_K8S_DRIVER_OPTIONS", impl.config.BuildxK8sDriverOptions, "err", err) - return 0, err - } - } - - // savedCiWf.LogLocation = impl.ciCdConfig.CiDefaultBuildLogsKeyPrefix + "/" + workflowRequest.WorkflowNamePrefix + "/main.log" - savedCiWf.LogLocation = fmt.Sprintf("%s/%s/main.log", impl.config.GetDefaultBuildLogsKeyPrefix(), workflowRequest.WorkflowNamePrefix) - err = impl.updateCiWorkflow(workflowRequest, savedCiWf) - - appLabels, err := impl.appCrudOperationService.GetLabelsByAppId(pipeline.AppId) - if err != nil { - return 0, err - } - workflowRequest.AppLabels = appLabels - workflowRequest.Env = envModal - if isJob { - workflowRequest.Type = pipelineConfigBean.JOB_WORKFLOW_PIPELINE_TYPE - } else { - workflowRequest.Type = pipelineConfigBean.CI_WORKFLOW_PIPELINE_TYPE - } - - workflowRequest.CiPipelineType = trigger.PipelineType - err = impl.executeCiPipeline(workflowRequest) - if err != nil { - impl.Logger.Errorw("error in executing ci pipeline", "err", err) - dbErr := impl.markCurrentCiWorkflowFailed(savedCiWf, err) - if dbErr != nil { - impl.Logger.Errorw("update ci workflow error", "err", dbErr) - } - return 0, err - } - impl.Logger.Debugw("ci triggered", " pipeline ", trigger.PipelineId) - - var variableSnapshotHistories = sliceUtil.GetBeansPtr( - repository4.GetSnapshotBean(savedCiWf.Id, repository4.HistoryReferenceTypeCIWORKFLOW, variableSnapshot)) - if len(variableSnapshotHistories) > 0 { - err = impl.scopedVariableManager.SaveVariableHistoriesForTrigger(variableSnapshotHistories, trigger.TriggeredBy) - if err != nil { - impl.Logger.Errorf("Not able to save variable snapshot for CI trigger %s", err) - } - } - - middleware.CiTriggerCounter.WithLabelValues(pipeline.App.AppName, pipeline.Name).Inc() - go impl.WriteCITriggerEvent(trigger, pipeline, workflowRequest) - return savedCiWf.Id, err -} - -func (impl *CiServiceImpl) setBuildxK8sDriverData(workflowRequest *types.WorkflowRequest) error { - ciBuildConfig := workflowRequest.CiBuildConfig - if ciBuildConfig != nil { - if dockerBuildConfig := ciBuildConfig.DockerBuildConfig; dockerBuildConfig != nil { - k8sDriverOptions, err := impl.getK8sDriverOptions() - if err != nil { - errMsg := "error in parsing BUILDX_K8S_DRIVER_OPTIONS from the devtron-cm, " - err = errors.New(errMsg + "error : " + err.Error()) - impl.Logger.Errorw(errMsg, "err", err) - } - dockerBuildConfig.BuildxK8sDriverOptions = k8sDriverOptions - - } - } - return nil -} - -func (impl *CiServiceImpl) getK8sDriverOptions() ([]map[string]string, error) { - buildxK8sDriverOptions := make([]map[string]string, 0) - err := json.Unmarshal([]byte(impl.config.BuildxK8sDriverOptions), &buildxK8sDriverOptions) - if err != nil { - return nil, err - } else { - return buildxK8sDriverOptions, nil - } -} - -func (impl *CiServiceImpl) getEnvironmentForJob(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger) (*repository6.Environment, bool, error) { - app, err := impl.appRepository.FindById(pipeline.AppId) - if err != nil { - impl.Logger.Errorw("could not find app", "err", err) - return nil, false, err - } - - var env *repository6.Environment - isJob := false - if app.AppType == helper.Job { - isJob = true - if trigger.EnvironmentId != 0 { - env, err = impl.envRepository.FindById(trigger.EnvironmentId) - if err != nil { - impl.Logger.Errorw("could not find environment", "err", err) - return nil, isJob, err - } - return env, isJob, nil - } - } - return nil, isJob, nil -} - func (impl *CiServiceImpl) WriteCITriggerEvent(trigger types.Trigger, pipeline *pipelineConfig.CiPipeline, workflowRequest *types.WorkflowRequest) { event, _ := impl.eventFactory.Build(util2.Trigger, &pipeline.Id, pipeline.AppId, nil, util2.CI) material := &buildBean.MaterialTriggerInfo{} @@ -504,638 +85,6 @@ func (impl *CiServiceImpl) WriteCITriggerEvent(trigger types.Trigger, pipeline * } } -// TODO: Send all trigger data -func (impl *CiServiceImpl) BuildPayload(trigger types.Trigger, pipeline *pipelineConfig.CiPipeline) *client.Payload { - payload := &client.Payload{} - payload.AppName = pipeline.App.AppName - payload.PipelineName = pipeline.Name - return payload -} - -func (impl *CiServiceImpl) saveNewWorkflow(pipeline *pipelineConfig.CiPipeline, ciWorkflowConfigNamespace string, - commitHashes map[int]pipelineConfig.GitCommit, userId int32, EnvironmentId int, isJob bool, refCiWorkflowId int) (wf *pipelineConfig.CiWorkflow, error error) { - - ciWorkflow := &pipelineConfig.CiWorkflow{ - Name: pipeline.Name + "-" + strconv.Itoa(pipeline.Id), - Status: cdWorkflow.WorkflowStarting, // starting CIStage - Message: "", - StartedOn: time.Now(), - CiPipelineId: pipeline.Id, - Namespace: impl.config.GetDefaultNamespace(), - BlobStorageEnabled: impl.config.BlobStorageEnabled, - GitTriggers: commitHashes, - LogLocation: "", - TriggeredBy: userId, - ReferenceCiWorkflowId: refCiWorkflowId, - ExecutorType: impl.config.GetWorkflowExecutorType(), - } - if isJob { - ciWorkflow.Namespace = ciWorkflowConfigNamespace - ciWorkflow.EnvironmentId = EnvironmentId - } - err := impl.SaveCiWorkflowWithStage(ciWorkflow) - if err != nil { - impl.Logger.Errorw("saving workflow error", "err", err) - return &pipelineConfig.CiWorkflow{}, err - } - impl.Logger.Debugw("workflow saved ", "id", ciWorkflow.Id) - return ciWorkflow, nil -} - -func (impl *CiServiceImpl) executeCiPipeline(workflowRequest *types.WorkflowRequest) error { - _, _, err := impl.workflowService.SubmitWorkflow(workflowRequest) - if err != nil { - impl.Logger.Errorw("workflow error", "err", err) - return err - } - return nil -} - -func (impl *CiServiceImpl) buildS3ArtifactLocation(ciWorkflowConfigLogsBucket string, savedWf *pipelineConfig.CiWorkflow) (string, string, string) { - ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() - ArtifactLocation := fmt.Sprintf("s3://"+path.Join(ciWorkflowConfigLogsBucket, ciArtifactLocationFormat), savedWf.Id, savedWf.Id) - artifactFileName := fmt.Sprintf(ciArtifactLocationFormat, savedWf.Id, savedWf.Id) - return ArtifactLocation, ciWorkflowConfigLogsBucket, artifactFileName -} - -func (impl *CiServiceImpl) buildDefaultArtifactLocation(savedWf *pipelineConfig.CiWorkflow) string { - ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() - ArtifactLocation := fmt.Sprintf(ciArtifactLocationFormat, savedWf.Id, savedWf.Id) - return ArtifactLocation -} - -func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, savedWf *pipelineConfig.CiWorkflow, ciWorkflowConfigNamespace string, ciPipelineScripts []*pipelineConfig.CiPipelineScript, preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, refPluginsData []*pipelineConfigBean.RefPluginObject, isJob bool) (*types.WorkflowRequest, error) { - var ciProjectDetails []pipelineConfigBean.CiProjectDetails - commitHashes := trigger.CommitHashes - for _, ciMaterial := range ciMaterials { - // ignore those materials which have inactive git material - if ciMaterial == nil || ciMaterial.GitMaterial == nil || !ciMaterial.GitMaterial.Active { - continue - } - commitHashForPipelineId := commitHashes[ciMaterial.Id] - ciProjectDetail := pipelineConfigBean.CiProjectDetails{ - GitRepository: ciMaterial.GitMaterial.Url, - MaterialName: ciMaterial.GitMaterial.Name, - CheckoutPath: ciMaterial.GitMaterial.CheckoutPath, - FetchSubmodules: ciMaterial.GitMaterial.FetchSubmodules, - CommitHash: commitHashForPipelineId.Commit, - Author: commitHashForPipelineId.Author, - SourceType: ciMaterial.Type, - SourceValue: ciMaterial.Value, - GitTag: ciMaterial.GitTag, - Message: commitHashForPipelineId.Message, - Type: string(ciMaterial.Type), - CommitTime: commitHashForPipelineId.Date.Format(bean.LayoutRFC3339), - GitOptions: pipelineConfigBean.GitOptions{ - UserName: ciMaterial.GitMaterial.GitProvider.UserName, - Password: ciMaterial.GitMaterial.GitProvider.Password, - SshPrivateKey: ciMaterial.GitMaterial.GitProvider.SshPrivateKey, - AccessToken: ciMaterial.GitMaterial.GitProvider.AccessToken, - AuthMode: ciMaterial.GitMaterial.GitProvider.AuthMode, - EnableTLSVerification: ciMaterial.GitMaterial.GitProvider.EnableTLSVerification, - TlsKey: ciMaterial.GitMaterial.GitProvider.TlsKey, - TlsCert: ciMaterial.GitMaterial.GitProvider.TlsCert, - CaCert: ciMaterial.GitMaterial.GitProvider.CaCert, - }, - } - - if ciMaterial.Type == constants.SOURCE_TYPE_WEBHOOK { - webhookData := commitHashForPipelineId.WebhookData - ciProjectDetail.WebhookData = pipelineConfig.WebhookData{ - Id: webhookData.Id, - EventActionType: webhookData.EventActionType, - Data: webhookData.Data, - } - } - - ciProjectDetails = append(ciProjectDetails, ciProjectDetail) - } - - var beforeDockerBuildScripts []*bean.CiScript - var afterDockerBuildScripts []*bean.CiScript - for _, ciPipelineScript := range ciPipelineScripts { - ciTask := &bean.CiScript{ - Id: ciPipelineScript.Id, - Index: ciPipelineScript.Index, - Name: ciPipelineScript.Name, - Script: ciPipelineScript.Script, - OutputLocation: ciPipelineScript.OutputLocation, - } - - if ciPipelineScript.Stage == BEFORE_DOCKER_BUILD { - beforeDockerBuildScripts = append(beforeDockerBuildScripts, ciTask) - } else if ciPipelineScript.Stage == AFTER_DOCKER_BUILD { - afterDockerBuildScripts = append(afterDockerBuildScripts, ciTask) - } - } - - if !(len(beforeDockerBuildScripts) == 0 && len(afterDockerBuildScripts) == 0) { - // found beforeDockerBuildScripts/afterDockerBuildScripts - // building preCiSteps & postCiSteps from them, refPluginsData not needed - preCiSteps = buildCiStepsDataFromDockerBuildScripts(beforeDockerBuildScripts) - postCiSteps = buildCiStepsDataFromDockerBuildScripts(afterDockerBuildScripts) - refPluginsData = []*pipelineConfigBean.RefPluginObject{} - } - host, err := impl.attributeService.GetByKey(bean4.HostUrlKey) - if err != nil { - impl.Logger.Errorw("error in getting host url", "err", err, "hostUrl", host.Value) - return nil, err - } - ciWorkflowConfigCiCacheBucket := impl.config.DefaultCacheBucket - - ciWorkflowConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion - - ciWorkflowConfigCiImage := impl.config.GetDefaultImage() - ciTemplate := pipeline.CiTemplate - ciLevelArgs := pipeline.DockerArgs - - if ciLevelArgs == "" { - ciLevelArgs = "{}" - } - - if pipeline.CiTemplate.DockerBuildOptions == "" { - pipeline.CiTemplate.DockerBuildOptions = "{}" - } - userEmailId, err := impl.userService.GetActiveEmailById(trigger.TriggeredBy) - if err != nil { - impl.Logger.Errorw("unable to find user email by id", "err", err, "id", trigger.TriggeredBy) - return nil, err - } - var dockerfilePath string - var dockerRepository string - var checkoutPath string - var ciBuildConfigBean *buildBean.CiBuildConfigBean - dockerRegistry := &repository3.DockerArtifactStore{} - ciBaseBuildConfigEntity := ciTemplate.CiBuildConfig - ciBaseBuildConfigBean, err := adapter.ConvertDbBuildConfigToBean(ciBaseBuildConfigEntity) - if err != nil { - impl.Logger.Errorw("error occurred while converting buildconfig dbEntity to configBean", "ciBuildConfigEntity", ciBaseBuildConfigEntity, "err", err) - return nil, errors.New("error while parsing ci build config") - } - if !pipeline.IsExternal && pipeline.IsDockerConfigOverridden { - templateOverrideBean, err := impl.ciTemplateService.FindTemplateOverrideByCiPipelineId(pipeline.Id) - if err != nil { - return nil, err - } - ciBuildConfigBean = templateOverrideBean.CiBuildConfig - // updating args coming from ciBaseBuildConfigEntity because it is not part of Ci override - if ciBuildConfigBean != nil && ciBuildConfigBean.DockerBuildConfig != nil && ciBaseBuildConfigBean != nil && ciBaseBuildConfigBean.DockerBuildConfig != nil { - ciBuildConfigBean.DockerBuildConfig.Args = ciBaseBuildConfigBean.DockerBuildConfig.Args - } - templateOverride := templateOverrideBean.CiTemplateOverride - checkoutPath = templateOverride.GitMaterial.CheckoutPath - dockerfilePath = templateOverride.DockerfilePath - dockerRepository = templateOverride.DockerRepository - dockerRegistry = templateOverride.DockerRegistry - } else { - checkoutPath = ciTemplate.GitMaterial.CheckoutPath - dockerfilePath = ciTemplate.DockerfilePath - dockerRegistry = ciTemplate.DockerRegistry - dockerRepository = ciTemplate.DockerRepository - ciBuildConfigBean = ciBaseBuildConfigBean - if ciBuildConfigBean != nil { - ciBuildConfigBean.BuildContextGitMaterialId = ciTemplate.BuildContextGitMaterialId - } - - } - if checkoutPath == "" { - checkoutPath = "./" - } - var dockerImageTag string - customTag, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id)) - if err != nil && err != pg.ErrNoRows { - return nil, err - } - if customTag.Id != 0 && customTag.Enabled == true { - imagePathReservation, err := impl.customTagService.GenerateImagePath(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id), dockerRegistry.RegistryURL, dockerRepository) - if err != nil { - if errors.Is(err, pipelineConfigBean.ErrImagePathInUse) { - errMsg := pipelineConfigBean.ImageTagUnavailableMessage - validationErr := util.NewApiError(http.StatusConflict, errMsg, errMsg) - dbErr := impl.markCurrentCiWorkflowFailed(savedWf, validationErr) - if dbErr != nil { - impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag", "err", dbErr, "savedWf", savedWf.Id) - } - return nil, err - } - return nil, err - } - savedWf.ImagePathReservationIds = []int{imagePathReservation.Id} - // imagePath = docker.io/avd0/dashboard:fd23414b - imagePathSplit := strings.Split(imagePathReservation.ImagePath, ":") - if len(imagePathSplit) >= 1 { - dockerImageTag = imagePathSplit[len(imagePathSplit)-1] - } - } else { - dockerImageTag = impl.buildImageTag(commitHashes, pipeline.Id, savedWf.Id) - } - - // copyContainerImage plugin specific logic - var registryCredentialMap map[string]bean2.RegistryCredentials - var pluginArtifactStage string - var imageReservationIds []int - var registryDestinationImageMap map[string][]string - if !isJob { - registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imageReservationIds, err = impl.GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps, postCiSteps, dockerImageTag, customTag.Id, - fmt.Sprintf(pipelineConfigBean.ImagePathPattern, - dockerRegistry.RegistryURL, - dockerRepository, - dockerImageTag), - dockerRegistry.Id) - if err != nil { - impl.Logger.Errorw("error in getting env variables for copyContainerImage plugin") - dbErr := impl.markCurrentCiWorkflowFailed(savedWf, err) - if dbErr != nil { - impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag", "err", dbErr, "savedWf", savedWf.Id) - } - return nil, err - } - - savedWf.ImagePathReservationIds = append(savedWf.ImagePathReservationIds, imageReservationIds...) - } - // mergedArgs := string(merged) - oldArgs := ciTemplate.Args - ciBuildConfigBean, err = adapter.OverrideCiBuildConfig(dockerfilePath, oldArgs, ciLevelArgs, ciTemplate.DockerBuildOptions, ciTemplate.TargetPlatform, ciBuildConfigBean) - if err != nil { - impl.Logger.Errorw("error occurred while overriding ci build config", "oldArgs", oldArgs, "ciLevelArgs", ciLevelArgs, "error", err) - return nil, errors.New("error while parsing ci build config") - } - buildContextCheckoutPath, err := impl.ciPipelineMaterialRepository.GetCheckoutPath(ciBuildConfigBean.BuildContextGitMaterialId) - if err != nil && err != pg.ErrNoRows { - impl.Logger.Errorw("error occurred while getting checkout path from git material", "gitMaterialId", ciBuildConfigBean.BuildContextGitMaterialId, "error", err) - return nil, err - } - if buildContextCheckoutPath == "" { - buildContextCheckoutPath = checkoutPath - } - if ciBuildConfigBean.UseRootBuildContext { - // use root build context i.e '.' - buildContextCheckoutPath = "." - } - - ciBuildConfigBean.PipelineType = trigger.PipelineType - - if ciBuildConfigBean.CiBuildType == buildBean.SELF_DOCKERFILE_BUILD_TYPE || ciBuildConfigBean.CiBuildType == buildBean.MANAGED_DOCKERFILE_BUILD_TYPE { - ciBuildConfigBean.DockerBuildConfig.BuildContext = filepath.Join(buildContextCheckoutPath, ciBuildConfigBean.DockerBuildConfig.BuildContext) - dockerBuildConfig := ciBuildConfigBean.DockerBuildConfig - dockerfilePath = filepath.Join(checkoutPath, dockerBuildConfig.DockerfilePath) - dockerBuildConfig.DockerfilePath = dockerfilePath - checkoutPath = dockerfilePath[:strings.LastIndex(dockerfilePath, "/")+1] - } else if ciBuildConfigBean.CiBuildType == buildBean.BUILDPACK_BUILD_TYPE { - buildPackConfig := ciBuildConfigBean.BuildPackConfig - checkoutPath = filepath.Join(checkoutPath, buildPackConfig.ProjectPath) - } - - defaultTargetPlatform := impl.config.DefaultTargetPlatform - useBuildx := impl.config.UseBuildx - - if ciBuildConfigBean.DockerBuildConfig != nil { - if ciBuildConfigBean.DockerBuildConfig.TargetPlatform == "" && useBuildx { - ciBuildConfigBean.DockerBuildConfig.TargetPlatform = defaultTargetPlatform - ciBuildConfigBean.DockerBuildConfig.UseBuildx = useBuildx - } - ciBuildConfigBean.DockerBuildConfig.BuildxProvenanceMode = impl.config.BuildxProvenanceMode - } - - workflowRequest := &types.WorkflowRequest{ - WorkflowNamePrefix: strconv.Itoa(savedWf.Id) + "-" + savedWf.Name, - PipelineName: pipeline.Name, - PipelineId: pipeline.Id, - CiCacheFileName: pipeline.Name + "-" + strconv.Itoa(pipeline.Id) + ".tar.gz", - CiProjectDetails: ciProjectDetails, - Namespace: ciWorkflowConfigNamespace, - BlobStorageConfigured: savedWf.BlobStorageEnabled, - CiImage: ciWorkflowConfigCiImage, - WorkflowId: savedWf.Id, - TriggeredBy: savedWf.TriggeredBy, - CacheLimit: impl.config.CacheLimit, - ScanEnabled: pipeline.ScanEnabled, - CloudProvider: impl.config.CloudProvider, - DefaultAddressPoolBaseCidr: impl.config.GetDefaultAddressPoolBaseCidr(), - DefaultAddressPoolSize: impl.config.GetDefaultAddressPoolSize(), - PreCiSteps: preCiSteps, - PostCiSteps: postCiSteps, - RefPlugins: refPluginsData, - AppName: pipeline.App.AppName, - TriggerByAuthor: userEmailId, - CiBuildConfig: ciBuildConfigBean, - CiBuildDockerMtuValue: impl.config.CiRunnerDockerMTUValue, - CacheInvalidate: trigger.InvalidateCache, - SystemEnvironmentVariables: trigger.RuntimeParameters.GetSystemVariables(), - EnableBuildContext: impl.config.EnableBuildContext, - OrchestratorHost: impl.config.OrchestratorHost, - HostUrl: host.Value, - OrchestratorToken: impl.config.OrchestratorToken, - ImageRetryCount: impl.config.ImageRetryCount, - ImageRetryInterval: impl.config.ImageRetryInterval, - WorkflowExecutor: impl.config.GetWorkflowExecutorType(), - Type: pipelineConfigBean.CI_WORKFLOW_PIPELINE_TYPE, - CiArtifactLastFetch: trigger.CiArtifactLastFetch, - RegistryDestinationImageMap: registryDestinationImageMap, - RegistryCredentialMap: registryCredentialMap, - PluginArtifactStage: pluginArtifactStage, - ImageScanMaxRetries: impl.config.ImageScanMaxRetries, - ImageScanRetryDelay: impl.config.ImageScanRetryDelay, - UseDockerApiToGetDigest: impl.config.UseDockerApiToGetDigest, - } - workflowRequest.SetAwsInspectorConfig("") - //in oss, there is no pipeline level workflow cache config, so we pass inherit to get the app level config - workflowCacheConfig := impl.ciCdPipelineOrchestrator.GetWorkflowCacheConfig(pipeline.App.AppType, trigger.PipelineType, common.WorkflowCacheConfigInherit) - workflowRequest.IgnoreDockerCachePush = !workflowCacheConfig.Value - workflowRequest.IgnoreDockerCachePull = !workflowCacheConfig.Value - impl.Logger.Debugw("Ignore Cache values", "IgnoreDockerCachePush", workflowRequest.IgnoreDockerCachePush, "IgnoreDockerCachePull", workflowRequest.IgnoreDockerCachePull) - if pipeline.App.AppType == helper.Job { - workflowRequest.AppName = pipeline.App.DisplayName - } - if pipeline.ScanEnabled { - scanToolMetadata, scanVia, err := impl.fetchImageScanExecutionMedium() - if err != nil { - impl.Logger.Errorw("error occurred getting scanned via", "err", err) - return nil, err - } - workflowRequest.SetExecuteImageScanningVia(scanVia) - if scanVia.IsScanMediumExternal() { - imageScanExecutionSteps, refPlugins, err := impl.fetchImageScanExecutionStepsForWfRequest(scanToolMetadata) - if err != nil { - impl.Logger.Errorw("error occurred, fetchImageScanExecutionStepsForWfRequest", "scanToolMetadata", scanToolMetadata, "err", err) - return nil, err - } - workflowRequest.SetImageScanningSteps(imageScanExecutionSteps) - workflowRequest.RefPlugins = append(workflowRequest.RefPlugins, refPlugins...) - } - } - - if dockerRegistry != nil { - workflowRequest.DockerRegistryId = dockerRegistry.Id - workflowRequest.DockerRegistryType = string(dockerRegistry.RegistryType) - workflowRequest.DockerImageTag = dockerImageTag - workflowRequest.DockerRegistryURL = dockerRegistry.RegistryURL - workflowRequest.DockerRepository = dockerRepository - workflowRequest.CheckoutPath = checkoutPath - workflowRequest.DockerUsername = dockerRegistry.Username - workflowRequest.DockerPassword = dockerRegistry.Password - workflowRequest.AwsRegion = dockerRegistry.AWSRegion - workflowRequest.AccessKey = dockerRegistry.AWSAccessKeyId - workflowRequest.SecretKey = dockerRegistry.AWSSecretAccessKey - workflowRequest.DockerConnection = dockerRegistry.Connection - workflowRequest.DockerCert = dockerRegistry.Cert - - } - ciWorkflowConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() - - switch workflowRequest.CloudProvider { - case types.BLOB_STORAGE_S3: - // No AccessKey is used for uploading artifacts, instead IAM based auth is used - workflowRequest.CiCacheRegion = ciWorkflowConfigCiCacheRegion - workflowRequest.CiCacheLocation = ciWorkflowConfigCiCacheBucket - workflowRequest.CiArtifactLocation, workflowRequest.CiArtifactBucket, workflowRequest.CiArtifactFileName = impl.buildS3ArtifactLocation(ciWorkflowConfigLogsBucket, savedWf) - workflowRequest.BlobStorageS3Config = &blob_storage.BlobStorageS3Config{ - AccessKey: impl.config.BlobStorageS3AccessKey, - Passkey: impl.config.BlobStorageS3SecretKey, - EndpointUrl: impl.config.BlobStorageS3Endpoint, - IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - CiCacheBucketName: ciWorkflowConfigCiCacheBucket, - CiCacheRegion: ciWorkflowConfigCiCacheRegion, - CiCacheBucketVersioning: impl.config.BlobStorageS3BucketVersioned, - CiArtifactBucketName: workflowRequest.CiArtifactBucket, - CiArtifactRegion: impl.config.GetDefaultCdLogsBucketRegion(), - CiArtifactBucketVersioning: impl.config.BlobStorageS3BucketVersioned, - CiLogBucketName: impl.config.GetDefaultBuildLogsBucket(), - CiLogRegion: impl.config.GetDefaultCdLogsBucketRegion(), - CiLogBucketVersioning: impl.config.BlobStorageS3BucketVersioned, - } - case types.BLOB_STORAGE_GCP: - workflowRequest.GcpBlobConfig = &blob_storage.GcpBlobConfig{ - CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - CacheBucketName: ciWorkflowConfigCiCacheBucket, - LogBucketName: ciWorkflowConfigLogsBucket, - ArtifactBucketName: ciWorkflowConfigLogsBucket, - } - workflowRequest.CiArtifactLocation = impl.buildDefaultArtifactLocation(savedWf) - workflowRequest.CiArtifactFileName = workflowRequest.CiArtifactLocation - case types.BLOB_STORAGE_AZURE: - workflowRequest.AzureBlobConfig = &blob_storage.AzureBlobConfig{ - Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, - AccountName: impl.config.AzureAccountName, - BlobContainerCiCache: impl.config.AzureBlobContainerCiCache, - AccountKey: impl.config.AzureAccountKey, - BlobContainerCiLog: impl.config.AzureBlobContainerCiLog, - BlobContainerArtifact: impl.config.AzureBlobContainerCiLog, - } - workflowRequest.BlobStorageS3Config = &blob_storage.BlobStorageS3Config{ - EndpointUrl: impl.config.AzureGatewayUrl, - IsInSecure: impl.config.AzureGatewayConnectionInsecure, - CiLogBucketName: impl.config.AzureBlobContainerCiLog, - CiLogRegion: impl.config.DefaultCacheBucketRegion, - CiLogBucketVersioning: impl.config.BlobStorageS3BucketVersioned, - AccessKey: impl.config.AzureAccountName, - } - workflowRequest.CiArtifactLocation = impl.buildDefaultArtifactLocation(savedWf) - workflowRequest.CiArtifactFileName = workflowRequest.CiArtifactLocation - default: - if impl.config.BlobStorageEnabled { - return nil, fmt.Errorf("blob storage %s not supported", workflowRequest.CloudProvider) - } - } - return workflowRequest, nil -} - -func (impl *CiServiceImpl) GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, customTag string, customTagId int, buildImagePath string, buildImagedockerRegistryId string) (map[string][]string, map[string]bean2.RegistryCredentials, string, []int, error) { - - copyContainerImagePluginDetail, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(COPY_CONTAINER_IMAGE) - if err != nil && err != pg.ErrNoRows { - impl.Logger.Errorw("error in getting copyContainerImage plugin id", "err", err) - return nil, nil, "", nil, err - } - - pluginIdToVersionMap := make(map[int]string) - for _, p := range copyContainerImagePluginDetail { - pluginIdToVersionMap[p.Id] = p.Version - } - - for _, step := range preCiSteps { - if _, ok := pluginIdToVersionMap[step.RefPluginId]; ok { - // for copyContainerImage plugin parse destination images and save its data in image path reservation table - return nil, nil, "", nil, errors.New("copyContainerImage plugin not allowed in pre-ci step, please remove it and try again") - } - } - - registryCredentialMap := make(map[string]bean2.RegistryCredentials) - registryDestinationImageMap := make(map[string][]string) - var allDestinationImages []string //saving all images to be reserved in this array - - for _, step := range postCiSteps { - if version, ok := pluginIdToVersionMap[step.RefPluginId]; ok { - destinationImageMap, credentialMap, err := impl.pluginInputVariableParser.HandleCopyContainerImagePluginInputVariables(step.InputVars, customTag, buildImagePath, buildImagedockerRegistryId) - if err != nil { - impl.Logger.Errorw("error in parsing copyContainerImage input variable", "err", err) - return nil, nil, "", nil, err - } - if version == COPY_CONTAINER_IMAGE_VERSION_V1 { - // this is needed in ci runner only for v1 - registryDestinationImageMap = destinationImageMap - } - for _, images := range destinationImageMap { - allDestinationImages = append(allDestinationImages, images...) - } - for k, v := range credentialMap { - registryCredentialMap[k] = v - } - } - } - - pluginArtifactStage := repository5.POST_CI - for _, image := range allDestinationImages { - if image == buildImagePath { - return nil, registryCredentialMap, pluginArtifactStage, nil, - pipelineConfigBean.ErrImagePathInUse - } - } - savedCIArtifacts, err := impl.ciArtifactRepository.FindCiArtifactByImagePaths(allDestinationImages) - if err != nil { - impl.Logger.Errorw("error in fetching artifacts by image path", "err", err) - return nil, nil, pluginArtifactStage, nil, err - } - if len(savedCIArtifacts) > 0 { - // if already present in ci artifact, return "image path already in use error" - return nil, nil, pluginArtifactStage, nil, pipelineConfigBean.ErrImagePathInUse - } - imagePathReservationIds, err := impl.ReserveImagesGeneratedAtPlugin(customTagId, allDestinationImages) - if err != nil { - return nil, nil, pluginArtifactStage, imagePathReservationIds, err - } - return registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imagePathReservationIds, nil -} - -func (impl *CiServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, destinationImages []string) ([]int, error) { - var imagePathReservationIds []int - for _, image := range destinationImages { - imagePathReservationData, err := impl.customTagService.ReserveImagePath(image, customTagId) - if err != nil { - impl.Logger.Errorw("Error in marking custom tag reserved", "err", err) - return imagePathReservationIds, err - } - imagePathReservationIds = append(imagePathReservationIds, imagePathReservationData.Id) - } - return imagePathReservationIds, nil -} - -func buildCiStepsDataFromDockerBuildScripts(dockerBuildScripts []*bean.CiScript) []*pipelineConfigBean.StepObject { - // before plugin support, few variables were set as env vars in ci-runner - // these variables are now moved to global vars in plugin steps, but to avoid error in old scripts adding those variables in payload - inputVars := []*commonBean.VariableObject{ - { - Name: "DOCKER_IMAGE_TAG", - Format: "STRING", - VariableType: commonBean.VariableTypeRefGlobal, - ReferenceVariableName: "DOCKER_IMAGE_TAG", - }, - { - Name: "DOCKER_REPOSITORY", - Format: "STRING", - VariableType: commonBean.VariableTypeRefGlobal, - ReferenceVariableName: "DOCKER_REPOSITORY", - }, - { - Name: "DOCKER_REGISTRY_URL", - Format: "STRING", - VariableType: commonBean.VariableTypeRefGlobal, - ReferenceVariableName: "DOCKER_REGISTRY_URL", - }, - { - Name: "DOCKER_IMAGE", - Format: "STRING", - VariableType: commonBean.VariableTypeRefGlobal, - ReferenceVariableName: "DOCKER_IMAGE", - }, - } - var ciSteps []*pipelineConfigBean.StepObject - for _, dockerBuildScript := range dockerBuildScripts { - ciStep := &pipelineConfigBean.StepObject{ - Name: dockerBuildScript.Name, - Index: dockerBuildScript.Index, - Script: dockerBuildScript.Script, - ArtifactPaths: []string{dockerBuildScript.OutputLocation}, - StepType: string(repository.PIPELINE_STEP_TYPE_INLINE), - ExecutorType: string(repository2.SCRIPT_TYPE_SHELL), - InputVars: inputVars, - } - ciSteps = append(ciSteps, ciStep) - } - return ciSteps -} - -func (impl *CiServiceImpl) buildImageTag(commitHashes map[int]pipelineConfig.GitCommit, id int, wfId int) string { - dockerImageTag := "" - toAppendDevtronParamInTag := true - for _, v := range commitHashes { - if v.WebhookData.Id == 0 { - if v.Commit == "" { - continue - } - dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _getTruncatedImageTag(v.Commit)) - } else { - _targetCheckout := v.WebhookData.Data[bean.WEBHOOK_SELECTOR_TARGET_CHECKOUT_NAME] - if _targetCheckout == "" { - continue - } - // if not PR based then meaning tag based - isPRBasedEvent := v.WebhookData.EventActionType == bean.WEBHOOK_EVENT_MERGED_ACTION_TYPE - if !isPRBasedEvent && impl.config.CiCdConfig.UseImageTagFromGitProviderForTagBasedBuild { - dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _targetCheckout) - } else { - dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _getTruncatedImageTag(_targetCheckout)) - } - if isPRBasedEvent { - _sourceCheckout := v.WebhookData.Data[bean.WEBHOOK_SELECTOR_SOURCE_CHECKOUT_NAME] - dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _getTruncatedImageTag(_sourceCheckout)) - } else { - toAppendDevtronParamInTag = !impl.config.CiCdConfig.UseImageTagFromGitProviderForTagBasedBuild - } - } - } - toAppendDevtronParamInTag = toAppendDevtronParamInTag && dockerImageTag != "" - if toAppendDevtronParamInTag { - dockerImageTag = fmt.Sprintf("%s-%d-%d", dockerImageTag, id, wfId) - } - // replace / with underscore, as docker image tag doesn't support slash. it gives error - dockerImageTag = strings.ReplaceAll(dockerImageTag, "/", "_") - return dockerImageTag -} - -func getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, commitOrCheckOutData string) string { - if dockerImageTag == "" { - dockerImageTag = commitOrCheckOutData - } else { - if commitOrCheckOutData != "" { - dockerImageTag = fmt.Sprintf("%s-%s", dockerImageTag, commitOrCheckOutData) - } - } - return dockerImageTag -} - -func (impl *CiServiceImpl) updateCiWorkflow(request *types.WorkflowRequest, savedWf *pipelineConfig.CiWorkflow) error { - ciBuildConfig := request.CiBuildConfig - ciBuildType := string(ciBuildConfig.CiBuildType) - savedWf.CiBuildType = ciBuildType - return impl.UpdateCiWorkflowWithStage(savedWf) -} - -func _getTruncatedImageTag(imageTag string) string { - _length := len(imageTag) - if _length == 0 { - return imageTag - } - - _truncatedLength := 8 - - if _length < _truncatedLength { - return imageTag - } else { - return imageTag[:_truncatedLength] - } -} - func (impl *CiServiceImpl) WriteCIFailEvent(ciWorkflow *pipelineConfig.CiWorkflow) { event, _ := impl.eventFactory.Build(util2.Fail, &ciWorkflow.CiPipelineId, ciWorkflow.CiPipeline.AppId, nil, util2.CI) material := &buildBean.MaterialTriggerInfo{} diff --git a/pkg/pipeline/pipelineStageVariableParser.go b/pkg/pipeline/pipelineStageVariableParser.go index b97517d30b..bd0d3bce77 100644 --- a/pkg/pipeline/pipelineStageVariableParser.go +++ b/pkg/pipeline/pipelineStageVariableParser.go @@ -30,13 +30,9 @@ import ( ) type copyContainerImagePluginInputVariable = string -type RefPluginName = string const ( - COPY_CONTAINER_IMAGE RefPluginName = "Copy container image" - COPY_CONTAINER_IMAGE_VERSION_V1 = "1.0.0" - COPY_CONTAINER_IMAGE_VERSION_V2 = "2.0.0" - EMPTY_STRING = " " + EMPTY_STRING = " " ) const ( From 3bf9cd7f474e4390581c027b5413aa2113cafedb Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Wed, 2 Apr 2025 10:37:03 +0530 Subject: [PATCH 04/17] wire file missed --- pkg/build/trigger/wire_trigger.go | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 pkg/build/trigger/wire_trigger.go diff --git a/pkg/build/trigger/wire_trigger.go b/pkg/build/trigger/wire_trigger.go new file mode 100644 index 0000000000..dc462bcd88 --- /dev/null +++ b/pkg/build/trigger/wire_trigger.go @@ -0,0 +1,10 @@ +package trigger + +import ( + "github.com/google/wire" +) + +var WireSet = wire.NewSet( + NewServiceImpl, + wire.Bind(new(Service), new(*ServiceImpl)), +) From 3d23b352dcdb4d2f36555c452894258f87e9de11 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Wed, 2 Apr 2025 18:55:20 +0530 Subject: [PATCH 05/17] trigger service moved completely --- .../configure/BuildPipelineRestHandler.go | 2 +- .../configure/PipelineConfigRestHandler.go | 5 +- client/cron/CiStatusUpdateCron.go | 11 +- client/cron/CiTriggerCron.go | 11 +- pkg/build/git/gitWebhook/GitWebhookService.go | 11 +- pkg/build/trigger/Service.go | 433 ++++++++++++++ pkg/build/trigger/Service_ent.go | 10 + pkg/bulkAction/service/BulkUpdateService.go | 8 +- .../in/WorkflowEventProcessorService.go | 10 +- pkg/pipeline/CIHandler_ent.go | 10 - pkg/pipeline/CiHandler.go | 536 +----------------- pkg/workflow/dag/WorkflowDagExecutor.go | 150 +++++ wire_gen.go | 130 ++--- 13 files changed, 700 insertions(+), 627 deletions(-) diff --git a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go index 32448e0abf..9e749ec207 100644 --- a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go @@ -740,7 +740,7 @@ func (handler *PipelineConfigRestHandlerImpl) TriggerCiPipeline(w http.ResponseW ciTriggerRequest.TriggeredBy = userId handler.Logger.Infow("request payload, TriggerCiPipeline", "payload", ciTriggerRequest) response := make(map[string]string) - resp, err := handler.ciHandler.HandleCIManual(ciTriggerRequest) + resp, err := handler.ciTriggerService.HandleCIManual(ciTriggerRequest) if errors.Is(err, bean1.ErrImagePathInUse) { handler.Logger.Errorw("service err duplicate image tag, TriggerCiPipeline", "err", err, "payload", ciTriggerRequest) common.WriteJsonResp(w, err, err, http.StatusConflict) diff --git a/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go b/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go index 5e3e81defd..0980abae24 100644 --- a/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go +++ b/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go @@ -26,6 +26,7 @@ import ( read2 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/read" gitProviderRead "github.com/devtron-labs/devtron/pkg/build/git/gitProvider/read" bean3 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/build/trigger" "github.com/devtron-labs/devtron/pkg/chart/gitOpsConfig" read5 "github.com/devtron-labs/devtron/pkg/chart/read" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" @@ -138,6 +139,7 @@ type PipelineConfigRestHandlerImpl struct { teamReadService read3.TeamReadService environmentRepository repository2.EnvironmentRepository chartReadService read5.ChartReadService + ciTriggerService trigger.Service } func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger *zap.SugaredLogger, @@ -171,7 +173,8 @@ func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger gitProviderReadService gitProviderRead.GitProviderReadService, teamReadService read3.TeamReadService, EnvironmentRepository repository2.EnvironmentRepository, - chartReadService read5.ChartReadService) *PipelineConfigRestHandlerImpl { + chartReadService read5.ChartReadService, + ciTriggerService trigger.Service) *PipelineConfigRestHandlerImpl { envConfig := &PipelineRestHandlerEnvConfig{} err := env.Parse(envConfig) if err != nil { diff --git a/client/cron/CiStatusUpdateCron.go b/client/cron/CiStatusUpdateCron.go index 696feaed10..9aa4e55554 100644 --- a/client/cron/CiStatusUpdateCron.go +++ b/client/cron/CiStatusUpdateCron.go @@ -21,7 +21,7 @@ import ( "github.com/caarlos0/env" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/app" - "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/workflow/dag" cron2 "github.com/devtron-labs/devtron/util/cron" "github.com/robfig/cron/v3" "go.uber.org/zap" @@ -38,12 +38,13 @@ type CiStatusUpdateCronImpl struct { appService app.AppService ciWorkflowStatusUpdateConfig *CiWorkflowStatusUpdateConfig ciPipelineRepository pipelineConfig.CiPipelineRepository - ciHandler pipeline.CiHandler + workflowDagExecutor dag.WorkflowDagExecutor } func NewCiStatusUpdateCronImpl(logger *zap.SugaredLogger, appService app.AppService, ciWorkflowStatusUpdateConfig *CiWorkflowStatusUpdateConfig, ciPipelineRepository pipelineConfig.CiPipelineRepository, - ciHandler pipeline.CiHandler, cronLogger *cron2.CronLoggerImpl) *CiStatusUpdateCronImpl { + cronLogger *cron2.CronLoggerImpl, + workflowDagExecutor dag.WorkflowDagExecutor) *CiStatusUpdateCronImpl { cron := cron.New( cron.WithChain(cron.Recover(cronLogger))) cron.Start() @@ -53,7 +54,7 @@ func NewCiStatusUpdateCronImpl(logger *zap.SugaredLogger, appService app.AppServ appService: appService, ciWorkflowStatusUpdateConfig: ciWorkflowStatusUpdateConfig, ciPipelineRepository: ciPipelineRepository, - ciHandler: ciHandler, + workflowDagExecutor: workflowDagExecutor, } // execute periodically, update ci workflow status for failed process @@ -87,7 +88,7 @@ func (impl *CiStatusUpdateCronImpl) UpdateCiWorkflowStatusFailedCron() { impl.logger.Errorw("error in converting string to int", "err", err) return } - err = impl.ciHandler.UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild) + err = impl.workflowDagExecutor.UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild) if err != nil { impl.logger.Errorw("error in updating ci workflow status for failed workflows", "err", err) return diff --git a/client/cron/CiTriggerCron.go b/client/cron/CiTriggerCron.go index 35f1e9876a..b5b9305afa 100644 --- a/client/cron/CiTriggerCron.go +++ b/client/cron/CiTriggerCron.go @@ -23,7 +23,7 @@ import ( bean2 "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" - "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/build/trigger" "github.com/devtron-labs/devtron/pkg/pipeline/repository" repository3 "github.com/devtron-labs/devtron/pkg/plugin/repository" cron2 "github.com/devtron-labs/devtron/util/cron" @@ -40,13 +40,14 @@ type CiTriggerCronImpl struct { cron *cron.Cron cfg *CiTriggerCronConfig pipelineStageRepository repository.PipelineStageRepository - ciHandler pipeline.CiHandler ciArtifactRepository repository2.CiArtifactRepository globalPluginRepository repository3.GlobalPluginRepository + ciTriggerService trigger.Service } func NewCiTriggerCronImpl(logger *zap.SugaredLogger, cfg *CiTriggerCronConfig, pipelineStageRepository repository.PipelineStageRepository, - ciHandler pipeline.CiHandler, ciArtifactRepository repository2.CiArtifactRepository, globalPluginRepository repository3.GlobalPluginRepository, cronLogger *cron2.CronLoggerImpl) *CiTriggerCronImpl { + ciArtifactRepository repository2.CiArtifactRepository, globalPluginRepository repository3.GlobalPluginRepository, cronLogger *cron2.CronLoggerImpl, + ciTriggerService trigger.Service) *CiTriggerCronImpl { cron := cron.New( cron.WithChain(cron.Recover(cronLogger))) cron.Start() @@ -54,10 +55,10 @@ func NewCiTriggerCronImpl(logger *zap.SugaredLogger, cfg *CiTriggerCronConfig, p logger: logger, cron: cron, pipelineStageRepository: pipelineStageRepository, - ciHandler: ciHandler, cfg: cfg, ciArtifactRepository: ciArtifactRepository, globalPluginRepository: globalPluginRepository, + ciTriggerService: ciTriggerService, } _, err := cron.AddFunc(fmt.Sprintf("@every %dm", cfg.SourceControllerCronTime), impl.TriggerCiCron) @@ -103,7 +104,7 @@ func (impl *CiTriggerCronImpl) TriggerCiCron() { InvalidateCache: false, PipelineType: string(common.CI_JOB), } - _, err = impl.ciHandler.HandleCIManual(ciTriggerRequest) + _, err = impl.ciTriggerService.HandleCIManual(ciTriggerRequest) if err != nil { return } diff --git a/pkg/build/git/gitWebhook/GitWebhookService.go b/pkg/build/git/gitWebhook/GitWebhookService.go index 3f301db311..8b4405a68b 100644 --- a/pkg/build/git/gitWebhook/GitWebhookService.go +++ b/pkg/build/git/gitWebhook/GitWebhookService.go @@ -23,7 +23,7 @@ import ( bean2 "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/build/git/gitWebhook/repository" - "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/build/trigger" "go.uber.org/zap" ) @@ -33,15 +33,16 @@ type GitWebhookService interface { type GitWebhookServiceImpl struct { logger *zap.SugaredLogger - ciHandler pipeline.CiHandler gitWebhookRepository repository.GitWebhookRepository + ciTriggerService trigger.Service } -func NewGitWebhookServiceImpl(Logger *zap.SugaredLogger, ciHandler pipeline.CiHandler, gitWebhookRepository repository.GitWebhookRepository) *GitWebhookServiceImpl { +func NewGitWebhookServiceImpl(Logger *zap.SugaredLogger, gitWebhookRepository repository.GitWebhookRepository, + ciTriggerService trigger.Service) *GitWebhookServiceImpl { return &GitWebhookServiceImpl{ logger: Logger, - ciHandler: ciHandler, gitWebhookRepository: gitWebhookRepository, + ciTriggerService: ciTriggerService, } } @@ -70,7 +71,7 @@ func (impl *GitWebhookServiceImpl) HandleGitWebhook(gitWebhookRequest gitSensor. } } - resp, err := impl.ciHandler.HandleCIWebhook(bean.GitCiTriggerRequest{ + resp, err := impl.ciTriggerService.HandleCIWebhook(bean.GitCiTriggerRequest{ CiPipelineMaterial: ciPipelineMaterial, TriggeredBy: bean2.SYSTEM_USER_ID, // Automatic trigger, system user ExtraEnvironmentVariables: gitWebhookRequest.ExtraEnvironmentVariables, diff --git a/pkg/build/trigger/Service.go b/pkg/build/trigger/Service.go index 67cff996ab..48f217c40e 100644 --- a/pkg/build/trigger/Service.go +++ b/pkg/build/trigger/Service.go @@ -1,15 +1,18 @@ package trigger import ( + "context" "encoding/json" "errors" "fmt" + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/caarlos0/env" blob_storage "github.com/devtron-labs/common-lib/blob-storage" "github.com/devtron-labs/common-lib/utils" bean4 "github.com/devtron-labs/common-lib/utils/bean" commonBean "github.com/devtron-labs/common-lib/workflow" client "github.com/devtron-labs/devtron/client/events" + "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/middleware" "github.com/devtron-labs/devtron/internal/sql/constants" repository5 "github.com/devtron-labs/devtron/internal/sql/repository" @@ -23,6 +26,7 @@ import ( "github.com/devtron-labs/devtron/pkg/attributes" bean3 "github.com/devtron-labs/devtron/pkg/attributes/bean" "github.com/devtron-labs/devtron/pkg/auth/user" + bean6 "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/bean/common" "github.com/devtron-labs/devtron/pkg/build/pipeline" @@ -32,6 +36,7 @@ import ( pipeline2 "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/adapter" pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/plugin" @@ -53,6 +58,10 @@ import ( ) type Service interface { + HandlePodDeleted(ciWorkflow *pipelineConfig.CiWorkflow) + CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error + HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) + HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) TriggerCiPipeline(trigger types.Trigger) (int, error) } @@ -82,6 +91,8 @@ type ServiceImpl struct { pluginInputVariableParser pipeline2.PluginInputVariableParser globalPluginService plugin.GlobalPluginService ciService pipeline2.CiService + ciWorkflowRepository pipelineConfig.CiWorkflowRepository + gitSensorClient gitSensor.Client } func NewServiceImpl(Logger *zap.SugaredLogger, workflowService pipeline2.WorkflowService, @@ -100,6 +111,8 @@ func NewServiceImpl(Logger *zap.SugaredLogger, workflowService pipeline2.Workflo pluginInputVariableParser pipeline2.PluginInputVariableParser, globalPluginService plugin.GlobalPluginService, ciService pipeline2.CiService, + ciWorkflowRepository pipelineConfig.CiWorkflowRepository, + gitSensorClient gitSensor.Client, ) *ServiceImpl { buildxCacheFlags := &BuildxCacheFlags{} err := env.Parse(buildxCacheFlags) @@ -126,6 +139,8 @@ func NewServiceImpl(Logger *zap.SugaredLogger, workflowService pipeline2.Workflo pluginInputVariableParser: pluginInputVariableParser, globalPluginService: globalPluginService, ciService: ciService, + ciWorkflowRepository: ciWorkflowRepository, + gitSensorClient: gitSensorClient, } config, err := types.GetCiConfig() if err != nil { @@ -135,6 +150,424 @@ func NewServiceImpl(Logger *zap.SugaredLogger, workflowService pipeline2.Workflo return cis } +func (impl *ServiceImpl) HandlePodDeleted(ciWorkflow *pipelineConfig.CiWorkflow) { + if !impl.config.WorkflowRetriesEnabled() { + impl.Logger.Debug("ci workflow retry feature disabled") + return + } + retryCount, refCiWorkflow, err := impl.getRefWorkflowAndCiRetryCount(ciWorkflow) + if err != nil { + impl.Logger.Errorw("error in getRefWorkflowAndCiRetryCount", "ciWorkflowId", ciWorkflow.Id, "err", err) + } + impl.Logger.Infow("re-triggering ci by UpdateCiWorkflowStatusFailedCron", "refCiWorkflowId", refCiWorkflow.Id, "ciWorkflow.Status", ciWorkflow.Status, "ciWorkflow.Message", ciWorkflow.Message, "retryCount", retryCount) + err = impl.reTriggerCi(retryCount, refCiWorkflow) + if err != nil { + impl.Logger.Errorw("error in reTriggerCi", "ciWorkflowId", refCiWorkflow.Id, "workflowStatus", ciWorkflow.Status, "ciWorkflowMessage", "ciWorkflow.Message", "retryCount", retryCount, "err", err) + } +} + +func (impl *ServiceImpl) CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error { + + // return if re-trigger feature is disabled + if !impl.config.WorkflowRetriesEnabled() { + impl.Logger.Debug("CI re-trigger is disabled") + return nil + } + + status, message, ciWorkFlow, err := impl.extractPodStatusAndWorkflow(workflowStatus) + if err != nil { + impl.Logger.Errorw("error in extractPodStatusAndWorkflow", "err", err) + return err + } + + if !executors.CheckIfReTriggerRequired(status, message, ciWorkFlow.Status) { + impl.Logger.Debugw("not re-triggering ci", "status", status, "message", message, "ciWorkflowStatus", ciWorkFlow.Status) + return nil + } + + impl.Logger.Debugw("re-triggering ci", "status", status, "message", message, "ciWorkflowStatus", ciWorkFlow.Status, "ciWorkFlowId", ciWorkFlow.Id) + + retryCount, refCiWorkflow, err := impl.getRefWorkflowAndCiRetryCount(ciWorkFlow) + if err != nil { + impl.Logger.Errorw("error while getting retry count value for a ciWorkflow", "ciWorkFlowId", ciWorkFlow.Id) + return err + } + + err = impl.reTriggerCi(retryCount, refCiWorkflow) + if err != nil { + impl.Logger.Errorw("error in reTriggerCi", "err", err, "status", status, "message", message, "retryCount", retryCount, "ciWorkFlowId", ciWorkFlow.Id) + } + return err +} + +func (impl *ServiceImpl) reTriggerCi(retryCount int, refCiWorkflow *pipelineConfig.CiWorkflow) error { + if retryCount >= impl.config.MaxCiWorkflowRetries { + impl.Logger.Infow("maximum retries exhausted for this ciWorkflow", "ciWorkflowId", refCiWorkflow.Id, "retries", retryCount, "configuredRetries", impl.config.MaxCiWorkflowRetries) + return nil + } + impl.Logger.Infow("re-triggering ci for a ci workflow", "ReferenceCiWorkflowId", refCiWorkflow.Id) + ciPipelineMaterialIds := make([]int, 0, len(refCiWorkflow.GitTriggers)) + for id, _ := range refCiWorkflow.GitTriggers { + ciPipelineMaterialIds = append(ciPipelineMaterialIds, id) + } + ciMaterials, err := impl.ciPipelineMaterialRepository.GetByIdsIncludeDeleted(ciPipelineMaterialIds) + if err != nil { + impl.Logger.Errorw("error in getting ci Pipeline Materials using ciPipeline Material Ids", "ciPipelineMaterialIds", ciPipelineMaterialIds, "err", err) + return err + } + + trigger := types.Trigger{} + trigger.BuildTriggerObject(refCiWorkflow, ciMaterials, bean6.SYSTEM_USER_ID, true, nil, "") + + // updating runtime params + trigger.RuntimeParameters, err = impl.updateRuntimeParamsForAutoCI(trigger.PipelineId, trigger.RuntimeParameters) + if err != nil { + impl.Logger.Errorw("err, updateRuntimeParamsForAutoCI", "ciPipelineId", trigger.PipelineId, + "runtimeParameters", trigger.RuntimeParameters, "err", err) + return err + } + _, err = impl.TriggerCiPipeline(trigger) + + if err != nil { + impl.Logger.Errorw("error occurred in re-triggering ciWorkflow", "triggerDetails", trigger, "err", err) + return err + } + return nil +} + +func (impl *ServiceImpl) HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) { + impl.Logger.Debugw("HandleCIManual for pipeline ", "PipelineId", ciTriggerRequest.PipelineId) + commitHashes, runtimeParams, err := impl.buildManualTriggerCommitHashes(ciTriggerRequest) + if err != nil { + return 0, err + } + + ciArtifact, err := impl.ciArtifactRepository.GetLatestArtifactTimeByCiPipelineId(ciTriggerRequest.PipelineId) + if err != nil && err != pg.ErrNoRows { + impl.Logger.Errorw("Error in GetLatestArtifactTimeByCiPipelineId", "err", err, "pipelineId", ciTriggerRequest.PipelineId) + return 0, err + } + + createdOn := time.Time{} + if err != pg.ErrNoRows { + createdOn = ciArtifact.CreatedOn + } + + trigger := types.Trigger{ + PipelineId: ciTriggerRequest.PipelineId, + CommitHashes: commitHashes, + CiMaterials: nil, + TriggeredBy: ciTriggerRequest.TriggeredBy, + InvalidateCache: ciTriggerRequest.InvalidateCache, + RuntimeParameters: runtimeParams, + EnvironmentId: ciTriggerRequest.EnvironmentId, + PipelineType: ciTriggerRequest.PipelineType, + CiArtifactLastFetch: createdOn, + } + id, err := impl.TriggerCiPipeline(trigger) + + if err != nil { + return 0, err + } + return id, nil +} + +func (impl *ServiceImpl) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) { + impl.Logger.Debugw("HandleCIWebhook for material ", "material", gitCiTriggerRequest.CiPipelineMaterial) + ciPipeline, err := impl.GetCiPipeline(gitCiTriggerRequest.CiPipelineMaterial.Id) + if err != nil { + impl.Logger.Errorw("err in getting ci_pipeline by ciPipelineMaterialId", "ciPipelineMaterialId", gitCiTriggerRequest.CiPipelineMaterial.Id, "err", err) + return 0, err + } + if ciPipeline.IsManual || ciPipeline.PipelineType == buildCommonBean.LINKED_CD.ToString() { + impl.Logger.Debugw("not handling for manual pipeline or in case of linked cd", "pipelineId", ciPipeline.Id) + return 0, err + } + + ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(ciPipeline.Id) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return 0, err + } + isValidBuildSequence, err := impl.validateBuildSequence(gitCiTriggerRequest, ciPipeline.Id) + if !isValidBuildSequence { + return 0, errors.New("ignoring older build for ciMaterial " + strconv.Itoa(gitCiTriggerRequest.CiPipelineMaterial.Id) + + " commit " + gitCiTriggerRequest.CiPipelineMaterial.GitCommit.Commit) + } + // updating runtime params + runtimeParams := common.NewRuntimeParameters() + for k, v := range gitCiTriggerRequest.ExtraEnvironmentVariables { + runtimeParams = runtimeParams.AddSystemVariable(k, v) + } + runtimeParams, err = impl.updateRuntimeParamsForAutoCI(ciPipeline.Id, runtimeParams) + if err != nil { + impl.Logger.Errorw("err, updateRuntimeParamsForAutoCI", "ciPipelineId", ciPipeline.Id, + "runtimeParameters", runtimeParams, "err", err) + return 0, err + } + commitHashes, err := impl.buildAutomaticTriggerCommitHashes(ciMaterials, gitCiTriggerRequest) + if err != nil { + return 0, err + } + + trigger := types.Trigger{ + PipelineId: ciPipeline.Id, + CommitHashes: commitHashes, + CiMaterials: ciMaterials, + TriggeredBy: gitCiTriggerRequest.TriggeredBy, + RuntimeParameters: runtimeParams, + } + id, err := impl.TriggerCiPipeline(trigger) + if err != nil { + return 0, err + } + return id, nil +} + +func (impl *ServiceImpl) extractPodStatusAndWorkflow(workflowStatus v1alpha1.WorkflowStatus) (string, string, *pipelineConfig.CiWorkflow, error) { + workflowName, status, _, message, _, _ := pipeline2.ExtractWorkflowStatus(workflowStatus) + if workflowName == "" { + impl.Logger.Errorw("extract workflow status, invalid wf name", "workflowName", workflowName, "status", status, "message", message) + return status, message, nil, errors.New("invalid wf name") + } + workflowId, err := strconv.Atoi(workflowName[:strings.Index(workflowName, "-")]) + if err != nil { + impl.Logger.Errorw("extract workflowId, invalid wf name", "workflowName", workflowName, "err", err) + return status, message, nil, err + } + + savedWorkflow, err := impl.ciWorkflowRepository.FindById(workflowId) + if err != nil { + impl.Logger.Errorw("cannot get saved wf", "workflowId", workflowId, "err", err) + return status, message, nil, err + } + + return status, message, savedWorkflow, err + +} + +func (impl *ServiceImpl) getRefWorkflowAndCiRetryCount(savedWorkflow *pipelineConfig.CiWorkflow) (int, *pipelineConfig.CiWorkflow, error) { + var err error + + if savedWorkflow.ReferenceCiWorkflowId != 0 { + savedWorkflow, err = impl.ciWorkflowRepository.FindById(savedWorkflow.ReferenceCiWorkflowId) + } + if err != nil { + impl.Logger.Errorw("cannot get saved wf", "err", err) + return 0, savedWorkflow, err + } + retryCount, err := impl.ciWorkflowRepository.FindRetriedWorkflowCountByReferenceId(savedWorkflow.Id) + return retryCount, savedWorkflow, err +} + +func (impl *ServiceImpl) validateBuildSequence(gitCiTriggerRequest bean.GitCiTriggerRequest, pipelineId int) (bool, error) { + isValid := true + lastTriggeredBuild, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflow(pipelineId) + if !(lastTriggeredBuild.Status == string(v1alpha1.NodePending) || lastTriggeredBuild.Status == string(v1alpha1.NodeRunning)) { + return true, nil + } + if err != nil && !util.IsErrNoRows(err) { + impl.Logger.Errorw("cannot get last build for pipeline", "pipelineId", pipelineId) + return false, err + } + + ciPipelineMaterial := gitCiTriggerRequest.CiPipelineMaterial + + if ciPipelineMaterial.Type == string(constants.SOURCE_TYPE_BRANCH_FIXED) { + if ciPipelineMaterial.GitCommit.Date.Before(lastTriggeredBuild.GitTriggers[ciPipelineMaterial.Id].Date) { + impl.Logger.Warnw("older commit cannot be built for pipeline", "pipelineId", pipelineId, "ciMaterial", gitCiTriggerRequest.CiPipelineMaterial.Id) + isValid = false + } + } + + return isValid, nil +} + +func (impl *ServiceImpl) buildAutomaticTriggerCommitHashes(ciMaterials []*pipelineConfig.CiPipelineMaterial, request bean.GitCiTriggerRequest) (map[int]pipelineConfig.GitCommit, error) { + commitHashes := map[int]pipelineConfig.GitCommit{} + for _, ciMaterial := range ciMaterials { + if ciMaterial.Id == request.CiPipelineMaterial.Id || len(ciMaterials) == 1 { + request.CiPipelineMaterial.GitCommit = SetGitCommitValuesForBuildingCommitHash(ciMaterial, request.CiPipelineMaterial.GitCommit) + commitHashes[ciMaterial.Id] = request.CiPipelineMaterial.GitCommit + } else { + // this is possible in case of non Webhook, as there would be only one pipeline material per git material in case of PR + lastCommit, err := impl.getLastSeenCommit(ciMaterial.Id) + if err != nil { + return map[int]pipelineConfig.GitCommit{}, err + } + lastCommit = SetGitCommitValuesForBuildingCommitHash(ciMaterial, lastCommit) + commitHashes[ciMaterial.Id] = lastCommit + } + } + return commitHashes, nil +} + +func (impl *ServiceImpl) GetCiPipeline(ciMaterialId int) (*pipelineConfig.CiPipeline, error) { + ciMaterial, err := impl.ciPipelineMaterialRepository.GetById(ciMaterialId) + if err != nil { + return nil, err + } + ciPipeline := ciMaterial.CiPipeline + return ciPipeline, nil +} + +func (impl *ServiceImpl) buildManualTriggerCommitHashes(ciTriggerRequest bean.CiTriggerRequest) (map[int]pipelineConfig.GitCommit, *common.RuntimeParameters, error) { + commitHashes := map[int]pipelineConfig.GitCommit{} + runtimeParams := impl.getRuntimeParamsForBuildingManualTriggerHashes(ciTriggerRequest) + for _, ciPipelineMaterial := range ciTriggerRequest.CiPipelineMaterial { + + pipeLineMaterialFromDb, err := impl.ciPipelineMaterialRepository.GetById(ciPipelineMaterial.Id) + if err != nil { + impl.Logger.Errorw("err in fetching pipeline material by id", "err", err) + return map[int]pipelineConfig.GitCommit{}, nil, err + } + + pipelineType := pipeLineMaterialFromDb.Type + if pipelineType == constants.SOURCE_TYPE_BRANCH_FIXED { + gitCommit, err := impl.BuildManualTriggerCommitHashesForSourceTypeBranchFix(ciPipelineMaterial, pipeLineMaterialFromDb) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return map[int]pipelineConfig.GitCommit{}, nil, err + } + commitHashes[ciPipelineMaterial.Id] = gitCommit + + } else if pipelineType == constants.SOURCE_TYPE_WEBHOOK { + gitCommit, extraEnvVariables, err := impl.BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial, pipeLineMaterialFromDb) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return map[int]pipelineConfig.GitCommit{}, nil, err + } + commitHashes[ciPipelineMaterial.Id] = gitCommit + for key, value := range extraEnvVariables { + runtimeParams = runtimeParams.AddSystemVariable(key, value) + } + } + } + return commitHashes, runtimeParams, nil +} + +func (impl *ServiceImpl) BuildManualTriggerCommitHashesForSourceTypeBranchFix(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (pipelineConfig.GitCommit, error) { + commitMetadataRequest := &gitSensor.CommitMetadataRequest{ + PipelineMaterialId: ciPipelineMaterial.Id, + GitHash: ciPipelineMaterial.GitCommit.Commit, + GitTag: ciPipelineMaterial.GitTag, + } + gitCommitResponse, err := impl.gitSensorClient.GetCommitMetadataForPipelineMaterial(context.Background(), commitMetadataRequest) + if err != nil { + impl.Logger.Errorw("err in fetching commit metadata", "commitMetadataRequest", commitMetadataRequest, "err", err) + return pipelineConfig.GitCommit{}, err + } + if gitCommitResponse == nil { + return pipelineConfig.GitCommit{}, errors.New("commit not found") + } + + gitCommit := pipelineConfig.GitCommit{ + Commit: gitCommitResponse.Commit, + Author: gitCommitResponse.Author, + Date: gitCommitResponse.Date, + Message: gitCommitResponse.Message, + Changes: gitCommitResponse.Changes, + GitRepoName: pipeLineMaterialFromDb.GitMaterial.Name[strings.Index(pipeLineMaterialFromDb.GitMaterial.Name, "-")+1:], + GitRepoUrl: pipeLineMaterialFromDb.GitMaterial.Url, + CiConfigureSourceValue: pipeLineMaterialFromDb.Value, + CiConfigureSourceType: pipeLineMaterialFromDb.Type, + } + + return gitCommit, nil +} + +func (impl *ServiceImpl) BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (pipelineConfig.GitCommit, map[string]string, error) { + webhookDataInput := ciPipelineMaterial.GitCommit.WebhookData + + // fetch webhook data on the basis of Id + webhookDataRequest := &gitSensor.WebhookDataRequest{ + Id: webhookDataInput.Id, + CiPipelineMaterialId: ciPipelineMaterial.Id, + } + + webhookAndCiData, err := impl.gitSensorClient.GetWebhookData(context.Background(), webhookDataRequest) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return pipelineConfig.GitCommit{}, nil, err + } + webhookData := webhookAndCiData.WebhookData + + // if webhook event is of merged type, then fetch latest commit for target branch + if webhookData.EventActionType == bean.WEBHOOK_EVENT_MERGED_ACTION_TYPE { + + // get target branch name from webhook + targetBranchName := webhookData.Data[bean.WEBHOOK_SELECTOR_TARGET_BRANCH_NAME_NAME] + if targetBranchName == "" { + impl.Logger.Error("target branch not found from webhook data") + return pipelineConfig.GitCommit{}, nil, err + } + + // get latest commit hash for target branch + latestCommitMetadataRequest := &gitSensor.CommitMetadataRequest{ + PipelineMaterialId: ciPipelineMaterial.Id, + BranchName: targetBranchName, + } + + latestCommit, err := impl.gitSensorClient.GetCommitMetadata(context.Background(), latestCommitMetadataRequest) + + if err != nil { + impl.Logger.Errorw("err", "err", err) + return pipelineConfig.GitCommit{}, nil, err + } + + // update webhookData (local) with target latest hash + webhookData.Data[bean.WEBHOOK_SELECTOR_TARGET_CHECKOUT_NAME] = latestCommit.Commit + + } + + // build git commit + gitCommit := pipelineConfig.GitCommit{ + GitRepoName: pipeLineMaterialFromDb.GitMaterial.Name[strings.Index(pipeLineMaterialFromDb.GitMaterial.Name, "-")+1:], + GitRepoUrl: pipeLineMaterialFromDb.GitMaterial.Url, + CiConfigureSourceValue: pipeLineMaterialFromDb.Value, + CiConfigureSourceType: pipeLineMaterialFromDb.Type, + WebhookData: pipelineConfig.WebhookData{ + Id: int(webhookData.Id), + EventActionType: webhookData.EventActionType, + Data: webhookData.Data, + }, + } + + return gitCommit, webhookAndCiData.ExtraEnvironmentVariables, nil +} + +func (impl *ServiceImpl) getLastSeenCommit(ciMaterialId int) (pipelineConfig.GitCommit, error) { + var materialIds []int + materialIds = append(materialIds, ciMaterialId) + headReq := &gitSensor.HeadRequest{ + MaterialIds: materialIds, + } + res, err := impl.gitSensorClient.GetHeadForPipelineMaterials(context.Background(), headReq) + if err != nil { + return pipelineConfig.GitCommit{}, err + } + if len(res) == 0 { + return pipelineConfig.GitCommit{}, errors.New("received empty response") + } + gitCommit := pipelineConfig.GitCommit{ + Commit: res[0].GitCommit.Commit, + Author: res[0].GitCommit.Author, + Date: res[0].GitCommit.Date, + Message: res[0].GitCommit.Message, + Changes: res[0].GitCommit.Changes, + } + return gitCommit, nil +} + +func SetGitCommitValuesForBuildingCommitHash(ciMaterial *pipelineConfig.CiPipelineMaterial, oldGitCommit pipelineConfig.GitCommit) pipelineConfig.GitCommit { + newGitCommit := oldGitCommit + newGitCommit.CiConfigureSourceType = ciMaterial.Type + newGitCommit.CiConfigureSourceValue = ciMaterial.Value + newGitCommit.GitRepoUrl = ciMaterial.GitMaterial.Url + newGitCommit.GitRepoName = ciMaterial.GitMaterial.Name[strings.Index(ciMaterial.GitMaterial.Name, "-")+1:] + return newGitCommit +} + func (impl *ServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { impl.Logger.Debug("ci pipeline manual trigger") ciMaterials, err := impl.GetCiMaterials(trigger.PipelineId, trigger.CiMaterials) diff --git a/pkg/build/trigger/Service_ent.go b/pkg/build/trigger/Service_ent.go index b20094eca7..cd22daa921 100644 --- a/pkg/build/trigger/Service_ent.go +++ b/pkg/build/trigger/Service_ent.go @@ -2,11 +2,21 @@ package trigger import ( "github.com/devtron-labs/common-lib/imageScan/bean" + bean3 "github.com/devtron-labs/devtron/pkg/bean" + "github.com/devtron-labs/devtron/pkg/bean/common" bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" ) +func (impl *ServiceImpl) updateRuntimeParamsForAutoCI(ciPipelineId int, runtimeParameters *common.RuntimeParameters) (*common.RuntimeParameters, error) { + return runtimeParameters, nil +} + +func (impl *ServiceImpl) getRuntimeParamsForBuildingManualTriggerHashes(ciTriggerRequest bean3.CiTriggerRequest) *common.RuntimeParameters { + return common.NewRuntimeParameters() +} + func (impl *ServiceImpl) fetchImageScanExecutionMedium() (*repository.ScanToolMetadata, bean.ScanExecutionMedium, error) { return &repository.ScanToolMetadata{}, "", nil } diff --git a/pkg/bulkAction/service/BulkUpdateService.go b/pkg/bulkAction/service/BulkUpdateService.go index 3b94648b69..afd8efeb41 100644 --- a/pkg/bulkAction/service/BulkUpdateService.go +++ b/pkg/bulkAction/service/BulkUpdateService.go @@ -35,6 +35,7 @@ import ( "github.com/devtron-labs/devtron/internal/util" appWorkflow2 "github.com/devtron-labs/devtron/pkg/appWorkflow" bean2 "github.com/devtron-labs/devtron/pkg/bean" + "github.com/devtron-labs/devtron/pkg/build/trigger" bean4 "github.com/devtron-labs/devtron/pkg/bulkAction/bean" "github.com/devtron-labs/devtron/pkg/bulkAction/utils" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" @@ -102,6 +103,7 @@ type BulkUpdateServiceImpl struct { chartRefService chartRef.ChartRefService deployedAppService deployedApp.DeployedAppService cdPipelineEventPublishService out.CDPipelineEventPublishService + ciTriggerService trigger.Service } func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateRepository, @@ -121,7 +123,8 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateReposito deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, chartRefService chartRef.ChartRefService, deployedAppService deployedApp.DeployedAppService, - cdPipelineEventPublishService out.CDPipelineEventPublishService) *BulkUpdateServiceImpl { + cdPipelineEventPublishService out.CDPipelineEventPublishService, + ciTriggerService trigger.Service) *BulkUpdateServiceImpl { return &BulkUpdateServiceImpl{ bulkUpdateRepository: bulkUpdateRepository, logger: logger, @@ -141,6 +144,7 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateReposito chartRefService: chartRefService, deployedAppService: deployedAppService, cdPipelineEventPublishService: cdPipelineEventPublishService, + ciTriggerService: ciTriggerService, } } @@ -1467,7 +1471,7 @@ func (impl BulkUpdateServiceImpl) BulkBuildTrigger(request *bean4.BulkApplicatio } ciTriggerRequest := latestCommitsMap[pipeline.CiPipelineId] - _, err = impl.ciHandler.HandleCIManual(ciTriggerRequest) + _, err = impl.ciTriggerService.HandleCIManual(ciTriggerRequest) if err != nil { impl.logger.Errorw("service err, HandleCIManual", "err", err, "ciTriggerRequest", ciTriggerRequest) //return nil, err diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index 2800e9a3bb..5e97d5d99a 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -35,6 +35,7 @@ import ( util3 "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" userBean "github.com/devtron-labs/devtron/pkg/auth/user/bean" + "github.com/devtron-labs/devtron/pkg/build/trigger" "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/deployedApp" deploymentBean "github.com/devtron-labs/devtron/pkg/deployment/deployedApp/bean" @@ -91,6 +92,9 @@ type WorkflowEventProcessorImpl struct { devtronAppReleaseContextMapLock *sync.Mutex appServiceConfig *app.AppServiceConfig + //ent only + ciTriggerService trigger.Service + // repositories import to be removed pipelineRepository pipelineConfig.PipelineRepository ciArtifactRepository repository.CiArtifactRepository @@ -118,7 +122,8 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, ciArtifactRepository repository.CiArtifactRepository, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, - deploymentConfigService common.DeploymentConfigService) (*WorkflowEventProcessorImpl, error) { + deploymentConfigService common.DeploymentConfigService, + ciTriggerService trigger.Service) (*WorkflowEventProcessorImpl, error) { impl := &WorkflowEventProcessorImpl{ logger: logger, pubSubClient: pubSubClient, @@ -145,6 +150,7 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, ciArtifactRepository: ciArtifactRepository, cdWorkflowRepository: cdWorkflowRepository, deploymentConfigService: deploymentConfigService, + ciTriggerService: ciTriggerService, } appServiceConfig, err := app.GetAppServiceConfig() if err != nil { @@ -394,7 +400,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCIWorkflowStatusUpdate() error return } - err = impl.ciHandler.CheckAndReTriggerCI(wfStatus) + err = impl.ciTriggerService.CheckAndReTriggerCI(wfStatus) if err != nil { impl.logger.Errorw("error in checking and re triggering ci", "err", err) //don't return as we have to update the workflow status diff --git a/pkg/pipeline/CIHandler_ent.go b/pkg/pipeline/CIHandler_ent.go index 7e1986b518..4d41cf80a3 100644 --- a/pkg/pipeline/CIHandler_ent.go +++ b/pkg/pipeline/CIHandler_ent.go @@ -2,25 +2,15 @@ package pipeline import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/pkg/bean" - "github.com/devtron-labs/devtron/pkg/bean/common" ) type CiHandlerEnt interface { } -func (impl *CiHandlerImpl) updateRuntimeParamsForAutoCI(ciPipelineId int, runtimeParameters *common.RuntimeParameters) (*common.RuntimeParameters, error) { - return runtimeParameters, nil -} - func (impl *CiHandlerImpl) updateResourceStatusInCache(ciWorkflowId int, podName string, namespace string, status string) { //do nothing } -func (impl *CiHandlerImpl) getRuntimeParamsForBuildingManualTriggerHashes(ciTriggerRequest bean.CiTriggerRequest) *common.RuntimeParameters { - return common.NewRuntimeParameters() -} - func (impl *CiHandlerImpl) getPipelineIdForTriggerView(pipeline *pipelineConfig.CiPipeline) (pipelineId int) { if pipeline.ParentCiPipeline == 0 { pipelineId = pipeline.Id diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index aa018f6b96..20d18c93eb 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -23,13 +23,9 @@ import ( "fmt" "github.com/devtron-labs/common-lib/utils" "github.com/devtron-labs/common-lib/utils/workFlow" - constants2 "github.com/devtron-labs/devtron/internal/sql/constants" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" - bean6 "github.com/devtron-labs/devtron/pkg/auth/user/bean" - "github.com/devtron-labs/devtron/pkg/bean/common" "github.com/devtron-labs/devtron/pkg/build/artifacts/imageTagging" buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" - buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" "github.com/devtron-labs/devtron/pkg/cluster/adapter" clusterBean "github.com/devtron-labs/devtron/pkg/cluster/bean" "github.com/devtron-labs/devtron/pkg/cluster/environment" @@ -59,7 +55,6 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/util/rbac" - errors2 "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/rest" "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" @@ -73,9 +68,9 @@ import ( ) type CiHandler interface { - HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) - HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) - CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error + // HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) + //HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) + //CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error FetchMaterialsByPipelineId(pipelineId int, showAll bool) ([]buildBean.CiPipelineMaterialResponse, error) FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId int, gitMaterialId int, showAll bool) ([]buildBean.CiPipelineMaterialResponse, error) FetchWorkflowDetails(appId int, pipelineId int, buildId int) (types.WorkflowResponse, error) @@ -93,7 +88,7 @@ type CiHandler interface { FetchCiStatusForTriggerViewV1(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) RefreshMaterialByCiPipelineMaterialId(gitMaterialId int) (refreshRes *gitSensor.RefreshGitMaterialResponse, err error) FetchMaterialInfoByArtifactId(ciArtifactId int, envId int) (*types.GitTriggerInfoResponse, error) - UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error + //UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error FetchCiStatusForTriggerViewForEnvironment(request resourceGroup.ResourceGroupingRequest, token string) ([]*pipelineConfig.CiWorkflowStatus, error) CiHandlerEnt } @@ -172,187 +167,6 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline return cih } -func (impl *CiHandlerImpl) CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error { - - // return if re-trigger feature is disabled - if !impl.config.WorkflowRetriesEnabled() { - impl.Logger.Debug("CI re-trigger is disabled") - return nil - } - - status, message, ciWorkFlow, err := impl.extractPodStatusAndWorkflow(workflowStatus) - if err != nil { - impl.Logger.Errorw("error in extractPodStatusAndWorkflow", "err", err) - return err - } - - if !executors.CheckIfReTriggerRequired(status, message, ciWorkFlow.Status) { - impl.Logger.Debugw("not re-triggering ci", "status", status, "message", message, "ciWorkflowStatus", ciWorkFlow.Status) - return nil - } - - impl.Logger.Debugw("re-triggering ci", "status", status, "message", message, "ciWorkflowStatus", ciWorkFlow.Status, "ciWorkFlowId", ciWorkFlow.Id) - - retryCount, refCiWorkflow, err := impl.getRefWorkflowAndCiRetryCount(ciWorkFlow) - if err != nil { - impl.Logger.Errorw("error while getting retry count value for a ciWorkflow", "ciWorkFlowId", ciWorkFlow.Id) - return err - } - - err = impl.reTriggerCi(retryCount, refCiWorkflow) - if err != nil { - impl.Logger.Errorw("error in reTriggerCi", "err", err, "status", status, "message", message, "retryCount", retryCount, "ciWorkFlowId", ciWorkFlow.Id) - } - return err -} - -func (impl *CiHandlerImpl) reTriggerCi(retryCount int, refCiWorkflow *pipelineConfig.CiWorkflow) error { - if retryCount >= impl.config.MaxCiWorkflowRetries { - impl.Logger.Infow("maximum retries exhausted for this ciWorkflow", "ciWorkflowId", refCiWorkflow.Id, "retries", retryCount, "configuredRetries", impl.config.MaxCiWorkflowRetries) - return nil - } - impl.Logger.Infow("re-triggering ci for a ci workflow", "ReferenceCiWorkflowId", refCiWorkflow.Id) - ciPipelineMaterialIds := make([]int, 0, len(refCiWorkflow.GitTriggers)) - for id, _ := range refCiWorkflow.GitTriggers { - ciPipelineMaterialIds = append(ciPipelineMaterialIds, id) - } - ciMaterials, err := impl.ciPipelineMaterialRepository.GetByIdsIncludeDeleted(ciPipelineMaterialIds) - if err != nil { - impl.Logger.Errorw("error in getting ci Pipeline Materials using ciPipeline Material Ids", "ciPipelineMaterialIds", ciPipelineMaterialIds, "err", err) - return err - } - - trigger := types.Trigger{} - trigger.BuildTriggerObject(refCiWorkflow, ciMaterials, bean6.SYSTEM_USER_ID, true, nil, "") - - // updating runtime params - trigger.RuntimeParameters, err = impl.updateRuntimeParamsForAutoCI(trigger.PipelineId, trigger.RuntimeParameters) - if err != nil { - impl.Logger.Errorw("err, updateRuntimeParamsForAutoCI", "ciPipelineId", trigger.PipelineId, - "runtimeParameters", trigger.RuntimeParameters, "err", err) - return err - } - _, err = impl.ciService.TriggerCiPipeline(trigger) - - if err != nil { - impl.Logger.Errorw("error occurred in re-triggering ciWorkflow", "triggerDetails", trigger, "err", err) - return err - } - return nil -} - -func (impl *CiHandlerImpl) HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) { - impl.Logger.Debugw("HandleCIManual for pipeline ", "PipelineId", ciTriggerRequest.PipelineId) - commitHashes, runtimeParams, err := impl.buildManualTriggerCommitHashes(ciTriggerRequest) - if err != nil { - return 0, err - } - - ciArtifact, err := impl.ciArtifactRepository.GetLatestArtifactTimeByCiPipelineId(ciTriggerRequest.PipelineId) - if err != nil && err != pg.ErrNoRows { - impl.Logger.Errorw("Error in GetLatestArtifactTimeByCiPipelineId", "err", err, "pipelineId", ciTriggerRequest.PipelineId) - return 0, err - } - - createdOn := time.Time{} - if err != pg.ErrNoRows { - createdOn = ciArtifact.CreatedOn - } - - trigger := types.Trigger{ - PipelineId: ciTriggerRequest.PipelineId, - CommitHashes: commitHashes, - CiMaterials: nil, - TriggeredBy: ciTriggerRequest.TriggeredBy, - InvalidateCache: ciTriggerRequest.InvalidateCache, - RuntimeParameters: runtimeParams, - EnvironmentId: ciTriggerRequest.EnvironmentId, - PipelineType: ciTriggerRequest.PipelineType, - CiArtifactLastFetch: createdOn, - } - id, err := impl.ciService.TriggerCiPipeline(trigger) - - if err != nil { - return 0, err - } - return id, nil -} - -func (impl *CiHandlerImpl) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) { - impl.Logger.Debugw("HandleCIWebhook for material ", "material", gitCiTriggerRequest.CiPipelineMaterial) - ciPipeline, err := impl.GetCiPipeline(gitCiTriggerRequest.CiPipelineMaterial.Id) - if err != nil { - impl.Logger.Errorw("err in getting ci_pipeline by ciPipelineMaterialId", "ciPipelineMaterialId", gitCiTriggerRequest.CiPipelineMaterial.Id, "err", err) - return 0, err - } - if ciPipeline.IsManual || ciPipeline.PipelineType == buildCommonBean.LINKED_CD.ToString() { - impl.Logger.Debugw("not handling for manual pipeline or in case of linked cd", "pipelineId", ciPipeline.Id) - return 0, err - } - - ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(ciPipeline.Id) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return 0, err - } - isValidBuildSequence, err := impl.validateBuildSequence(gitCiTriggerRequest, ciPipeline.Id) - if !isValidBuildSequence { - return 0, errors.New("ignoring older build for ciMaterial " + strconv.Itoa(gitCiTriggerRequest.CiPipelineMaterial.Id) + - " commit " + gitCiTriggerRequest.CiPipelineMaterial.GitCommit.Commit) - } - // updating runtime params - runtimeParams := common.NewRuntimeParameters() - for k, v := range gitCiTriggerRequest.ExtraEnvironmentVariables { - runtimeParams = runtimeParams.AddSystemVariable(k, v) - } - runtimeParams, err = impl.updateRuntimeParamsForAutoCI(ciPipeline.Id, runtimeParams) - if err != nil { - impl.Logger.Errorw("err, updateRuntimeParamsForAutoCI", "ciPipelineId", ciPipeline.Id, - "runtimeParameters", runtimeParams, "err", err) - return 0, err - } - commitHashes, err := impl.buildAutomaticTriggerCommitHashes(ciMaterials, gitCiTriggerRequest) - if err != nil { - return 0, err - } - - trigger := types.Trigger{ - PipelineId: ciPipeline.Id, - CommitHashes: commitHashes, - CiMaterials: ciMaterials, - TriggeredBy: gitCiTriggerRequest.TriggeredBy, - RuntimeParameters: runtimeParams, - } - id, err := impl.ciService.TriggerCiPipeline(trigger) - if err != nil { - return 0, err - } - return id, nil -} - -func (impl *CiHandlerImpl) validateBuildSequence(gitCiTriggerRequest bean.GitCiTriggerRequest, pipelineId int) (bool, error) { - isValid := true - lastTriggeredBuild, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflow(pipelineId) - if !(lastTriggeredBuild.Status == string(v1alpha1.NodePending) || lastTriggeredBuild.Status == string(v1alpha1.NodeRunning)) { - return true, nil - } - if err != nil && !util.IsErrNoRows(err) { - impl.Logger.Errorw("cannot get last build for pipeline", "pipelineId", pipelineId) - return false, err - } - - ciPipelineMaterial := gitCiTriggerRequest.CiPipelineMaterial - - if ciPipelineMaterial.Type == string(constants2.SOURCE_TYPE_BRANCH_FIXED) { - if ciPipelineMaterial.GitCommit.Date.Before(lastTriggeredBuild.GitTriggers[ciPipelineMaterial.Id].Date) { - impl.Logger.Warnw("older commit cannot be built for pipeline", "pipelineId", pipelineId, "ciMaterial", gitCiTriggerRequest.CiPipelineMaterial.Id) - isValid = false - } - } - - return isValid, nil -} - func (impl *CiHandlerImpl) RefreshMaterialByCiPipelineMaterialId(gitMaterialId int) (refreshRes *gitSensor.RefreshGitMaterialResponse, err error) { impl.Logger.Debugw("refreshing git material", "id", gitMaterialId) refreshRes, err = impl.gitSensorClient.RefreshGitMaterial(context.Background(), @@ -1157,42 +971,6 @@ func ExtractWorkflowStatus(workflowStatus v1alpha1.WorkflowStatus) (string, stri return workflowName, status, podStatus, message, logLocation, podName } -func (impl *CiHandlerImpl) extractPodStatusAndWorkflow(workflowStatus v1alpha1.WorkflowStatus) (string, string, *pipelineConfig.CiWorkflow, error) { - workflowName, status, _, message, _, _ := ExtractWorkflowStatus(workflowStatus) - if workflowName == "" { - impl.Logger.Errorw("extract workflow status, invalid wf name", "workflowName", workflowName, "status", status, "message", message) - return status, message, nil, errors.New("invalid wf name") - } - workflowId, err := strconv.Atoi(workflowName[:strings.Index(workflowName, "-")]) - if err != nil { - impl.Logger.Errorw("extract workflowId, invalid wf name", "workflowName", workflowName, "err", err) - return status, message, nil, err - } - - savedWorkflow, err := impl.ciWorkflowRepository.FindById(workflowId) - if err != nil { - impl.Logger.Errorw("cannot get saved wf", "workflowId", workflowId, "err", err) - return status, message, nil, err - } - - return status, message, savedWorkflow, err - -} - -func (impl *CiHandlerImpl) getRefWorkflowAndCiRetryCount(savedWorkflow *pipelineConfig.CiWorkflow) (int, *pipelineConfig.CiWorkflow, error) { - var err error - - if savedWorkflow.ReferenceCiWorkflowId != 0 { - savedWorkflow, err = impl.ciWorkflowRepository.FindById(savedWorkflow.ReferenceCiWorkflowId) - } - if err != nil { - impl.Logger.Errorw("cannot get saved wf", "err", err) - return 0, savedWorkflow, err - } - retryCount, err := impl.ciWorkflowRepository.FindRetriedWorkflowCountByReferenceId(savedWorkflow.Id) - return retryCount, savedWorkflow, err -} - func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, error) { workflowName, status, podStatus, message, _, podName := ExtractWorkflowStatus(workflowStatus) if workflowName == "" { @@ -1285,191 +1063,6 @@ func (impl *CiHandlerImpl) stateChanged(status string, podStatus string, msg str return savedWorkflow.Status != status || savedWorkflow.PodStatus != podStatus || savedWorkflow.Message != msg || savedWorkflow.FinishedOn != finishedAt } -func (impl *CiHandlerImpl) GetCiPipeline(ciMaterialId int) (*pipelineConfig.CiPipeline, error) { - ciMaterial, err := impl.ciPipelineMaterialRepository.GetById(ciMaterialId) - if err != nil { - return nil, err - } - ciPipeline := ciMaterial.CiPipeline - return ciPipeline, nil -} - -func (impl *CiHandlerImpl) buildAutomaticTriggerCommitHashes(ciMaterials []*pipelineConfig.CiPipelineMaterial, request bean.GitCiTriggerRequest) (map[int]pipelineConfig.GitCommit, error) { - commitHashes := map[int]pipelineConfig.GitCommit{} - for _, ciMaterial := range ciMaterials { - if ciMaterial.Id == request.CiPipelineMaterial.Id || len(ciMaterials) == 1 { - request.CiPipelineMaterial.GitCommit = SetGitCommitValuesForBuildingCommitHash(ciMaterial, request.CiPipelineMaterial.GitCommit) - commitHashes[ciMaterial.Id] = request.CiPipelineMaterial.GitCommit - } else { - // this is possible in case of non Webhook, as there would be only one pipeline material per git material in case of PR - lastCommit, err := impl.getLastSeenCommit(ciMaterial.Id) - if err != nil { - return map[int]pipelineConfig.GitCommit{}, err - } - lastCommit = SetGitCommitValuesForBuildingCommitHash(ciMaterial, lastCommit) - commitHashes[ciMaterial.Id] = lastCommit - } - } - return commitHashes, nil -} - -func SetGitCommitValuesForBuildingCommitHash(ciMaterial *pipelineConfig.CiPipelineMaterial, oldGitCommit pipelineConfig.GitCommit) pipelineConfig.GitCommit { - newGitCommit := oldGitCommit - newGitCommit.CiConfigureSourceType = ciMaterial.Type - newGitCommit.CiConfigureSourceValue = ciMaterial.Value - newGitCommit.GitRepoUrl = ciMaterial.GitMaterial.Url - newGitCommit.GitRepoName = ciMaterial.GitMaterial.Name[strings.Index(ciMaterial.GitMaterial.Name, "-")+1:] - return newGitCommit -} - -func (impl *CiHandlerImpl) buildManualTriggerCommitHashes(ciTriggerRequest bean.CiTriggerRequest) (map[int]pipelineConfig.GitCommit, *common.RuntimeParameters, error) { - commitHashes := map[int]pipelineConfig.GitCommit{} - runtimeParams := impl.getRuntimeParamsForBuildingManualTriggerHashes(ciTriggerRequest) - for _, ciPipelineMaterial := range ciTriggerRequest.CiPipelineMaterial { - - pipeLineMaterialFromDb, err := impl.ciPipelineMaterialRepository.GetById(ciPipelineMaterial.Id) - if err != nil { - impl.Logger.Errorw("err in fetching pipeline material by id", "err", err) - return map[int]pipelineConfig.GitCommit{}, nil, err - } - - pipelineType := pipeLineMaterialFromDb.Type - if pipelineType == constants2.SOURCE_TYPE_BRANCH_FIXED { - gitCommit, err := impl.BuildManualTriggerCommitHashesForSourceTypeBranchFix(ciPipelineMaterial, pipeLineMaterialFromDb) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return map[int]pipelineConfig.GitCommit{}, nil, err - } - commitHashes[ciPipelineMaterial.Id] = gitCommit - - } else if pipelineType == constants2.SOURCE_TYPE_WEBHOOK { - gitCommit, extraEnvVariables, err := impl.BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial, pipeLineMaterialFromDb) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return map[int]pipelineConfig.GitCommit{}, nil, err - } - commitHashes[ciPipelineMaterial.Id] = gitCommit - for key, value := range extraEnvVariables { - runtimeParams = runtimeParams.AddSystemVariable(key, value) - } - } - } - return commitHashes, runtimeParams, nil -} - -func (impl *CiHandlerImpl) BuildManualTriggerCommitHashesForSourceTypeBranchFix(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (pipelineConfig.GitCommit, error) { - commitMetadataRequest := &gitSensor.CommitMetadataRequest{ - PipelineMaterialId: ciPipelineMaterial.Id, - GitHash: ciPipelineMaterial.GitCommit.Commit, - GitTag: ciPipelineMaterial.GitTag, - } - gitCommitResponse, err := impl.gitSensorClient.GetCommitMetadataForPipelineMaterial(context.Background(), commitMetadataRequest) - if err != nil { - impl.Logger.Errorw("err in fetching commit metadata", "commitMetadataRequest", commitMetadataRequest, "err", err) - return pipelineConfig.GitCommit{}, err - } - if gitCommitResponse == nil { - return pipelineConfig.GitCommit{}, errors.New("commit not found") - } - - gitCommit := pipelineConfig.GitCommit{ - Commit: gitCommitResponse.Commit, - Author: gitCommitResponse.Author, - Date: gitCommitResponse.Date, - Message: gitCommitResponse.Message, - Changes: gitCommitResponse.Changes, - GitRepoName: pipeLineMaterialFromDb.GitMaterial.Name[strings.Index(pipeLineMaterialFromDb.GitMaterial.Name, "-")+1:], - GitRepoUrl: pipeLineMaterialFromDb.GitMaterial.Url, - CiConfigureSourceValue: pipeLineMaterialFromDb.Value, - CiConfigureSourceType: pipeLineMaterialFromDb.Type, - } - - return gitCommit, nil -} - -func (impl *CiHandlerImpl) BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (pipelineConfig.GitCommit, map[string]string, error) { - webhookDataInput := ciPipelineMaterial.GitCommit.WebhookData - - // fetch webhook data on the basis of Id - webhookDataRequest := &gitSensor.WebhookDataRequest{ - Id: webhookDataInput.Id, - CiPipelineMaterialId: ciPipelineMaterial.Id, - } - - webhookAndCiData, err := impl.gitSensorClient.GetWebhookData(context.Background(), webhookDataRequest) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return pipelineConfig.GitCommit{}, nil, err - } - webhookData := webhookAndCiData.WebhookData - - // if webhook event is of merged type, then fetch latest commit for target branch - if webhookData.EventActionType == bean.WEBHOOK_EVENT_MERGED_ACTION_TYPE { - - // get target branch name from webhook - targetBranchName := webhookData.Data[bean.WEBHOOK_SELECTOR_TARGET_BRANCH_NAME_NAME] - if targetBranchName == "" { - impl.Logger.Error("target branch not found from webhook data") - return pipelineConfig.GitCommit{}, nil, err - } - - // get latest commit hash for target branch - latestCommitMetadataRequest := &gitSensor.CommitMetadataRequest{ - PipelineMaterialId: ciPipelineMaterial.Id, - BranchName: targetBranchName, - } - - latestCommit, err := impl.gitSensorClient.GetCommitMetadata(context.Background(), latestCommitMetadataRequest) - - if err != nil { - impl.Logger.Errorw("err", "err", err) - return pipelineConfig.GitCommit{}, nil, err - } - - // update webhookData (local) with target latest hash - webhookData.Data[bean.WEBHOOK_SELECTOR_TARGET_CHECKOUT_NAME] = latestCommit.Commit - - } - - // build git commit - gitCommit := pipelineConfig.GitCommit{ - GitRepoName: pipeLineMaterialFromDb.GitMaterial.Name[strings.Index(pipeLineMaterialFromDb.GitMaterial.Name, "-")+1:], - GitRepoUrl: pipeLineMaterialFromDb.GitMaterial.Url, - CiConfigureSourceValue: pipeLineMaterialFromDb.Value, - CiConfigureSourceType: pipeLineMaterialFromDb.Type, - WebhookData: pipelineConfig.WebhookData{ - Id: int(webhookData.Id), - EventActionType: webhookData.EventActionType, - Data: webhookData.Data, - }, - } - - return gitCommit, webhookAndCiData.ExtraEnvironmentVariables, nil -} - -func (impl *CiHandlerImpl) getLastSeenCommit(ciMaterialId int) (pipelineConfig.GitCommit, error) { - var materialIds []int - materialIds = append(materialIds, ciMaterialId) - headReq := &gitSensor.HeadRequest{ - MaterialIds: materialIds, - } - res, err := impl.gitSensorClient.GetHeadForPipelineMaterials(context.Background(), headReq) - if err != nil { - return pipelineConfig.GitCommit{}, err - } - if len(res) == 0 { - return pipelineConfig.GitCommit{}, errors.New("received empty response") - } - gitCommit := pipelineConfig.GitCommit{ - Commit: res[0].GitCommit.Commit, - Author: res[0].GitCommit.Author, - Date: res[0].GitCommit.Date, - Message: res[0].GitCommit.Message, - Changes: res[0].GitCommit.Changes, - } - return gitCommit, nil -} - func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewV1(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) { ciWorkflowStatuses, err := impl.ciWorkflowRepository.FIndCiWorkflowStatusesByAppId(appId) if err != nil && !util.IsErrNoRows(err) { @@ -1626,127 +1219,6 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId return gitTriggerInfoResponse, nil } -func (impl *CiHandlerImpl) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error { - ciWorkflows, err := impl.ciWorkflowRepository.FindByStatusesIn([]string{constants.Starting, constants.Running}) - if err != nil { - impl.Logger.Errorw("error on fetching ci workflows", "err", err) - return err - } - client, err := impl.K8sUtil.GetClientForInCluster() - if err != nil { - impl.Logger.Errorw("error while fetching k8s client", "error", err) - return err - } - - for _, ciWorkflow := range ciWorkflows { - var isExt bool - var env *repository2.Environment - var restConfig *rest.Config - if ciWorkflow.Namespace != constants.DefaultCiWorkflowNamespace { - isExt = true - env, err = impl.envRepository.FindById(ciWorkflow.EnvironmentId) - if err != nil { - impl.Logger.Errorw("could not fetch stage env", "err", err) - return err - } - restConfig, err = impl.getRestConfig(ciWorkflow) - if err != nil { - return err - } - } - - isEligibleToMarkFailed := false - isPodDeleted := false - if time.Since(ciWorkflow.StartedOn) > (time.Minute * time.Duration(timeoutForFailureCiBuild)) { - - //check weather pod is exists or not, if exits check its status - wf, err := impl.workflowService.GetWorkflowStatus(ciWorkflow.ExecutorType, ciWorkflow.Name, ciWorkflow.Namespace, restConfig) - if err != nil { - impl.Logger.Warnw("unable to fetch ci workflow", "err", err) - statusError, ok := err.(*errors2.StatusError) - if ok && statusError.Status().Code == http.StatusNotFound { - impl.Logger.Warnw("ci workflow not found", "err", err) - isEligibleToMarkFailed = true - } else { - continue - // skip this and process for next ci workflow - } - } - - //if ci workflow is exists, check its pod - if !isEligibleToMarkFailed { - ns := constants.DefaultCiWorkflowNamespace - if isExt { - _, client, err = impl.k8sCommonService.GetCoreClientByClusterId(env.ClusterId) - if err != nil { - impl.Logger.Warnw("error in getting core v1 client using GetCoreClientByClusterId", "err", err, "clusterId", env.Cluster.Id) - continue - } - ns = env.Namespace - } - _, err := impl.K8sUtil.GetPodByName(ns, ciWorkflow.PodName, client) - if err != nil { - impl.Logger.Warnw("unable to fetch ci workflow - pod", "err", err) - statusError, ok := err.(*errors2.StatusError) - if ok && statusError.Status().Code == http.StatusNotFound { - impl.Logger.Warnw("pod not found", "err", err) - isEligibleToMarkFailed = true - } else { - continue - // skip this and process for next ci workflow - } - } - if ciWorkflow.ExecutorType == cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM { - if wf.Status == string(v1alpha1.WorkflowFailed) { - isPodDeleted = true - } - } else { - // check workflow status,get the status - if wf.Status == string(v1alpha1.WorkflowFailed) && wf.Message == cdWorkflow.POD_DELETED_MESSAGE { - isPodDeleted = true - } - } - } - } - if isEligibleToMarkFailed { - ciWorkflow.Status = "Failed" - ciWorkflow.PodStatus = "Failed" - if isPodDeleted { - ciWorkflow.Message = cdWorkflow.POD_DELETED_MESSAGE - // error logging handled inside handlePodDeleted - impl.handlePodDeleted(ciWorkflow) - } else { - ciWorkflow.Message = "marked failed by job" - } - err := impl.ciService.UpdateCiWorkflowWithStage(ciWorkflow) - if err != nil { - impl.Logger.Errorw("unable to update ci workflow, its eligible to mark failed", "err", err) - // skip this and process for next ci workflow - } - err = impl.customTagService.DeactivateImagePathReservation(ciWorkflow.ImagePathReservationId) - if err != nil { - impl.Logger.Errorw("unable to update ci workflow, its eligible to mark failed", "err", err) - } - } - } - return nil -} - -func (impl *CiHandlerImpl) handlePodDeleted(ciWorkflow *pipelineConfig.CiWorkflow) { - if !impl.config.WorkflowRetriesEnabled() { - impl.Logger.Debug("ci workflow retry feature disabled") - return - } - retryCount, refCiWorkflow, err := impl.getRefWorkflowAndCiRetryCount(ciWorkflow) - if err != nil { - impl.Logger.Errorw("error in getRefWorkflowAndCiRetryCount", "ciWorkflowId", ciWorkflow.Id, "err", err) - } - impl.Logger.Infow("re-triggering ci by UpdateCiWorkflowStatusFailedCron", "refCiWorkflowId", refCiWorkflow.Id, "ciWorkflow.Status", ciWorkflow.Status, "ciWorkflow.Message", ciWorkflow.Message, "retryCount", retryCount) - err = impl.reTriggerCi(retryCount, refCiWorkflow) - if err != nil { - impl.Logger.Errorw("error in reTriggerCi", "ciWorkflowId", refCiWorkflow.Id, "workflowStatus", ciWorkflow.Status, "ciWorkflowMessage", "ciWorkflow.Message", "retryCount", retryCount, "err", err) - } -} func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewForEnvironment(request resourceGroup.ResourceGroupingRequest, token string) ([]*pipelineConfig.CiWorkflowStatus, error) { ciWorkflowStatuses := make([]*pipelineConfig.CiWorkflowStatus, 0) var cdPipelines []*pipelineConfig.Pipeline diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index ce94737490..2a70ebe51e 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -24,6 +24,7 @@ import ( "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/devtron-labs/common-lib/async" "github.com/devtron-labs/common-lib/utils" + "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/common-lib/utils/workFlow" bean6 "github.com/devtron-labs/devtron/api/helm-app/bean" client2 "github.com/devtron-labs/devtron/api/helm-app/service" @@ -39,6 +40,9 @@ import ( "github.com/devtron-labs/devtron/pkg/build/artifacts" bean5 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" + "github.com/devtron-labs/devtron/pkg/build/trigger" + "github.com/devtron-labs/devtron/pkg/cluster/adapter" + repository5 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/manifest" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" @@ -46,7 +50,9 @@ import ( triggerBean "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/userDeploymentRequest/service" eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" + k8sPkg "github.com/devtron-labs/devtron/pkg/k8s" "github.com/devtron-labs/devtron/pkg/pipeline" + constants2 "github.com/devtron-labs/devtron/pkg/pipeline/constants" repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning" repository3 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/repository" @@ -58,6 +64,9 @@ import ( "github.com/devtron-labs/devtron/pkg/workflow/dag/helper" error2 "github.com/devtron-labs/devtron/util/error" util2 "github.com/devtron-labs/devtron/util/event" + errors2 "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/rest" + "net/http" "strings" "sync" "time" @@ -83,6 +92,7 @@ import ( ) type WorkflowDagExecutor interface { + UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error UpdateCiWorkflowForCiSuccess(request *bean2.CiArtifactWebhookRequest) (err error) HandleCiSuccessEvent(triggerContext triggerBean.TriggerContext, ciPipelineId int, request *bean2.CiArtifactWebhookRequest, imagePushedAt time.Time) (id int, err error) HandlePreStageSuccessEvent(triggerContext triggerBean.TriggerContext, cdStageCompleteEvent eventProcessorBean.CdStageCompleteEvent) error @@ -134,6 +144,12 @@ type WorkflowDagExecutorImpl struct { asyncRunnable *async.Runnable scanHistoryRepository repository3.ImageScanHistoryRepository imageScanService imageScanning.ImageScanService + + K8sUtil *k8s.K8sServiceImpl + envRepository repository5.EnvironmentRepository + k8sCommonService k8sPkg.K8sCommonService + workflowService pipeline.WorkflowService + ciTriggerService trigger.Service } func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, @@ -162,6 +178,11 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi asyncRunnable *async.Runnable, scanHistoryRepository repository3.ImageScanHistoryRepository, imageScanService imageScanning.ImageScanService, + K8sUtil *k8s.K8sServiceImpl, + envRepository repository5.EnvironmentRepository, + k8sCommonService k8sPkg.K8sCommonService, + workflowService pipeline.WorkflowService, + ciTriggerService trigger.Service, ) *WorkflowDagExecutorImpl { wde := &WorkflowDagExecutorImpl{logger: Logger, pipelineRepository: pipelineRepository, @@ -190,6 +211,11 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi imageScanService: imageScanService, cdWorkflowRunnerService: cdWorkflowRunnerService, ciService: ciService, + K8sUtil: K8sUtil, + envRepository: envRepository, + k8sCommonService: k8sCommonService, + workflowService: workflowService, + ciTriggerService: ciTriggerService, } config, err := types.GetCdConfig() if err != nil { @@ -209,6 +235,130 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi return wde } +func (impl *WorkflowDagExecutorImpl) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error { + ciWorkflows, err := impl.ciWorkflowRepository.FindByStatusesIn([]string{constants2.Starting, constants2.Running}) + if err != nil { + impl.logger.Errorw("error on fetching ci workflows", "err", err) + return err + } + client, err := impl.K8sUtil.GetClientForInCluster() + if err != nil { + impl.logger.Errorw("error while fetching k8s client", "error", err) + return err + } + + for _, ciWorkflow := range ciWorkflows { + var isExt bool + var env *repository5.Environment + var restConfig *rest.Config + if ciWorkflow.Namespace != constants2.DefaultCiWorkflowNamespace { + isExt = true + env, err = impl.envRepository.FindById(ciWorkflow.EnvironmentId) + if err != nil { + impl.logger.Errorw("could not fetch stage env", "err", err) + return err + } + restConfig, err = impl.getRestConfig(ciWorkflow) + if err != nil { + return err + } + } + + isEligibleToMarkFailed := false + isPodDeleted := false + if time.Since(ciWorkflow.StartedOn) > (time.Minute * time.Duration(timeoutForFailureCiBuild)) { + + //check weather pod is exists or not, if exits check its status + wf, err := impl.workflowService.GetWorkflowStatus(ciWorkflow.ExecutorType, ciWorkflow.Name, ciWorkflow.Namespace, restConfig) + if err != nil { + impl.logger.Warnw("unable to fetch ci workflow", "err", err) + statusError, ok := err.(*errors2.StatusError) + if ok && statusError.Status().Code == http.StatusNotFound { + impl.logger.Warnw("ci workflow not found", "err", err) + isEligibleToMarkFailed = true + } else { + continue + // skip this and process for next ci workflow + } + } + + //if ci workflow is exists, check its pod + if !isEligibleToMarkFailed { + ns := constants2.DefaultCiWorkflowNamespace + if isExt { + _, client, err = impl.k8sCommonService.GetCoreClientByClusterId(env.ClusterId) + if err != nil { + impl.logger.Warnw("error in getting core v1 client using GetCoreClientByClusterId", "err", err, "clusterId", env.Cluster.Id) + continue + } + ns = env.Namespace + } + _, err := impl.K8sUtil.GetPodByName(ns, ciWorkflow.PodName, client) + if err != nil { + impl.logger.Warnw("unable to fetch ci workflow - pod", "err", err) + statusError, ok := err.(*errors2.StatusError) + if ok && statusError.Status().Code == http.StatusNotFound { + impl.logger.Warnw("pod not found", "err", err) + isEligibleToMarkFailed = true + } else { + continue + // skip this and process for next ci workflow + } + } + if ciWorkflow.ExecutorType == cdWorkflow2.WORKFLOW_EXECUTOR_TYPE_SYSTEM { + if wf.Status == string(v1alpha1.WorkflowFailed) { + isPodDeleted = true + } + } else { + // check workflow status,get the status + if wf.Status == string(v1alpha1.WorkflowFailed) && wf.Message == cdWorkflow2.POD_DELETED_MESSAGE { + isPodDeleted = true + } + } + } + } + if isEligibleToMarkFailed { + ciWorkflow.Status = "Failed" + ciWorkflow.PodStatus = "Failed" + if isPodDeleted { + ciWorkflow.Message = cdWorkflow2.POD_DELETED_MESSAGE + // error logging handled inside handlePodDeleted + impl.ciTriggerService.HandlePodDeleted(ciWorkflow) + } else { + ciWorkflow.Message = "marked failed by job" + } + err := impl.ciService.UpdateCiWorkflowWithStage(ciWorkflow) + if err != nil { + impl.logger.Errorw("unable to update ci workflow, its eligible to mark failed", "err", err) + // skip this and process for next ci workflow + } + err = impl.customTagService.DeactivateImagePathReservation(ciWorkflow.ImagePathReservationId) + if err != nil { + impl.logger.Errorw("unable to update ci workflow, its eligible to mark failed", "err", err) + } + } + } + return nil +} + +func (impl *WorkflowDagExecutorImpl) getRestConfig(workflow *pipelineConfig.CiWorkflow) (*rest.Config, error) { + env, err := impl.envRepository.FindById(workflow.EnvironmentId) + if err != nil { + impl.logger.Errorw("could not fetch stage env", "err", err) + return nil, err + } + + clusterBean := adapter.GetClusterBean(*env.Cluster) + + clusterConfig := clusterBean.GetClusterConfig() + restConfig, err := impl.K8sUtil.GetRestConfigByCluster(clusterConfig) + if err != nil { + impl.logger.Errorw("error in getting rest config by cluster id", "err", err) + return nil, err + } + return restConfig, nil +} + func (impl *WorkflowDagExecutorImpl) HandleCdStageReTrigger(runner *pipelineConfig.CdWorkflowRunner) error { // do not re-trigger if retries = 0 if !impl.config.WorkflowRetriesEnabled() { diff --git a/wire_gen.go b/wire_gen.go index ff2d593a71..a59599dfb9 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -49,7 +49,7 @@ import ( "github.com/devtron-labs/devtron/api/restHandler/app/pipeline/configure" history2 "github.com/devtron-labs/devtron/api/restHandler/app/pipeline/history" status3 "github.com/devtron-labs/devtron/api/restHandler/app/pipeline/status" - "github.com/devtron-labs/devtron/api/restHandler/app/pipeline/trigger" + trigger2 "github.com/devtron-labs/devtron/api/restHandler/app/pipeline/trigger" "github.com/devtron-labs/devtron/api/restHandler/app/pipeline/webhook" "github.com/devtron-labs/devtron/api/restHandler/app/workflow" "github.com/devtron-labs/devtron/api/restHandler/scopedVariable" @@ -62,7 +62,7 @@ import ( configure2 "github.com/devtron-labs/devtron/api/router/app/pipeline/configure" history3 "github.com/devtron-labs/devtron/api/router/app/pipeline/history" status4 "github.com/devtron-labs/devtron/api/router/app/pipeline/status" - trigger2 "github.com/devtron-labs/devtron/api/router/app/pipeline/trigger" + trigger3 "github.com/devtron-labs/devtron/api/router/app/pipeline/trigger" workflow2 "github.com/devtron-labs/devtron/api/router/app/workflow" server2 "github.com/devtron-labs/devtron/api/server" "github.com/devtron-labs/devtron/api/sse" @@ -100,7 +100,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/deploymentConfig" repository9 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/helper" - repository22 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" + repository23 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/resourceGroup" "github.com/devtron-labs/devtron/internal/util" @@ -147,14 +147,15 @@ import ( read21 "github.com/devtron-labs/devtron/pkg/build/git/gitHost/read" repository26 "github.com/devtron-labs/devtron/pkg/build/git/gitHost/repository" read15 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/read" - repository20 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/repository" + repository21 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/repository" "github.com/devtron-labs/devtron/pkg/build/git/gitProvider" read9 "github.com/devtron-labs/devtron/pkg/build/git/gitProvider/read" - repository11 "github.com/devtron-labs/devtron/pkg/build/git/gitProvider/repository" + repository12 "github.com/devtron-labs/devtron/pkg/build/git/gitProvider/repository" "github.com/devtron-labs/devtron/pkg/build/git/gitWebhook" - repository23 "github.com/devtron-labs/devtron/pkg/build/git/gitWebhook/repository" + repository11 "github.com/devtron-labs/devtron/pkg/build/git/gitWebhook/repository" pipeline2 "github.com/devtron-labs/devtron/pkg/build/pipeline" read14 "github.com/devtron-labs/devtron/pkg/build/pipeline/read" + "github.com/devtron-labs/devtron/pkg/build/trigger" service7 "github.com/devtron-labs/devtron/pkg/bulkAction/service" "github.com/devtron-labs/devtron/pkg/chart" "github.com/devtron-labs/devtron/pkg/chart/gitOpsConfig" @@ -184,7 +185,7 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/manifest/configMapAndSecret" read19 "github.com/devtron-labs/devtron/pkg/deployment/manifest/configMapAndSecret/read" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" - repository16 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" + repository17 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" read12 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/read" @@ -198,7 +199,7 @@ import ( "github.com/devtron-labs/devtron/pkg/devtronResource" "github.com/devtron-labs/devtron/pkg/devtronResource/history/deployment/cdPipeline" read10 "github.com/devtron-labs/devtron/pkg/devtronResource/read" - repository13 "github.com/devtron-labs/devtron/pkg/devtronResource/repository" + repository14 "github.com/devtron-labs/devtron/pkg/devtronResource/repository" "github.com/devtron-labs/devtron/pkg/dockerRegistry" "github.com/devtron-labs/devtron/pkg/eventProcessor" "github.com/devtron-labs/devtron/pkg/eventProcessor/celEvaluator" @@ -212,7 +213,7 @@ import ( "github.com/devtron-labs/devtron/pkg/gitops" "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" config4 "github.com/devtron-labs/devtron/pkg/infraConfig/config" - repository14 "github.com/devtron-labs/devtron/pkg/infraConfig/repository" + repository15 "github.com/devtron-labs/devtron/pkg/infraConfig/repository" "github.com/devtron-labs/devtron/pkg/infraConfig/repository/audit" service2 "github.com/devtron-labs/devtron/pkg/infraConfig/service" audit2 "github.com/devtron-labs/devtron/pkg/infraConfig/service/audit" @@ -231,21 +232,21 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/history" - repository21 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" + repository22 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders" "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders/infraGetters/ci" "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders/infraGetters/job" - repository18 "github.com/devtron-labs/devtron/pkg/pipeline/repository" + repository19 "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" - repository17 "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus/repository" + repository18 "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus/repository" "github.com/devtron-labs/devtron/pkg/plugin" - repository19 "github.com/devtron-labs/devtron/pkg/plugin/repository" + repository20 "github.com/devtron-labs/devtron/pkg/plugin/repository" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning" read18 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/read" repository24 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/repository" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool" - repository15 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" + repository16 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" "github.com/devtron-labs/devtron/pkg/server" @@ -260,7 +261,7 @@ import ( util3 "github.com/devtron-labs/devtron/pkg/util" "github.com/devtron-labs/devtron/pkg/variables" "github.com/devtron-labs/devtron/pkg/variables/parsers" - repository12 "github.com/devtron-labs/devtron/pkg/variables/repository" + repository13 "github.com/devtron-labs/devtron/pkg/variables/repository" "github.com/devtron-labs/devtron/pkg/webhook/helm" "github.com/devtron-labs/devtron/pkg/workflow/cd" read20 "github.com/devtron-labs/devtron/pkg/workflow/cd/read" @@ -485,11 +486,12 @@ func InitializeApp() (*App, error) { clusterRbacServiceImpl := rbac2.NewClusterRbacServiceImpl(environmentServiceImpl, enforcerImpl, enforcerUtilImpl, clusterServiceImplExtended, sugaredLogger, userServiceImpl, clusterReadServiceImpl) clusterRestHandlerImpl := cluster3.NewClusterRestHandlerImpl(clusterServiceImplExtended, genericNoteServiceImpl, clusterDescriptionServiceImpl, sugaredLogger, userServiceImpl, validate, enforcerImpl, deleteServiceExtendedImpl, environmentServiceImpl, clusterRbacServiceImpl) clusterRouterImpl := cluster3.NewClusterRouterImpl(clusterRestHandlerImpl) + gitWebhookRepositoryImpl := repository11.NewGitWebhookRepositoryImpl(db) ciCdConfig, err := types.GetCiCdConfig() if err != nil { return nil, err } - gitProviderRepositoryImpl := repository11.NewGitProviderRepositoryImpl(db) + gitProviderRepositoryImpl := repository12.NewGitProviderRepositoryImpl(db) gitProviderReadServiceImpl := read9.NewGitProviderReadService(sugaredLogger, gitProviderRepositoryImpl) commonBaseServiceImpl := commonService.NewCommonBaseServiceImpl(sugaredLogger, environmentVariables, moduleReadServiceImpl) commonServiceImpl := commonService.NewCommonServiceImpl(sugaredLogger, chartRepositoryImpl, envConfigOverrideRepositoryImpl, dockerArtifactStoreRepositoryImpl, attributesRepositoryImpl, environmentRepositoryImpl, appRepositoryImpl, gitOpsConfigReadServiceImpl, gitProviderReadServiceImpl, envConfigOverrideReadServiceImpl, commonBaseServiceImpl, teamReadServiceImpl) @@ -497,8 +499,8 @@ func InitializeApp() (*App, error) { mergeUtil := util.MergeUtil{ Logger: sugaredLogger, } - scopedVariableRepositoryImpl := repository12.NewScopedVariableRepository(db, sugaredLogger, transactionUtilImpl) - devtronResourceSearchableKeyRepositoryImpl := repository13.NewDevtronResourceSearchableKeyRepositoryImpl(sugaredLogger, db) + scopedVariableRepositoryImpl := repository13.NewScopedVariableRepository(db, sugaredLogger, transactionUtilImpl) + devtronResourceSearchableKeyRepositoryImpl := repository14.NewDevtronResourceSearchableKeyRepositoryImpl(sugaredLogger, db) devtronResourceSearchableKeyServiceImpl, err := read10.NewDevtronResourceSearchableKeyServiceImpl(sugaredLogger, devtronResourceSearchableKeyRepositoryImpl) if err != nil { return nil, err @@ -515,9 +517,9 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - variableEntityMappingRepositoryImpl := repository12.NewVariableEntityMappingRepository(sugaredLogger, db, transactionUtilImpl) + variableEntityMappingRepositoryImpl := repository13.NewVariableEntityMappingRepository(sugaredLogger, db, transactionUtilImpl) variableEntityMappingServiceImpl := variables.NewVariableEntityMappingServiceImpl(variableEntityMappingRepositoryImpl, sugaredLogger) - variableSnapshotHistoryRepositoryImpl := repository12.NewVariableSnapshotHistoryRepository(sugaredLogger, db) + variableSnapshotHistoryRepositoryImpl := repository13.NewVariableSnapshotHistoryRepository(sugaredLogger, db) variableSnapshotHistoryServiceImpl := variables.NewVariableSnapshotHistoryServiceImpl(variableSnapshotHistoryRepositoryImpl, sugaredLogger) variableTemplateParserImpl, err := parsers.NewVariableTemplateParserImpl(sugaredLogger) if err != nil { @@ -538,7 +540,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - infraConfigRepositoryImpl := repository14.NewInfraProfileRepositoryImpl(db, transactionUtilImpl) + infraConfigRepositoryImpl := repository15.NewInfraProfileRepositoryImpl(db, transactionUtilImpl) pipelineOverrideRepositoryImpl := chartConfig.NewPipelineOverrideRepository(db) utilMergeUtil := &util.MergeUtil{ Logger: sugaredLogger, @@ -571,7 +573,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - scanToolMetadataRepositoryImpl := repository15.NewScanToolMetadataRepositoryImpl(db, sugaredLogger) + scanToolMetadataRepositoryImpl := repository16.NewScanToolMetadataRepositoryImpl(db, sugaredLogger) scanToolMetadataServiceImpl := scanTool.NewScanToolMetadataServiceImpl(sugaredLogger, scanToolMetadataRepositoryImpl) moduleServiceImpl := module.NewModuleServiceImpl(sugaredLogger, serverEnvConfigServerEnvConfig, moduleRepositoryImpl, moduleActionAuditLogRepositoryImpl, helmAppServiceImpl, serverDataStoreServerDataStore, serverCacheServiceImpl, moduleCacheServiceImpl, moduleCronServiceImpl, moduleServiceHelperImpl, moduleResourceStatusRepositoryImpl, scanToolMetadataServiceImpl) eventRESTClientImpl := client2.NewEventRESTClientImpl(sugaredLogger, httpClient, eventClientConfig, pubSubClientServiceImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, attributesRepositoryImpl, moduleServiceImpl) @@ -604,11 +606,11 @@ func InitializeApp() (*App, error) { ciTemplateOverrideRepositoryImpl := pipelineConfig.NewCiTemplateOverrideRepositoryImpl(db, sugaredLogger) ciPipelineConfigReadServiceImpl := read14.NewCiPipelineConfigReadServiceImpl(sugaredLogger, ciPipelineRepositoryImpl, ciTemplateOverrideRepositoryImpl) dockerRegistryIpsConfigServiceImpl := dockerRegistry.NewDockerRegistryIpsConfigServiceImpl(sugaredLogger, dockerRegistryIpsConfigRepositoryImpl, k8sServiceImpl, dockerArtifactStoreRepositoryImpl, clusterReadServiceImpl, ciPipelineConfigReadServiceImpl) - appLevelMetricsRepositoryImpl := repository16.NewAppLevelMetricsRepositoryImpl(db, sugaredLogger) - envLevelAppMetricsRepositoryImpl := repository16.NewEnvLevelAppMetricsRepositoryImpl(db, sugaredLogger) + appLevelMetricsRepositoryImpl := repository17.NewAppLevelMetricsRepositoryImpl(db, sugaredLogger) + envLevelAppMetricsRepositoryImpl := repository17.NewEnvLevelAppMetricsRepositoryImpl(db, sugaredLogger) deployedAppMetricsServiceImpl := deployedAppMetrics.NewDeployedAppMetricsServiceImpl(sugaredLogger, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, chartRefServiceImpl) appListingServiceImpl := app2.NewAppListingServiceImpl(sugaredLogger, appListingRepositoryImpl, appDetailsReadServiceImpl, appRepositoryImpl, appListingViewBuilderImpl, pipelineRepositoryImpl, linkoutsRepositoryImpl, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, environmentRepositoryImpl, chartRepositoryImpl, ciPipelineRepositoryImpl, dockerRegistryIpsConfigServiceImpl, userRepositoryImpl, deployedAppMetricsServiceImpl, ciArtifactRepositoryImpl, envConfigOverrideReadServiceImpl, ciPipelineConfigReadServiceImpl) - workflowStageRepositoryImpl := repository17.NewWorkflowStageRepositoryImpl(sugaredLogger, db) + workflowStageRepositoryImpl := repository18.NewWorkflowStageRepositoryImpl(sugaredLogger, db) workFlowStageStatusServiceImpl := workflowStatus.NewWorkflowStageFlowStatusServiceImpl(sugaredLogger, workflowStageRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, transactionUtilImpl) cdWorkflowRunnerServiceImpl := cd.NewCdWorkflowRunnerServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl, workFlowStageStatusServiceImpl, transactionUtilImpl) appServiceImpl := app2.NewAppService(pipelineOverrideRepositoryImpl, utilMergeUtil, sugaredLogger, pipelineRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, appRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, chartTemplateServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, appStatusServiceImpl, installedAppReadServiceImpl, installedAppVersionHistoryRepositoryImpl, scopedVariableCMCSManagerImpl, acdConfig, gitOpsConfigReadServiceImpl, gitOperationServiceImpl, deploymentTemplateServiceImpl, appListingServiceImpl, deploymentConfigServiceImpl, envConfigOverrideReadServiceImpl, cdWorkflowRunnerServiceImpl) @@ -627,8 +629,8 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - pipelineStageRepositoryImpl := repository18.NewPipelineStageRepository(sugaredLogger, db) - globalPluginRepositoryImpl := repository19.NewGlobalPluginRepository(sugaredLogger, db) + pipelineStageRepositoryImpl := repository19.NewPipelineStageRepository(sugaredLogger, db) + globalPluginRepositoryImpl := repository20.NewGlobalPluginRepository(sugaredLogger, db) globalPluginServiceImpl := plugin.NewGlobalPluginService(sugaredLogger, globalPluginRepositoryImpl, pipelineStageRepositoryImpl, userServiceImpl) pipelineStageServiceImpl := pipeline.NewPipelineStageService(sugaredLogger, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, pipelineRepositoryImpl, scopedVariableManagerImpl, globalPluginServiceImpl) ciTemplateRepositoryImpl := pipelineConfig.NewCiTemplateRepositoryImpl(db, sugaredLogger) @@ -638,12 +640,11 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - materialRepositoryImpl := repository20.NewMaterialRepositoryImpl(db) + materialRepositoryImpl := repository21.NewMaterialRepositoryImpl(db) gitMaterialReadServiceImpl := read15.NewGitMaterialReadServiceImpl(sugaredLogger, materialRepositoryImpl) appCrudOperationServiceImpl := app2.NewAppCrudOperationServiceImpl(appLabelRepositoryImpl, sugaredLogger, appRepositoryImpl, userRepositoryImpl, installedAppRepositoryImpl, genericNoteServiceImpl, installedAppDBServiceImpl, crudOperationServiceConfig, dbMigrationServiceImpl, gitMaterialReadServiceImpl) imageTagRepositoryImpl := repository2.NewImageTagRepository(db, sugaredLogger) customTagServiceImpl := pipeline.NewCustomTagService(sugaredLogger, imageTagRepositoryImpl) - pluginInputVariableParserImpl := pipeline.NewPluginInputVariableParserImpl(sugaredLogger, dockerRegistryConfigImpl, customTagServiceImpl) clientConfig, err := gitSensor.GetConfig() if err != nil { return nil, err @@ -652,54 +653,49 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - prePostCdScriptHistoryRepositoryImpl := repository21.NewPrePostCdScriptHistoryRepositoryImpl(sugaredLogger, db) - configMapHistoryRepositoryImpl := repository21.NewConfigMapHistoryRepositoryImpl(sugaredLogger, db, transactionUtilImpl) + prePostCdScriptHistoryRepositoryImpl := repository22.NewPrePostCdScriptHistoryRepositoryImpl(sugaredLogger, db) + configMapHistoryRepositoryImpl := repository22.NewConfigMapHistoryRepositoryImpl(sugaredLogger, db, transactionUtilImpl) configMapHistoryServiceImpl := configMapAndSecret.NewConfigMapHistoryServiceImpl(sugaredLogger, configMapHistoryRepositoryImpl, pipelineRepositoryImpl, configMapRepositoryImpl, userServiceImpl, scopedVariableCMCSManagerImpl) prePostCdScriptHistoryServiceImpl := history.NewPrePostCdScriptHistoryServiceImpl(sugaredLogger, prePostCdScriptHistoryRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl) - gitMaterialHistoryRepositoryImpl := repository21.NewGitMaterialHistoryRepositoyImpl(db) + gitMaterialHistoryRepositoryImpl := repository22.NewGitMaterialHistoryRepositoyImpl(db) gitMaterialHistoryServiceImpl := history.NewGitMaterialHistoryServiceImpl(gitMaterialHistoryRepositoryImpl, sugaredLogger) - ciPipelineHistoryRepositoryImpl := repository21.NewCiPipelineHistoryRepositoryImpl(db, sugaredLogger) + ciPipelineHistoryRepositoryImpl := repository22.NewCiPipelineHistoryRepositoryImpl(db, sugaredLogger) ciPipelineHistoryServiceImpl := history.NewCiPipelineHistoryServiceImpl(ciPipelineHistoryRepositoryImpl, sugaredLogger, ciPipelineRepositoryImpl) ciBuildConfigRepositoryImpl := pipelineConfig.NewCiBuildConfigRepositoryImpl(db, sugaredLogger) ciBuildConfigServiceImpl := pipeline.NewCiBuildConfigServiceImpl(sugaredLogger, ciBuildConfigRepositoryImpl) ciTemplateServiceImpl := pipeline.NewCiTemplateServiceImpl(sugaredLogger, ciBuildConfigServiceImpl, ciTemplateRepositoryImpl, ciTemplateOverrideRepositoryImpl) pipelineConfigRepositoryImpl := chartConfig.NewPipelineConfigRepository(db) configMapServiceImpl := pipeline.NewConfigMapServiceImpl(chartRepositoryImpl, sugaredLogger, chartRepoRepositoryImpl, mergeUtil, pipelineConfigRepositoryImpl, configMapRepositoryImpl, commonServiceImpl, appRepositoryImpl, configMapHistoryServiceImpl, environmentRepositoryImpl, scopedVariableCMCSManagerImpl) - deploymentTemplateHistoryRepositoryImpl := repository21.NewDeploymentTemplateHistoryRepositoryImpl(sugaredLogger, db) + deploymentTemplateHistoryRepositoryImpl := repository22.NewDeploymentTemplateHistoryRepositoryImpl(sugaredLogger, db) deploymentTemplateHistoryServiceImpl := deploymentTemplate.NewDeploymentTemplateHistoryServiceImpl(sugaredLogger, deploymentTemplateHistoryRepositoryImpl, pipelineRepositoryImpl, chartRepositoryImpl, userServiceImpl, cdWorkflowRepositoryImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) chartReadServiceImpl := read16.NewChartReadServiceImpl(sugaredLogger, chartRepositoryImpl, deploymentConfigServiceImpl, deployedAppMetricsServiceImpl, gitOpsConfigReadServiceImpl, chartRefReadServiceImpl) chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, mergeUtil, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl, envConfigOverrideReadServiceImpl, chartReadServiceImpl) ciCdPipelineOrchestratorImpl := pipeline.NewCiCdPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, clientImpl, ciCdConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, pipelineStageServiceImpl, gitMaterialHistoryServiceImpl, ciPipelineHistoryServiceImpl, ciTemplateReadServiceImpl, ciTemplateServiceImpl, dockerArtifactStoreRepositoryImpl, ciArtifactRepositoryImpl, configMapServiceImpl, customTagServiceImpl, genericNoteServiceImpl, chartServiceImpl, transactionUtilImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl, deploymentConfigReadServiceImpl, chartReadServiceImpl) - ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, workFlowStageStatusServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateReadServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl, infraProviderImpl, ciCdPipelineOrchestratorImpl, attributesServiceImpl, ciWorkflowRepositoryImpl, transactionUtilImpl) - ciLogServiceImpl, err := pipeline.NewCiLogServiceImpl(sugaredLogger, ciServiceImpl, k8sServiceImpl) + pluginInputVariableParserImpl := pipeline.NewPluginInputVariableParserImpl(sugaredLogger, dockerRegistryConfigImpl, customTagServiceImpl) + ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workFlowStageStatusServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciWorkflowRepositoryImpl, transactionUtilImpl) + serviceImpl := trigger.NewServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciPipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateReadServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, ciCdPipelineOrchestratorImpl, attributesServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl, ciServiceImpl, ciWorkflowRepositoryImpl, clientImpl) + gitWebhookServiceImpl := gitWebhook.NewGitWebhookServiceImpl(sugaredLogger, gitWebhookRepositoryImpl, serviceImpl) + gitWebhookRestHandlerImpl := restHandler.NewGitWebhookRestHandlerImpl(sugaredLogger, gitWebhookServiceImpl) + ecrConfig, err := pipeline.GetEcrConfig() if err != nil { return nil, err } + ciTemplateHistoryRepositoryImpl := repository22.NewCiTemplateHistoryRepositoryImpl(db, sugaredLogger) + ciTemplateHistoryServiceImpl := history.NewCiTemplateHistoryServiceImpl(ciTemplateHistoryRepositoryImpl, sugaredLogger) resourceGroupRepositoryImpl := resourceGroup.NewResourceGroupRepositoryImpl(db) resourceGroupMappingRepositoryImpl := resourceGroup.NewResourceGroupMappingRepositoryImpl(db) resourceGroupServiceImpl := resourceGroup2.NewResourceGroupServiceImpl(sugaredLogger, resourceGroupRepositoryImpl, resourceGroupMappingRepositoryImpl, enforcerUtilImpl, devtronResourceSearchableKeyServiceImpl, appStatusRepositoryImpl) - imageTaggingRepositoryImpl := repository22.NewImageTaggingRepositoryImpl(db, transactionUtilImpl) + buildPipelineSwitchServiceImpl := pipeline.NewBuildPipelineSwitchServiceImpl(sugaredLogger, ciPipelineConfigReadServiceImpl, ciPipelineRepositoryImpl, ciCdPipelineOrchestratorImpl, pipelineRepositoryImpl, ciWorkflowRepositoryImpl, appWorkflowRepositoryImpl, ciPipelineHistoryServiceImpl, ciTemplateOverrideRepositoryImpl, ciPipelineMaterialRepositoryImpl) + ciPipelineConfigServiceImpl := pipeline.NewCiPipelineConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, gitMaterialReadServiceImpl, appRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigReadServiceImpl, ciPipelineRepositoryImpl, ecrConfig, appWorkflowRepositoryImpl, ciCdConfig, attributesServiceImpl, pipelineStageServiceImpl, ciPipelineMaterialRepositoryImpl, ciTemplateServiceImpl, ciTemplateReadServiceImpl, ciTemplateOverrideRepositoryImpl, ciTemplateHistoryServiceImpl, enforcerUtilImpl, ciWorkflowRepositoryImpl, resourceGroupServiceImpl, customTagServiceImpl, cdWorkflowRepositoryImpl, buildPipelineSwitchServiceImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl) + ciMaterialConfigServiceImpl := pipeline.NewCiMaterialConfigServiceImpl(sugaredLogger, materialRepositoryImpl, ciTemplateReadServiceImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, gitMaterialHistoryServiceImpl, pipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, transactionUtilImpl, gitMaterialReadServiceImpl) + imageTaggingRepositoryImpl := repository23.NewImageTaggingRepositoryImpl(db, transactionUtilImpl) imageTaggingReadServiceImpl, err := read17.NewImageTaggingReadServiceImpl(imageTaggingRepositoryImpl, sugaredLogger) if err != nil { return nil, err } imageTaggingServiceImpl := imageTagging.NewImageTaggingServiceImpl(imageTaggingRepositoryImpl, imageTaggingReadServiceImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, sugaredLogger) - blobStorageConfigServiceImpl := pipeline.NewBlobStorageConfigServiceImpl(sugaredLogger, k8sServiceImpl, ciCdConfig) - ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, workflowServiceImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, environmentServiceImpl, workFlowStageStatusServiceImpl, k8sServiceImpl) - gitWebhookRepositoryImpl := repository23.NewGitWebhookRepositoryImpl(db) - gitWebhookServiceImpl := gitWebhook.NewGitWebhookServiceImpl(sugaredLogger, ciHandlerImpl, gitWebhookRepositoryImpl) - gitWebhookRestHandlerImpl := restHandler.NewGitWebhookRestHandlerImpl(sugaredLogger, gitWebhookServiceImpl) - ecrConfig, err := pipeline.GetEcrConfig() - if err != nil { - return nil, err - } - ciTemplateHistoryRepositoryImpl := repository21.NewCiTemplateHistoryRepositoryImpl(db, sugaredLogger) - ciTemplateHistoryServiceImpl := history.NewCiTemplateHistoryServiceImpl(ciTemplateHistoryRepositoryImpl, sugaredLogger) - buildPipelineSwitchServiceImpl := pipeline.NewBuildPipelineSwitchServiceImpl(sugaredLogger, ciPipelineConfigReadServiceImpl, ciPipelineRepositoryImpl, ciCdPipelineOrchestratorImpl, pipelineRepositoryImpl, ciWorkflowRepositoryImpl, appWorkflowRepositoryImpl, ciPipelineHistoryServiceImpl, ciTemplateOverrideRepositoryImpl, ciPipelineMaterialRepositoryImpl) - ciPipelineConfigServiceImpl := pipeline.NewCiPipelineConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, gitMaterialReadServiceImpl, appRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigReadServiceImpl, ciPipelineRepositoryImpl, ecrConfig, appWorkflowRepositoryImpl, ciCdConfig, attributesServiceImpl, pipelineStageServiceImpl, ciPipelineMaterialRepositoryImpl, ciTemplateServiceImpl, ciTemplateReadServiceImpl, ciTemplateOverrideRepositoryImpl, ciTemplateHistoryServiceImpl, enforcerUtilImpl, ciWorkflowRepositoryImpl, resourceGroupServiceImpl, customTagServiceImpl, cdWorkflowRepositoryImpl, buildPipelineSwitchServiceImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl) - ciMaterialConfigServiceImpl := pipeline.NewCiMaterialConfigServiceImpl(sugaredLogger, materialRepositoryImpl, ciTemplateReadServiceImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, gitMaterialHistoryServiceImpl, pipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, transactionUtilImpl, gitMaterialReadServiceImpl) deploymentGroupRepositoryImpl := repository2.NewDeploymentGroupRepositoryImpl(sugaredLogger, db) - pipelineStrategyHistoryRepositoryImpl := repository21.NewPipelineStrategyHistoryRepositoryImpl(sugaredLogger, db) + pipelineStrategyHistoryRepositoryImpl := repository22.NewPipelineStrategyHistoryRepositoryImpl(sugaredLogger, db) pipelineStrategyHistoryServiceImpl := history.NewPipelineStrategyHistoryServiceImpl(sugaredLogger, pipelineStrategyHistoryRepositoryImpl, userServiceImpl) propertiesConfigServiceImpl := pipeline.NewPropertiesConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, envConfigOverrideReadServiceImpl, deploymentConfigServiceImpl, chartServiceImpl) installedAppDBExtendedServiceImpl := FullMode.NewInstalledAppDBExtendedServiceImpl(installedAppDBServiceImpl, appStatusServiceImpl, gitOpsConfigReadServiceImpl) @@ -730,6 +726,12 @@ func InitializeApp() (*App, error) { pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, gitMaterialReadServiceImpl, chartRepositoryImpl, ciPipelineConfigServiceImpl, ciMaterialConfigServiceImpl, appArtifactManagerImpl, devtronAppCMCSServiceImpl, devtronAppStrategyServiceImpl, appDeploymentTypeChangeManagerImpl, cdPipelineConfigServiceImpl, devtronAppConfigServiceImpl) deploymentTemplateValidationServiceImpl := deploymentTemplate.NewDeploymentTemplateValidationServiceImpl(sugaredLogger, chartRefServiceImpl, scopedVariableManagerImpl) devtronAppGitOpConfigServiceImpl := gitOpsConfig.NewDevtronAppGitOpConfigServiceImpl(sugaredLogger, chartRepositoryImpl, chartServiceImpl, gitOpsConfigReadServiceImpl, gitOpsValidationServiceImpl, argoClientWrapperServiceImpl, deploymentConfigServiceImpl, chartReadServiceImpl) + ciLogServiceImpl, err := pipeline.NewCiLogServiceImpl(sugaredLogger, k8sServiceImpl) + if err != nil { + return nil, err + } + blobStorageConfigServiceImpl := pipeline.NewBlobStorageConfigServiceImpl(sugaredLogger, k8sServiceImpl, ciCdConfig) + ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, workflowServiceImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, environmentServiceImpl, workFlowStageStatusServiceImpl, k8sServiceImpl) cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, workflowServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, customTagServiceImpl, deploymentConfigServiceImpl, workFlowStageStatusServiceImpl, cdWorkflowRunnerServiceImpl) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl, chartServiceImpl, deploymentConfigServiceImpl, pipelineBuilderImpl) appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, attributesServiceImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, pipelineStageServiceImpl, ciTemplateReadServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigServiceImpl, gitOpsConfigReadServiceImpl, chartReadServiceImpl) @@ -748,7 +750,7 @@ func InitializeApp() (*App, error) { cveStoreRepositoryImpl := repository24.NewCveStoreRepositoryImpl(db, sugaredLogger) policyServiceImpl := imageScanning.NewPolicyServiceImpl(environmentServiceImpl, sugaredLogger, appRepositoryImpl, pipelineOverrideRepositoryImpl, cvePolicyRepositoryImpl, clusterServiceImplExtended, pipelineRepositoryImpl, imageScanResultRepositoryImpl, imageScanDeployInfoRepositoryImpl, imageScanObjectMetaRepositoryImpl, httpClient, ciArtifactRepositoryImpl, ciCdConfig, imageScanHistoryReadServiceImpl, cveStoreRepositoryImpl, ciTemplateRepositoryImpl, clusterReadServiceImpl, transactionUtilImpl) imageScanResultReadServiceImpl := read18.NewImageScanResultReadServiceImpl(sugaredLogger, imageScanResultRepositoryImpl) - pipelineConfigRestHandlerImpl := configure.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, deploymentTemplateValidationServiceImpl, chartServiceImpl, devtronAppGitOpConfigServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, generateManifestDeploymentTemplateServiceImpl, appWorkflowServiceImpl, gitMaterialReadServiceImpl, policyServiceImpl, imageScanResultReadServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingReadServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, ciCdPipelineOrchestratorImpl, gitProviderReadServiceImpl, teamReadServiceImpl, environmentRepositoryImpl, chartReadServiceImpl) + pipelineConfigRestHandlerImpl := configure.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, deploymentTemplateValidationServiceImpl, chartServiceImpl, devtronAppGitOpConfigServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, generateManifestDeploymentTemplateServiceImpl, appWorkflowServiceImpl, gitMaterialReadServiceImpl, policyServiceImpl, imageScanResultReadServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingReadServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, ciCdPipelineOrchestratorImpl, gitProviderReadServiceImpl, teamReadServiceImpl, environmentRepositoryImpl, chartReadServiceImpl, serviceImpl) gitOpsManifestPushServiceImpl := publish.NewGitOpsManifestPushServiceImpl(sugaredLogger, pipelineStatusTimelineServiceImpl, pipelineOverrideRepositoryImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, chartServiceImpl, gitOperationServiceImpl, argoClientWrapperServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, chartTemplateServiceImpl) manifestCreationServiceImpl := manifest.NewManifestCreationServiceImpl(sugaredLogger, dockerRegistryIpsConfigServiceImpl, chartRefServiceImpl, scopedVariableCMCSManagerImpl, k8sCommonServiceImpl, deployedAppMetricsServiceImpl, imageDigestPolicyServiceImpl, utilMergeUtil, appCrudOperationServiceImpl, deploymentTemplateServiceImpl, argoClientWrapperServiceImpl, configMapHistoryRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineOverrideRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, pipelineConfigRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, deploymentConfigServiceImpl, envConfigOverrideReadServiceImpl) configMapHistoryReadServiceImpl := read19.NewConfigMapHistoryReadService(sugaredLogger, configMapHistoryRepositoryImpl, scopedVariableCMCSManagerImpl) @@ -757,7 +759,7 @@ func InitializeApp() (*App, error) { userDeploymentRequestServiceImpl := service3.NewUserDeploymentRequestServiceImpl(sugaredLogger, userDeploymentRequestRepositoryImpl) imageScanDeployInfoReadServiceImpl := read18.NewImageScanDeployInfoReadService(sugaredLogger, imageScanDeployInfoRepositoryImpl) imageScanDeployInfoServiceImpl := imageScanning.NewImageScanDeployInfoService(sugaredLogger, imageScanDeployInfoRepositoryImpl) - manifestPushConfigRepositoryImpl := repository18.NewManifestPushConfigRepository(sugaredLogger, db) + manifestPushConfigRepositoryImpl := repository19.NewManifestPushConfigRepository(sugaredLogger, db) scanToolExecutionHistoryMappingRepositoryImpl := repository24.NewScanToolExecutionHistoryMappingRepositoryImpl(db, sugaredLogger) cdWorkflowReadServiceImpl := read20.NewCdWorkflowReadServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) imageScanServiceImpl := imageScanning.NewImageScanServiceImpl(sugaredLogger, imageScanHistoryRepositoryImpl, imageScanResultRepositoryImpl, imageScanObjectMetaRepositoryImpl, cveStoreRepositoryImpl, imageScanDeployInfoRepositoryImpl, userServiceImpl, appRepositoryImpl, environmentServiceImpl, ciArtifactRepositoryImpl, policyServiceImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, scanToolMetadataRepositoryImpl, scanToolExecutionHistoryMappingRepositoryImpl, cvePolicyRepositoryImpl, cdWorkflowReadServiceImpl) @@ -766,7 +768,7 @@ func InitializeApp() (*App, error) { return nil, err } commonArtifactServiceImpl := artifacts.NewCommonArtifactServiceImpl(sugaredLogger, ciArtifactRepositoryImpl) - workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, cdWorkflowRunnerServiceImpl, ciServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl, runnable, imageScanHistoryRepositoryImpl, imageScanServiceImpl) + workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, cdWorkflowRunnerServiceImpl, ciServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl, runnable, imageScanHistoryRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, environmentRepositoryImpl, k8sCommonServiceImpl, workflowServiceImpl, serviceImpl) externalCiRestHandlerImpl := restHandler.NewExternalCiRestHandlerImpl(sugaredLogger, validate, userServiceImpl, enforcerImpl, workflowDagExecutorImpl) pubSubClientRestHandlerImpl := restHandler.NewPubSubClientRestHandlerImpl(pubSubClientServiceImpl, sugaredLogger, ciCdConfig) webhookRouterImpl := router.NewWebhookRouterImpl(gitWebhookRestHandlerImpl, pipelineConfigRestHandlerImpl, externalCiRestHandlerImpl, pubSubClientRestHandlerImpl) @@ -937,7 +939,7 @@ func InitializeApp() (*App, error) { telemetryRouterImpl := router.NewTelemetryRouterImpl(sugaredLogger, telemetryRestHandlerImpl) bulkUpdateRepositoryImpl := bulkUpdate.NewBulkUpdateRepository(db, sugaredLogger) deployedAppServiceImpl := deployedApp.NewDeployedAppServiceImpl(sugaredLogger, k8sCommonServiceImpl, triggerServiceImpl, environmentRepositoryImpl, pipelineRepositoryImpl, cdWorkflowRepositoryImpl) - bulkUpdateServiceImpl := service7.NewBulkUpdateServiceImpl(bulkUpdateRepositoryImpl, sugaredLogger, environmentRepositoryImpl, pipelineRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineBuilderImpl, enforcerUtilImpl, ciHandlerImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, appWorkflowServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, deployedAppServiceImpl, cdPipelineEventPublishServiceImpl) + bulkUpdateServiceImpl := service7.NewBulkUpdateServiceImpl(bulkUpdateRepositoryImpl, sugaredLogger, environmentRepositoryImpl, pipelineRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineBuilderImpl, enforcerUtilImpl, ciHandlerImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, appWorkflowServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, deployedAppServiceImpl, cdPipelineEventPublishServiceImpl, serviceImpl) bulkUpdateRestHandlerImpl := restHandler.NewBulkUpdateRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, bulkUpdateServiceImpl, chartServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, environmentServiceImpl, gitRegistryConfigImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl) bulkUpdateRouterImpl := router.NewBulkUpdateRouterImpl(bulkUpdateRestHandlerImpl) webhookSecretValidatorImpl := gitWebhook.NewWebhookSecretValidatorImpl(sugaredLogger) @@ -948,18 +950,18 @@ func InitializeApp() (*App, error) { webhookListenerRouterImpl := router.NewWebhookListenerRouterImpl(webhookEventHandlerImpl) appFilteringRestHandlerImpl := appList.NewAppFilteringRestHandlerImpl(sugaredLogger, teamServiceImpl, enforcerImpl, userServiceImpl, clusterServiceImplExtended, environmentServiceImpl, teamReadServiceImpl) appFilteringRouterImpl := appList2.NewAppFilteringRouterImpl(appFilteringRestHandlerImpl) - serviceImpl := resourceTree.NewServiceImpl(sugaredLogger, appListingServiceImpl, appStatusServiceImpl, argoApplicationServiceExtendedImpl, cdApplicationStatusUpdateHandlerImpl, helmAppReadServiceImpl, helmAppServiceImpl, k8sApplicationServiceImpl, k8sCommonServiceImpl, environmentReadServiceImpl) - appListingRestHandlerImpl := appList.NewAppListingRestHandlerImpl(appListingServiceImpl, enforcerImpl, pipelineBuilderImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, userServiceImpl, k8sCommonServiceImpl, installedAppDBExtendedServiceImpl, installedAppResourceServiceImpl, pipelineRepositoryImpl, k8sApplicationServiceImpl, deploymentConfigServiceImpl, serviceImpl) + resourceTreeServiceImpl := resourceTree.NewServiceImpl(sugaredLogger, appListingServiceImpl, appStatusServiceImpl, argoApplicationServiceExtendedImpl, cdApplicationStatusUpdateHandlerImpl, helmAppReadServiceImpl, helmAppServiceImpl, k8sApplicationServiceImpl, k8sCommonServiceImpl, environmentReadServiceImpl) + appListingRestHandlerImpl := appList.NewAppListingRestHandlerImpl(appListingServiceImpl, enforcerImpl, pipelineBuilderImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, userServiceImpl, k8sCommonServiceImpl, installedAppDBExtendedServiceImpl, installedAppResourceServiceImpl, pipelineRepositoryImpl, k8sApplicationServiceImpl, deploymentConfigServiceImpl, resourceTreeServiceImpl) appListingRouterImpl := appList2.NewAppListingRouterImpl(appListingRestHandlerImpl) appInfoRestHandlerImpl := appInfo.NewAppInfoRestHandlerImpl(sugaredLogger, appCrudOperationServiceImpl, userServiceImpl, validate, enforcerUtilImpl, enforcerImpl, helmAppServiceImpl, enforcerUtilHelmImpl, genericNoteServiceImpl, commonEnforcementUtilImpl) appInfoRouterImpl := appInfo2.NewAppInfoRouterImpl(sugaredLogger, appInfoRestHandlerImpl) pipelineDeploymentConfigServiceImpl := pipeline.NewPipelineDeploymentConfigServiceImpl(sugaredLogger, chartRepositoryImpl, pipelineRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, scopedVariableCMCSManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, configMapHistoryReadServiceImpl, envConfigOverrideReadServiceImpl) - pipelineTriggerRestHandlerImpl := trigger.NewPipelineRestHandler(appServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, pipelineDeploymentConfigServiceImpl, deployedAppServiceImpl, triggerServiceImpl, workflowEventPublishServiceImpl) + pipelineTriggerRestHandlerImpl := trigger2.NewPipelineRestHandler(appServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, pipelineDeploymentConfigServiceImpl, deployedAppServiceImpl, triggerServiceImpl, workflowEventPublishServiceImpl) sseSSE := sse.NewSSE() - pipelineTriggerRouterImpl := trigger2.NewPipelineTriggerRouter(pipelineTriggerRestHandlerImpl, sseSSE) + pipelineTriggerRouterImpl := trigger3.NewPipelineTriggerRouter(pipelineTriggerRestHandlerImpl, sseSSE) webhookDataRestHandlerImpl := webhook.NewWebhookDataRestHandlerImpl(sugaredLogger, userServiceImpl, ciPipelineMaterialRepositoryImpl, enforcerUtilImpl, enforcerImpl, clientImpl, webhookEventDataConfigImpl) pipelineConfigRouterImpl := configure2.NewPipelineRouterImpl(pipelineConfigRestHandlerImpl, webhookDataRestHandlerImpl) - prePostCiScriptHistoryRepositoryImpl := repository21.NewPrePostCiScriptHistoryRepositoryImpl(sugaredLogger, db) + prePostCiScriptHistoryRepositoryImpl := repository22.NewPrePostCiScriptHistoryRepositoryImpl(sugaredLogger, db) prePostCiScriptHistoryServiceImpl := history.NewPrePostCiScriptHistoryServiceImpl(sugaredLogger, prePostCiScriptHistoryRepositoryImpl) pipelineHistoryRestHandlerImpl := history2.NewPipelineHistoryRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerImpl, pipelineStrategyHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, prePostCdScriptHistoryServiceImpl, enforcerUtilImpl, deployedConfigurationHistoryServiceImpl) pipelineHistoryRouterImpl := history3.NewPipelineHistoryRouterImpl(pipelineHistoryRestHandlerImpl) @@ -1030,7 +1032,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - ciStatusUpdateCronImpl := cron2.NewCiStatusUpdateCronImpl(sugaredLogger, appServiceImpl, ciWorkflowStatusUpdateConfig, ciPipelineRepositoryImpl, ciHandlerImpl, cronLoggerImpl) + ciStatusUpdateCronImpl := cron2.NewCiStatusUpdateCronImpl(sugaredLogger, appServiceImpl, ciWorkflowStatusUpdateConfig, ciPipelineRepositoryImpl, cronLoggerImpl, workflowDagExecutorImpl) resourceGroupRestHandlerImpl := restHandler.NewResourceGroupRestHandlerImpl(sugaredLogger, enforcerImpl, userServiceImpl, resourceGroupServiceImpl, validate) resourceGroupingRouterImpl := router.NewResourceGroupingRouterImpl(pipelineConfigRestHandlerImpl, appWorkflowRestHandlerImpl, resourceGroupRestHandlerImpl) rbacRoleServiceImpl := user.NewRbacRoleServiceImpl(sugaredLogger, rbacRoleDataRepositoryImpl) @@ -1042,7 +1044,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - ciTriggerCronImpl := cron2.NewCiTriggerCronImpl(sugaredLogger, ciTriggerCronConfig, pipelineStageRepositoryImpl, ciHandlerImpl, ciArtifactRepositoryImpl, globalPluginRepositoryImpl, cronLoggerImpl) + ciTriggerCronImpl := cron2.NewCiTriggerCronImpl(sugaredLogger, ciTriggerCronConfig, pipelineStageRepositoryImpl, ciArtifactRepositoryImpl, globalPluginRepositoryImpl, cronLoggerImpl, serviceImpl) proxyConfig, err := proxy.GetProxyConfig() if err != nil { return nil, err @@ -1078,7 +1080,7 @@ func InitializeApp() (*App, error) { cdWorkflowServiceImpl := cd.NewCdWorkflowServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) cdWorkflowRunnerReadServiceImpl := read20.NewCdWorkflowRunnerReadServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) webhookServiceImpl := pipeline.NewWebhookServiceImpl(ciArtifactRepositoryImpl, sugaredLogger, ciPipelineRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowCommonServiceImpl, workFlowStageStatusServiceImpl, ciServiceImpl) - workflowEventProcessorImpl, err := in.NewWorkflowEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowServiceImpl, cdWorkflowReadServiceImpl, cdWorkflowRunnerServiceImpl, cdWorkflowRunnerReadServiceImpl, workflowDagExecutorImpl, ciHandlerImpl, cdHandlerImpl, eventSimpleFactoryImpl, eventRESTClientImpl, triggerServiceImpl, deployedAppServiceImpl, webhookServiceImpl, validate, environmentVariables, cdWorkflowCommonServiceImpl, cdPipelineConfigServiceImpl, userDeploymentRequestServiceImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, cdWorkflowRepositoryImpl, deploymentConfigServiceImpl) + workflowEventProcessorImpl, err := in.NewWorkflowEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowServiceImpl, cdWorkflowReadServiceImpl, cdWorkflowRunnerServiceImpl, cdWorkflowRunnerReadServiceImpl, workflowDagExecutorImpl, ciHandlerImpl, cdHandlerImpl, eventSimpleFactoryImpl, eventRESTClientImpl, triggerServiceImpl, deployedAppServiceImpl, webhookServiceImpl, validate, environmentVariables, cdWorkflowCommonServiceImpl, cdPipelineConfigServiceImpl, userDeploymentRequestServiceImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, cdWorkflowRepositoryImpl, deploymentConfigServiceImpl, serviceImpl) if err != nil { return nil, err } From e9ac5319a779230c3779dd814c0a1950cf0d9fb9 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Sun, 6 Apr 2025 23:41:34 +0530 Subject: [PATCH 06/17] ci service ent oss completed --- .../CIPipelineRepository_ent.go | 8 + pkg/build/trigger/Service.go | 221 +++++++++--------- pkg/build/trigger/Service_ent.go | 82 ++++++- pkg/pipeline/types/Workflow_ent.go | 2 +- 4 files changed, 205 insertions(+), 108 deletions(-) create mode 100644 internal/sql/repository/pipelineConfig/CIPipelineRepository_ent.go diff --git a/internal/sql/repository/pipelineConfig/CIPipelineRepository_ent.go b/internal/sql/repository/pipelineConfig/CIPipelineRepository_ent.go new file mode 100644 index 0000000000..3a1538c1f8 --- /dev/null +++ b/internal/sql/repository/pipelineConfig/CIPipelineRepository_ent.go @@ -0,0 +1,8 @@ +package pipelineConfig + +import "github.com/devtron-labs/devtron/pkg/bean/common" + +func (p *CiPipeline) GetWorkflowCacheConfig() common.WorkflowCacheConfigType { + //in oss, there is no pipeline level workflow cache config, so we pass inherit to get the app level config + return common.WorkflowCacheConfigInherit +} diff --git a/pkg/build/trigger/Service.go b/pkg/build/trigger/Service.go index 48f217c40e..e35f2a3058 100644 --- a/pkg/build/trigger/Service.go +++ b/pkg/build/trigger/Service.go @@ -2,7 +2,6 @@ package trigger import ( "context" - "encoding/json" "errors" "fmt" "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" @@ -63,6 +62,8 @@ type Service interface { HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) TriggerCiPipeline(trigger types.Trigger) (int, error) + + StartCiWorkflowAndPrepareWfRequest(trigger types.Trigger) (*pipelineConfig.CiPipeline, map[string]string, *pipelineConfig.CiWorkflow, *types.WorkflowRequest, error) } type BuildxCacheFlags struct { @@ -569,15 +570,60 @@ func SetGitCommitValuesForBuildingCommitHash(ciMaterial *pipelineConfig.CiPipeli } func (impl *ServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { - impl.Logger.Debug("ci pipeline manual trigger") - ciMaterials, err := impl.GetCiMaterials(trigger.PipelineId, trigger.CiMaterials) + pipeline, variableSnapshot, savedCiWf, workflowRequest, err := impl.StartCiWorkflowAndPrepareWfRequest(trigger) if err != nil { return 0, err } + workflowRequest.CiPipelineType = trigger.PipelineType + err = impl.executeCiPipeline(workflowRequest) + if err != nil { + impl.Logger.Errorw("error in executing ci pipeline", "err", err) + dbErr := impl.markCurrentCiWorkflowFailed(savedCiWf, err) + if dbErr != nil { + impl.Logger.Errorw("update ci workflow error", "err", dbErr) + } + return 0, err + } + impl.Logger.Debugw("ci triggered", " pipeline ", trigger.PipelineId) + + var variableSnapshotHistories = sliceUtil.GetBeansPtr( + repository4.GetSnapshotBean(savedCiWf.Id, repository4.HistoryReferenceTypeCIWORKFLOW, variableSnapshot)) + if len(variableSnapshotHistories) > 0 { + err = impl.scopedVariableManager.SaveVariableHistoriesForTrigger(variableSnapshotHistories, trigger.TriggeredBy) + if err != nil { + impl.Logger.Errorf("Not able to save variable snapshot for CI trigger %s", err) + } + } + + middleware.CiTriggerCounter.WithLabelValues(pipeline.App.AppName, pipeline.Name).Inc() + go impl.ciService.WriteCITriggerEvent(trigger, pipeline, workflowRequest) + return savedCiWf.Id, err +} + +func (impl *ServiceImpl) GetCiMaterials(pipelineId int, ciMaterials []*pipelineConfig.CiPipelineMaterial) ([]*pipelineConfig.CiPipelineMaterial, error) { + if !(len(ciMaterials) == 0) { + return ciMaterials, nil + } else { + ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(pipelineId) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return nil, err + } + impl.Logger.Debug("ciMaterials for pipeline trigger ", ciMaterials) + return ciMaterials, nil + } +} + +func (impl *ServiceImpl) StartCiWorkflowAndPrepareWfRequest(trigger types.Trigger) (*pipelineConfig.CiPipeline, map[string]string, *pipelineConfig.CiWorkflow, *types.WorkflowRequest, error) { + impl.Logger.Debugw("ci pipeline manual trigger", "request", trigger) + ciMaterials, err := impl.GetCiMaterials(trigger.PipelineId, trigger.CiMaterials) + if err != nil { + return nil, nil, nil, nil, err + } ciPipelineScripts, err := impl.ciPipelineRepository.FindCiScriptsByCiPipelineId(trigger.PipelineId) if err != nil && !util.IsErrNoRows(err) { - return 0, err + return nil, nil, nil, nil, err } var pipeline *pipelineConfig.CiPipeline @@ -592,9 +638,15 @@ func (impl *ServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { ciWorkflowConfigNamespace := impl.config.GetDefaultNamespace() envModal, isJob, err := impl.getEnvironmentForJob(pipeline, trigger) if err != nil { - return 0, err + return nil, nil, nil, nil, err } if isJob && envModal != nil { + + err = impl.checkArgoSetupRequirement(envModal) + if err != nil { + return nil, nil, nil, nil, err + } + ciWorkflowConfigNamespace = envModal.Namespace // This will be populated for jobs running in selected environment @@ -613,15 +665,14 @@ func (impl *ServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { AppName: pipeline.App.AppName, } } - - savedCiWf, err := impl.saveNewWorkflow(pipeline, ciWorkflowConfigNamespace, trigger.CommitHashes, trigger.TriggeredBy, trigger.EnvironmentId, isJob, trigger.ReferenceCiWorkflowId) + savedCiWf, err := impl.saveNewWorkflowForCITrigger(pipeline, ciWorkflowConfigNamespace, trigger.CommitHashes, trigger.TriggeredBy, ciMaterials, trigger.EnvironmentId, isJob, trigger.ReferenceCiWorkflowId) if err != nil { impl.Logger.Errorw("could not save new workflow", "err", err) - return 0, err + return nil, nil, nil, nil, err } - // preCiSteps, postCiSteps, refPluginsData, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(pipeline.Id, ciEvent) request := pipelineConfigBean.NewBuildPrePostStepDataReq(pipeline.Id, pipelineConfigBean.CiStage, scope) + request = updateBuildPrePostStepDataReq(request, trigger) prePostAndRefPluginResponse, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(request) if err != nil { impl.Logger.Errorw("error in getting pre steps data for wf request", "err", err, "ciPipelineId", pipeline.Id) @@ -629,7 +680,7 @@ func (impl *ServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { if dbErr != nil { impl.Logger.Errorw("saving workflow error", "err", dbErr) } - return 0, err + return nil, nil, nil, nil, err } preCiSteps := prePostAndRefPluginResponse.PreStageSteps postCiSteps := prePostAndRefPluginResponse.PostStageSteps @@ -639,15 +690,14 @@ func (impl *ServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { if len(preCiSteps) == 0 && isJob { errMsg := fmt.Sprintf("No tasks are configured in this job pipeline") validationErr := util.NewApiError(http.StatusNotFound, errMsg, errMsg) - - return 0, validationErr + return nil, nil, nil, nil, validationErr } // get env variables of git trigger data and add it in the extraEnvVariables gitTriggerEnvVariables, _, err := impl.ciCdPipelineOrchestrator.GetGitCommitEnvVarDataForCICDStage(savedCiWf.GitTriggers) if err != nil { impl.Logger.Errorw("error in getting gitTrigger env data for stage", "gitTriggers", savedCiWf.GitTriggers, "err", err) - return 0, err + return nil, nil, nil, nil, err } for k, v := range gitTriggerEnvVariables { @@ -657,7 +707,7 @@ func (impl *ServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { workflowRequest, err := impl.buildWfRequestForCiPipeline(pipeline, trigger, ciMaterials, savedCiWf, ciWorkflowConfigNamespace, ciPipelineScripts, preCiSteps, postCiSteps, refPluginsData, isJob) if err != nil { impl.Logger.Errorw("make workflow req", "err", err) - return 0, err + return nil, nil, nil, nil, err } err = impl.handleRuntimeParamsValidations(trigger, ciMaterials, workflowRequest) if err != nil { @@ -667,102 +717,48 @@ func (impl *ServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { if err1 != nil { impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag") } - return 0, err + return nil, nil, nil, nil, err } workflowRequest.Scope = scope workflowRequest.AppId = pipeline.AppId - workflowRequest.BuildxCacheModeMin = impl.buildxCacheFlags.BuildxCacheModeMin - workflowRequest.AsyncBuildxCacheExport = impl.buildxCacheFlags.AsyncBuildxCacheExport - if impl.config != nil && impl.config.BuildxK8sDriverOptions != "" { - err = impl.setBuildxK8sDriverData(workflowRequest) - if err != nil { - impl.Logger.Errorw("error in setBuildxK8sDriverData", "BUILDX_K8S_DRIVER_OPTIONS", impl.config.BuildxK8sDriverOptions, "err", err) - return 0, err - } - } - - // savedCiWf.LogLocation = impl.ciCdConfig.CiDefaultBuildLogsKeyPrefix + "/" + workflowRequest.WorkflowNamePrefix + "/main.log" - savedCiWf.LogLocation = fmt.Sprintf("%s/%s/main.log", impl.config.GetDefaultBuildLogsKeyPrefix(), workflowRequest.WorkflowNamePrefix) - err = impl.updateCiWorkflow(workflowRequest, savedCiWf) - - appLabels, err := impl.appCrudOperationService.GetLabelsByAppId(pipeline.AppId) - if err != nil { - return 0, err - } - workflowRequest.AppLabels = appLabels workflowRequest.Env = envModal if isJob { workflowRequest.Type = pipelineConfigBean.JOB_WORKFLOW_PIPELINE_TYPE } else { workflowRequest.Type = pipelineConfigBean.CI_WORKFLOW_PIPELINE_TYPE } - - workflowRequest.CiPipelineType = trigger.PipelineType - err = impl.executeCiPipeline(workflowRequest) + workflowRequest, err = impl.updateWorkflowRequestWithBuildCacheData(workflowRequest, scope) if err != nil { - impl.Logger.Errorw("error in executing ci pipeline", "err", err) - dbErr := impl.markCurrentCiWorkflowFailed(savedCiWf, err) - if dbErr != nil { - impl.Logger.Errorw("update ci workflow error", "err", dbErr) - } - return 0, err + impl.Logger.Errorw("error, updateWorkflowRequestWithBuildCacheData", "workflowRequest", workflowRequest, "err", err) + return nil, nil, nil, nil, err } - impl.Logger.Debugw("ci triggered", " pipeline ", trigger.PipelineId) - - var variableSnapshotHistories = sliceUtil.GetBeansPtr( - repository4.GetSnapshotBean(savedCiWf.Id, repository4.HistoryReferenceTypeCIWORKFLOW, variableSnapshot)) - if len(variableSnapshotHistories) > 0 { - err = impl.scopedVariableManager.SaveVariableHistoriesForTrigger(variableSnapshotHistories, trigger.TriggeredBy) + if impl.canSetK8sDriverData(workflowRequest) { + err = impl.setBuildxK8sDriverData(workflowRequest) if err != nil { - impl.Logger.Errorf("Not able to save variable snapshot for CI trigger %s", err) + impl.Logger.Errorw("error in setBuildxK8sDriverData", "BUILDX_K8S_DRIVER_OPTIONS", impl.config.BuildxK8sDriverOptions, "err", err) + return nil, nil, nil, nil, err } } - - middleware.CiTriggerCounter.WithLabelValues(pipeline.App.AppName, pipeline.Name).Inc() - go impl.ciService.WriteCITriggerEvent(trigger, pipeline, workflowRequest) - return savedCiWf.Id, err -} - -func (impl *ServiceImpl) GetCiMaterials(pipelineId int, ciMaterials []*pipelineConfig.CiPipelineMaterial) ([]*pipelineConfig.CiPipelineMaterial, error) { - if !(len(ciMaterials) == 0) { - return ciMaterials, nil - } else { - ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(pipelineId) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return nil, err - } - impl.Logger.Debug("ciMaterials for pipeline trigger ", ciMaterials) - return ciMaterials, nil + savedCiWf.LogLocation = fmt.Sprintf("%s/%s/main.log", impl.config.GetDefaultBuildLogsKeyPrefix(), workflowRequest.WorkflowNamePrefix) + err = impl.updateCiWorkflow(workflowRequest, savedCiWf) + appLabels, err := impl.appCrudOperationService.GetLabelsByAppId(pipeline.AppId) + if err != nil { + return nil, nil, nil, nil, err } + workflowRequest.AppLabels = appLabels + workflowRequest = impl.updateWorkflowRequestWithEntSupportData(workflowRequest) + return pipeline, variableSnapshot, savedCiWf, workflowRequest, nil } func (impl *ServiceImpl) setBuildxK8sDriverData(workflowRequest *types.WorkflowRequest) error { - ciBuildConfig := workflowRequest.CiBuildConfig - if ciBuildConfig != nil { - if dockerBuildConfig := ciBuildConfig.DockerBuildConfig; dockerBuildConfig != nil { - k8sDriverOptions, err := impl.getK8sDriverOptions() - if err != nil { - errMsg := "error in parsing BUILDX_K8S_DRIVER_OPTIONS from the devtron-cm, " - err = errors.New(errMsg + "error : " + err.Error()) - impl.Logger.Errorw(errMsg, "err", err) - } - dockerBuildConfig.BuildxK8sDriverOptions = k8sDriverOptions - - } - } - return nil -} - -func (impl *ServiceImpl) getK8sDriverOptions() ([]map[string]string, error) { - buildxK8sDriverOptions := make([]map[string]string, 0) - err := json.Unmarshal([]byte(impl.config.BuildxK8sDriverOptions), &buildxK8sDriverOptions) + dockerBuildConfig := workflowRequest.CiBuildConfig.DockerBuildConfig + k8sDriverOptions, err := impl.getK8sDriverOptions(workflowRequest, dockerBuildConfig.TargetPlatform) if err != nil { - return nil, err - } else { - return buildxK8sDriverOptions, nil + impl.Logger.Errorw("error in parsing BUILDX_K8S_DRIVER_OPTIONS from the devtron-cm", "err", err) } + dockerBuildConfig.BuildxK8sDriverOptions = k8sDriverOptions + return nil } func (impl *ServiceImpl) getEnvironmentForJob(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger) (*repository6.Environment, bool, error) { @@ -796,9 +792,14 @@ func (impl *ServiceImpl) BuildPayload(trigger types.Trigger, pipeline *pipelineC return payload } -func (impl *ServiceImpl) saveNewWorkflow(pipeline *pipelineConfig.CiPipeline, ciWorkflowConfigNamespace string, - commitHashes map[int]pipelineConfig.GitCommit, userId int32, EnvironmentId int, isJob bool, refCiWorkflowId int) (wf *pipelineConfig.CiWorkflow, error error) { +func (impl *ServiceImpl) saveNewWorkflowForCITrigger(pipeline *pipelineConfig.CiPipeline, ciWorkflowConfigNamespace string, + commitHashes map[int]pipelineConfig.GitCommit, userId int32, ciMaterials []*pipelineConfig.CiPipelineMaterial, EnvironmentId int, isJob bool, refCiWorkflowId int) (*pipelineConfig.CiWorkflow, error) { + isCiTriggerBlocked, err := impl.checkIfCITriggerIsBlocked(pipeline, ciMaterials, isJob) + if err != nil { + impl.Logger.Errorw("error, checkIfCITriggerIsBlocked", "pipelineId", pipeline.Id, "err", err) + return &pipelineConfig.CiWorkflow{}, err + } ciWorkflow := &pipelineConfig.CiWorkflow{ Name: pipeline.Name + "-" + strconv.Itoa(pipeline.Id), Status: cdWorkflow.WorkflowStarting, // starting CIStage @@ -817,7 +818,10 @@ func (impl *ServiceImpl) saveNewWorkflow(pipeline *pipelineConfig.CiPipeline, ci ciWorkflow.Namespace = ciWorkflowConfigNamespace ciWorkflow.EnvironmentId = EnvironmentId } - err := impl.ciService.SaveCiWorkflowWithStage(ciWorkflow) + if isCiTriggerBlocked { + return impl.handleWFIfCITriggerIsBlocked(ciWorkflow) + } + err = impl.ciService.SaveCiWorkflowWithStage(ciWorkflow) if err != nil { impl.Logger.Errorw("saving workflow error", "err", err) return &pipelineConfig.CiWorkflow{}, err @@ -882,7 +886,12 @@ func (impl *ServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.Ci CaCert: ciMaterial.GitMaterial.GitProvider.CaCert, }, } - + var err error + ciProjectDetail, err = impl.updateCIProjectDetailWithCloningMode(pipeline.AppId, ciMaterial, ciProjectDetail) + if err != nil { + impl.Logger.Errorw("error, updateCIProjectDetailWithCloningMode", "pipelineId", pipeline.Id, "err", err) + return nil, err + } if ciMaterial.Type == constants.SOURCE_TYPE_WEBHOOK { webhookData := commitHashForPipelineId.WebhookData ciProjectDetail.WebhookData = pipelineConfig.WebhookData{ @@ -1072,15 +1081,8 @@ func (impl *ServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.Ci checkoutPath = filepath.Join(checkoutPath, buildPackConfig.ProjectPath) } - defaultTargetPlatform := impl.config.DefaultTargetPlatform - useBuildx := impl.config.UseBuildx - if ciBuildConfigBean.DockerBuildConfig != nil { - if ciBuildConfigBean.DockerBuildConfig.TargetPlatform == "" && useBuildx { - ciBuildConfigBean.DockerBuildConfig.TargetPlatform = defaultTargetPlatform - ciBuildConfigBean.DockerBuildConfig.UseBuildx = useBuildx - } - ciBuildConfigBean.DockerBuildConfig.BuildxProvenanceMode = impl.config.BuildxProvenanceMode + ciBuildConfigBean = impl.updateCIBuildConfig(ciBuildConfigBean) } workflowRequest := &types.WorkflowRequest{ @@ -1117,16 +1119,15 @@ func (impl *ServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.Ci WorkflowExecutor: impl.config.GetWorkflowExecutorType(), Type: pipelineConfigBean.CI_WORKFLOW_PIPELINE_TYPE, CiArtifactLastFetch: trigger.CiArtifactLastFetch, - RegistryDestinationImageMap: registryDestinationImageMap, RegistryCredentialMap: registryCredentialMap, PluginArtifactStage: pluginArtifactStage, ImageScanMaxRetries: impl.config.ImageScanMaxRetries, ImageScanRetryDelay: impl.config.ImageScanRetryDelay, UseDockerApiToGetDigest: impl.config.UseDockerApiToGetDigest, + RegistryDestinationImageMap: registryDestinationImageMap, } - workflowRequest.SetAwsInspectorConfig("") - //in oss, there is no pipeline level workflow cache config, so we pass inherit to get the app level config - workflowCacheConfig := impl.ciCdPipelineOrchestrator.GetWorkflowCacheConfig(pipeline.App.AppType, trigger.PipelineType, common.WorkflowCacheConfigInherit) + workflowRequest.SetEntOnlyFields(trigger, impl.config) + workflowCacheConfig := impl.ciCdPipelineOrchestrator.GetWorkflowCacheConfig(pipeline.App.AppType, trigger.PipelineType, pipeline.GetWorkflowCacheConfig()) workflowRequest.IgnoreDockerCachePush = !workflowCacheConfig.Value workflowRequest.IgnoreDockerCachePull = !workflowCacheConfig.Value impl.Logger.Debugw("Ignore Cache values", "IgnoreDockerCachePush", workflowRequest.IgnoreDockerCachePush, "IgnoreDockerCachePull", workflowRequest.IgnoreDockerCachePull) @@ -1151,6 +1152,11 @@ func (impl *ServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.Ci } } if dockerRegistry != nil { + workflowRequest, err = impl.updateWorkflowRequestWithRemoteConnConf(dockerRegistry, workflowRequest) + if err != nil { + impl.Logger.Errorw("error occurred updating workflow request", "dockerRegistryId", dockerRegistry.Id, "err", err) + return nil, err + } workflowRequest.DockerRegistryId = dockerRegistry.Id workflowRequest.DockerRegistryType = string(dockerRegistry.RegistryType) workflowRequest.DockerImageTag = dockerImageTag @@ -1457,6 +1463,11 @@ func (impl *ServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, c impl.Logger.Errorw("ci artifact already exists with same image name", "artifact", externalCiArtifact) return fmt.Errorf("ci artifact already exists with same image name") } + workflowRequest, err = impl.updateWorkflowRequestForDigestPull(trigger.PipelineId, workflowRequest) + if err != nil { + impl.Logger.Errorw("error in updating workflow request", "err", err) + return err + } } else { artifactExists, err = impl.ciArtifactRepository.IfArtifactExistByImageDigest(imageDigest, externalCiArtifact, trigger.PipelineId) if err != nil { diff --git a/pkg/build/trigger/Service_ent.go b/pkg/build/trigger/Service_ent.go index cd22daa921..0844a5986b 100644 --- a/pkg/build/trigger/Service_ent.go +++ b/pkg/build/trigger/Service_ent.go @@ -1,12 +1,20 @@ package trigger import ( + "encoding/json" "github.com/devtron-labs/common-lib/imageScan/bean" + repository3 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/util" bean3 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/bean/common" - bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + bean2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" + pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" + "github.com/devtron-labs/devtron/pkg/resourceQualifiers" + "net/http" ) func (impl *ServiceImpl) updateRuntimeParamsForAutoCI(ciPipelineId int, runtimeParameters *common.RuntimeParameters) (*common.RuntimeParameters, error) { @@ -21,6 +29,76 @@ func (impl *ServiceImpl) fetchImageScanExecutionMedium() (*repository.ScanToolMe return &repository.ScanToolMetadata{}, "", nil } -func (impl *ServiceImpl) fetchImageScanExecutionStepsForWfRequest(scanToolMetadata *repository.ScanToolMetadata) ([]*types.ImageScanningSteps, []*bean2.RefPluginObject, error) { +func (impl *ServiceImpl) fetchImageScanExecutionStepsForWfRequest(scanToolMetadata *repository.ScanToolMetadata) ([]*types.ImageScanningSteps, []*pipelineConfigBean.RefPluginObject, error) { return nil, nil, nil } + +func (impl *ServiceImpl) checkIfCITriggerIsBlocked(pipeline *pipelineConfig.CiPipeline, + ciMaterials []*pipelineConfig.CiPipelineMaterial, isJob bool) (bool, error) { + return false, nil +} + +func (impl *ServiceImpl) handleWFIfCITriggerIsBlocked(ciWorkflow *pipelineConfig.CiWorkflow) (*pipelineConfig.CiWorkflow, error) { + impl.Logger.Errorw("cannot trigger pipeline, blocked by mandatory plugin policy", "ciPipelineId", ciWorkflow.CiPipelineId) + return &pipelineConfig.CiWorkflow{}, util.GetApiErrorAdapter(http.StatusInternalServerError, "500", "Invalid flow access, corrupt data possibility", "Invalid flow access, corrupt data possibility") +} + +func (impl *ServiceImpl) checkArgoSetupRequirement(envModal *repository2.Environment) error { + return nil +} + +func (impl *ServiceImpl) updateWorkflowRequestForDigestPull(pipelineId int, workflowRequest *types.WorkflowRequest) (*types.WorkflowRequest, error) { + return workflowRequest, nil +} + +func (impl *ServiceImpl) updateCIProjectDetailWithCloningMode(appId int, ciMaterial *pipelineConfig.CiPipelineMaterial, + ciProjectDetail pipelineConfigBean.CiProjectDetails) (pipelineConfigBean.CiProjectDetails, error) { + return ciProjectDetail, nil +} + +func (impl *ServiceImpl) updateWorkflowRequestWithRemoteConnConf(dockerRegistry *repository3.DockerArtifactStore, + workflowRequest *types.WorkflowRequest) (*types.WorkflowRequest, error) { + return workflowRequest, nil +} + +func (impl *ServiceImpl) updateWorkflowRequestWithEntSupportData(workflowRequest *types.WorkflowRequest) *types.WorkflowRequest { + return workflowRequest +} + +func (impl *ServiceImpl) updateWorkflowRequestWithBuildCacheData(workflowRequest *types.WorkflowRequest, + scope resourceQualifiers.Scope) (*types.WorkflowRequest, error) { + workflowRequest.BuildxCacheModeMin = impl.buildxCacheFlags.BuildxCacheModeMin + workflowRequest.AsyncBuildxCacheExport = impl.buildxCacheFlags.AsyncBuildxCacheExport + return workflowRequest, nil +} + +func (impl *ServiceImpl) canSetK8sDriverData(workflowRequest *types.WorkflowRequest) bool { + return impl.config != nil && impl.config.BuildxK8sDriverOptions != "" && workflowRequest.CiBuildConfig != nil && + workflowRequest.CiBuildConfig.DockerBuildConfig != nil +} + +func (impl *ServiceImpl) getK8sDriverOptions(workflowRequest *types.WorkflowRequest, targetPlatforms string) ([]map[string]string, error) { + buildxK8sDriverOptions := make([]map[string]string, 0) + err := json.Unmarshal([]byte(impl.config.BuildxK8sDriverOptions), &buildxK8sDriverOptions) + if err != nil { + return nil, err + } + return buildxK8sDriverOptions, nil +} + +func (impl *ServiceImpl) updateCIBuildConfig(ciBuildConfigBean *bean2.CiBuildConfigBean) *bean2.CiBuildConfigBean { + defaultTargetPlatform := impl.config.DefaultTargetPlatform + useBuildx := impl.config.UseBuildx + if ciBuildConfigBean.DockerBuildConfig != nil { + if ciBuildConfigBean.DockerBuildConfig.TargetPlatform == "" && useBuildx { + ciBuildConfigBean.DockerBuildConfig.TargetPlatform = defaultTargetPlatform + ciBuildConfigBean.DockerBuildConfig.UseBuildx = useBuildx + } + ciBuildConfigBean.DockerBuildConfig.BuildxProvenanceMode = impl.config.BuildxProvenanceMode + } + return ciBuildConfigBean +} + +func updateBuildPrePostStepDataReq(req *pipelineConfigBean.BuildPrePostStepDataRequest, trigger types.Trigger) *pipelineConfigBean.BuildPrePostStepDataRequest { + return req +} diff --git a/pkg/pipeline/types/Workflow_ent.go b/pkg/pipeline/types/Workflow_ent.go index 2a5c03e78f..ecf78b9e1a 100644 --- a/pkg/pipeline/types/Workflow_ent.go +++ b/pkg/pipeline/types/Workflow_ent.go @@ -30,6 +30,6 @@ func (workflowRequest *WorkflowRequest) SetImageScanningSteps(imageScanningSteps return } -func (workflowRequest *WorkflowRequest) SetAwsInspectorConfig(awsInspectorConfig string) { +func (workflowRequest *WorkflowRequest) SetEntOnlyFields(trigger Trigger, ciConfig *CiConfig) { return } From f6391d776bc2f88aaaa7cb3689f14e1d32b0cf2c Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Mon, 7 Apr 2025 00:02:56 +0530 Subject: [PATCH 07/17] added app to subdirectory for categorising entity --- .../ArgoApplicationRestHandler.go | 4 ++-- api/argoApplication/wire_argoApplication.go | 16 ++++++++-------- api/helm-app/HelmAppRestHandler.go | 4 ++-- api/k8s/application/k8sApplicationRestHandler.go | 4 ++-- cmd/external-app/wire_gen.go | 6 +++--- pkg/app/AppService.go | 2 +- .../InstalledAppDeploymentTypeChangeService.go | 4 ++-- .../FullMode/resource/ResourceTreeService.go | 4 ++-- .../status/resourceTree/ResourceTreeService.go | 4 ++-- .../trigger/devtronApps/TriggerService_ent1.go | 2 +- pkg/k8s/K8sCommonService.go | 2 +- pkg/k8s/application/k8sApplicationService.go | 2 +- .../argoApplication/ArgoApplicationService.go | 6 +++--- .../ArgoApplicationServiceExtended.go | 4 ++-- pkg/{ => nucleus}/argoApplication/bean/bean.go | 0 .../helper/deploymentStatusHelper.go | 0 .../argoApplication/helper/helper.go | 2 +- .../read/ArgoApplicationReadService.go | 4 ++-- .../read/config/ArgoApplicationConfigService.go | 4 ++-- pkg/terminal/terminalSesion.go | 2 +- wire_gen.go | 6 +++--- 21 files changed, 41 insertions(+), 41 deletions(-) rename pkg/{ => nucleus}/argoApplication/ArgoApplicationService.go (97%) rename pkg/{ => nucleus}/argoApplication/ArgoApplicationServiceExtended.go (99%) rename pkg/{ => nucleus}/argoApplication/bean/bean.go (100%) rename pkg/{ => nucleus}/argoApplication/helper/deploymentStatusHelper.go (100%) rename pkg/{ => nucleus}/argoApplication/helper/helper.go (98%) rename pkg/{ => nucleus}/argoApplication/read/ArgoApplicationReadService.go (98%) rename pkg/{ => nucleus}/argoApplication/read/config/ArgoApplicationConfigService.go (97%) diff --git a/api/argoApplication/ArgoApplicationRestHandler.go b/api/argoApplication/ArgoApplicationRestHandler.go index edcbabdca9..27d9a8632e 100644 --- a/api/argoApplication/ArgoApplicationRestHandler.go +++ b/api/argoApplication/ArgoApplicationRestHandler.go @@ -20,9 +20,9 @@ import ( "context" "errors" "github.com/devtron-labs/devtron/api/restHandler/common" - "github.com/devtron-labs/devtron/pkg/argoApplication" - "github.com/devtron-labs/devtron/pkg/argoApplication/read" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read" "go.uber.org/zap" "net/http" "strconv" diff --git a/api/argoApplication/wire_argoApplication.go b/api/argoApplication/wire_argoApplication.go index e3c7381725..6373e274f8 100644 --- a/api/argoApplication/wire_argoApplication.go +++ b/api/argoApplication/wire_argoApplication.go @@ -17,9 +17,9 @@ package argoApplication import ( - "github.com/devtron-labs/devtron/pkg/argoApplication" - "github.com/devtron-labs/devtron/pkg/argoApplication/read" - "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" + argoApplication2 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read/config" "github.com/google/wire" ) @@ -30,9 +30,9 @@ var ArgoApplicationWireSetFull = wire.NewSet( config.NewArgoApplicationConfigServiceImpl, wire.Bind(new(config.ArgoApplicationConfigService), new(*config.ArgoApplicationConfigServiceImpl)), - argoApplication.NewArgoApplicationServiceImpl, - argoApplication.NewArgoApplicationServiceExtendedServiceImpl, - wire.Bind(new(argoApplication.ArgoApplicationService), new(*argoApplication.ArgoApplicationServiceExtendedImpl)), + argoApplication2.NewArgoApplicationServiceImpl, + argoApplication2.NewArgoApplicationServiceExtendedServiceImpl, + wire.Bind(new(argoApplication2.ArgoApplicationService), new(*argoApplication2.ArgoApplicationServiceExtendedImpl)), NewArgoApplicationRestHandlerImpl, wire.Bind(new(ArgoApplicationRestHandler), new(*ArgoApplicationRestHandlerImpl)), @@ -48,8 +48,8 @@ var ArgoApplicationWireSetEA = wire.NewSet( config.NewArgoApplicationConfigServiceImpl, wire.Bind(new(config.ArgoApplicationConfigService), new(*config.ArgoApplicationConfigServiceImpl)), - argoApplication.NewArgoApplicationServiceImpl, - wire.Bind(new(argoApplication.ArgoApplicationService), new(*argoApplication.ArgoApplicationServiceImpl)), + argoApplication2.NewArgoApplicationServiceImpl, + wire.Bind(new(argoApplication2.ArgoApplicationService), new(*argoApplication2.ArgoApplicationServiceImpl)), NewArgoApplicationRestHandlerImpl, wire.Bind(new(ArgoApplicationRestHandler), new(*ArgoApplicationRestHandlerImpl)), diff --git a/api/helm-app/HelmAppRestHandler.go b/api/helm-app/HelmAppRestHandler.go index a21cf730ff..feab53cc95 100644 --- a/api/helm-app/HelmAppRestHandler.go +++ b/api/helm-app/HelmAppRestHandler.go @@ -24,12 +24,12 @@ import ( service2 "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/EAMode" - "github.com/devtron-labs/devtron/pkg/argoApplication" - "github.com/devtron-labs/devtron/pkg/argoApplication/helper" clusterBean "github.com/devtron-labs/devtron/pkg/cluster/bean" clientErrors "github.com/devtron-labs/devtron/pkg/errors" "github.com/devtron-labs/devtron/pkg/fluxApplication" bean2 "github.com/devtron-labs/devtron/pkg/k8s/application/bean" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/pipeline" "net/http" "strconv" diff --git a/api/k8s/application/k8sApplicationRestHandler.go b/api/k8s/application/k8sApplicationRestHandler.go index dd68e18e15..55d42ed15c 100644 --- a/api/k8s/application/k8sApplicationRestHandler.go +++ b/api/k8s/application/k8sApplicationRestHandler.go @@ -33,8 +33,6 @@ import ( client "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/api/restHandler/common" util2 "github.com/devtron-labs/devtron/internal/util" - "github.com/devtron-labs/devtron/pkg/argoApplication/helper" - "github.com/devtron-labs/devtron/pkg/argoApplication/read" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/auth/user" bean4 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" @@ -44,6 +42,8 @@ import ( application2 "github.com/devtron-labs/devtron/pkg/k8s/application" bean2 "github.com/devtron-labs/devtron/pkg/k8s/application/bean" bean3 "github.com/devtron-labs/devtron/pkg/k8s/bean" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read" "github.com/devtron-labs/devtron/pkg/terminal" "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/rbac" diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index e827637c0a..7e1c13fc04 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -75,9 +75,6 @@ import ( "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/common" "github.com/devtron-labs/devtron/pkg/appStore/values/repository" service4 "github.com/devtron-labs/devtron/pkg/appStore/values/service" - "github.com/devtron-labs/devtron/pkg/argoApplication" - read9 "github.com/devtron-labs/devtron/pkg/argoApplication/read" - config3 "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/attributes" "github.com/devtron-labs/devtron/pkg/auth/authentication" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" @@ -118,6 +115,9 @@ import ( read7 "github.com/devtron-labs/devtron/pkg/module/read" "github.com/devtron-labs/devtron/pkg/module/repo" "github.com/devtron-labs/devtron/pkg/module/store" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" + read9 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read" + config3 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool" repository11 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 5bb33db0c8..5d44b6f4a4 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -29,7 +29,6 @@ import ( internalUtil "github.com/devtron-labs/devtron/internal/util" bean3 "github.com/devtron-labs/devtron/pkg/app/bean" installedAppReader "github.com/devtron-labs/devtron/pkg/appStore/installedApp/read" - "github.com/devtron-labs/devtron/pkg/argoApplication/helper" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" bean2 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" @@ -39,6 +38,7 @@ import ( bean6 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/bean" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/read" bean4 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/workflow/cd" "net/url" "strconv" diff --git a/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go b/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go index 2d713500d6..1895a3975b 100644 --- a/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go +++ b/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go @@ -37,8 +37,6 @@ import ( deployment2 "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/EAMode/deployment" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/FullMode/deployment" util2 "github.com/devtron-labs/devtron/pkg/appStore/util" - "github.com/devtron-labs/devtron/pkg/argoApplication" - bean4 "github.com/devtron-labs/devtron/pkg/argoApplication/bean" "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/cluster" "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" @@ -49,6 +47,8 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" bean2 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/k8s" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" + bean4 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" util3 "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" "go.uber.org/zap" diff --git a/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go b/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go index 5a4cd57313..082cb6856e 100644 --- a/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go +++ b/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go @@ -37,12 +37,12 @@ import ( "github.com/devtron-labs/devtron/pkg/appStore/bean" appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/repository" - "github.com/devtron-labs/devtron/pkg/argoApplication" - bean3 "github.com/devtron-labs/devtron/pkg/argoApplication/bean" "github.com/devtron-labs/devtron/pkg/deployment/common" bean2 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/k8s" application3 "github.com/devtron-labs/devtron/pkg/k8s/application" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" + bean3 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" util3 "github.com/devtron-labs/devtron/pkg/util" util2 "github.com/devtron-labs/devtron/util" "github.com/tidwall/gjson" diff --git a/pkg/deployment/deployedApp/status/resourceTree/ResourceTreeService.go b/pkg/deployment/deployedApp/status/resourceTree/ResourceTreeService.go index b6fd6b5f69..4f5ea83ca5 100644 --- a/pkg/deployment/deployedApp/status/resourceTree/ResourceTreeService.go +++ b/pkg/deployment/deployedApp/status/resourceTree/ResourceTreeService.go @@ -34,12 +34,12 @@ import ( "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" "github.com/devtron-labs/devtron/pkg/appStatus" - argoApplication2 "github.com/devtron-labs/devtron/pkg/argoApplication" - bean2 "github.com/devtron-labs/devtron/pkg/argoApplication/bean" read2 "github.com/devtron-labs/devtron/pkg/cluster/environment/read" commonBean "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/k8s" application2 "github.com/devtron-labs/devtron/pkg/k8s/application" + argoApplication2 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" + bean2 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" util2 "github.com/devtron-labs/devtron/util" "go.uber.org/zap" "strconv" diff --git a/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go b/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go index f008f72adc..6c54cf9c57 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go @@ -8,11 +8,11 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/app" bean4 "github.com/devtron-labs/devtron/pkg/app/bean" - "github.com/devtron-labs/devtron/pkg/argoApplication/helper" bean2 "github.com/devtron-labs/devtron/pkg/bean" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/deployment/manifest/publish" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/pipeline/repository" "helm.sh/helm/v3/pkg/chart" ) diff --git a/pkg/k8s/K8sCommonService.go b/pkg/k8s/K8sCommonService.go index 0c3a2a9401..d448dd2311 100644 --- a/pkg/k8s/K8sCommonService.go +++ b/pkg/k8s/K8sCommonService.go @@ -25,12 +25,12 @@ import ( "github.com/devtron-labs/devtron/api/bean/AppView" helmBean "github.com/devtron-labs/devtron/api/helm-app/service/bean" internalUtil "github.com/devtron-labs/devtron/internal/util" - "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" bean2 "github.com/devtron-labs/devtron/pkg/cluster/bean" bean4 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" "github.com/devtron-labs/devtron/pkg/cluster/read" bean3 "github.com/devtron-labs/devtron/pkg/k8s/application/bean" bean5 "github.com/devtron-labs/devtron/pkg/k8s/bean" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read/config" "github.com/devtron-labs/devtron/util" "go.opentelemetry.io/otel" "go.uber.org/zap" diff --git a/pkg/k8s/application/k8sApplicationService.go b/pkg/k8s/application/k8sApplicationService.go index 39588ef15e..7ab5a6b6b7 100644 --- a/pkg/k8s/application/k8sApplicationService.go +++ b/pkg/k8s/application/k8sApplicationService.go @@ -25,7 +25,6 @@ import ( "github.com/devtron-labs/devtron/api/helm-app/gRPC" client "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/api/helm-app/service/bean" - "github.com/devtron-labs/devtron/pkg/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" bean5 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" "github.com/devtron-labs/devtron/pkg/cluster/read" @@ -33,6 +32,7 @@ import ( "github.com/devtron-labs/devtron/pkg/fluxApplication" bean2 "github.com/devtron-labs/devtron/pkg/fluxApplication/bean" bean4 "github.com/devtron-labs/devtron/pkg/k8s/bean" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" "io" v1 "k8s.io/client-go/kubernetes/typed/core/v1" "net/http" diff --git a/pkg/argoApplication/ArgoApplicationService.go b/pkg/nucleus/argoApplication/ArgoApplicationService.go similarity index 97% rename from pkg/argoApplication/ArgoApplicationService.go rename to pkg/nucleus/argoApplication/ArgoApplicationService.go index 48cd31c56d..4bd4f893bc 100644 --- a/pkg/argoApplication/ArgoApplicationService.go +++ b/pkg/nucleus/argoApplication/ArgoApplicationService.go @@ -25,15 +25,15 @@ import ( "github.com/devtron-labs/devtron/api/helm-app/service" argoApplication "github.com/devtron-labs/devtron/client/argocdServer/bean" util2 "github.com/devtron-labs/devtron/internal/util" - "github.com/devtron-labs/devtron/pkg/argoApplication/bean" - "github.com/devtron-labs/devtron/pkg/argoApplication/helper" - "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/cluster/adapter" clusterRepository "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/deployment/common" commonBean "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/k8s/application" k8s2 "github.com/devtron-labs/devtron/pkg/k8s/bean" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read/config" "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/sliceUtil" "go.uber.org/zap" diff --git a/pkg/argoApplication/ArgoApplicationServiceExtended.go b/pkg/nucleus/argoApplication/ArgoApplicationServiceExtended.go similarity index 99% rename from pkg/argoApplication/ArgoApplicationServiceExtended.go rename to pkg/nucleus/argoApplication/ArgoApplicationServiceExtended.go index 0cfb81b2d3..2643bbe432 100644 --- a/pkg/argoApplication/ArgoApplicationServiceExtended.go +++ b/pkg/nucleus/argoApplication/ArgoApplicationServiceExtended.go @@ -28,9 +28,9 @@ import ( argoApplication "github.com/devtron-labs/devtron/client/argocdServer/bean" util2 "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app/appDetails/adapter" - "github.com/devtron-labs/devtron/pkg/argoApplication/bean" - "github.com/devtron-labs/devtron/pkg/argoApplication/read" "github.com/devtron-labs/devtron/pkg/cluster" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read" util3 "github.com/devtron-labs/devtron/pkg/util" "github.com/devtron-labs/devtron/util" "google.golang.org/grpc" diff --git a/pkg/argoApplication/bean/bean.go b/pkg/nucleus/argoApplication/bean/bean.go similarity index 100% rename from pkg/argoApplication/bean/bean.go rename to pkg/nucleus/argoApplication/bean/bean.go diff --git a/pkg/argoApplication/helper/deploymentStatusHelper.go b/pkg/nucleus/argoApplication/helper/deploymentStatusHelper.go similarity index 100% rename from pkg/argoApplication/helper/deploymentStatusHelper.go rename to pkg/nucleus/argoApplication/helper/deploymentStatusHelper.go diff --git a/pkg/argoApplication/helper/helper.go b/pkg/nucleus/argoApplication/helper/helper.go similarity index 98% rename from pkg/argoApplication/helper/helper.go rename to pkg/nucleus/argoApplication/helper/helper.go index 6197e150b4..f68840f416 100644 --- a/pkg/argoApplication/helper/helper.go +++ b/pkg/nucleus/argoApplication/helper/helper.go @@ -4,8 +4,8 @@ import ( "fmt" "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/api/helm-app/gRPC" - "github.com/devtron-labs/devtron/pkg/argoApplication/bean" "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" "strconv" "strings" ) diff --git a/pkg/argoApplication/read/ArgoApplicationReadService.go b/pkg/nucleus/argoApplication/read/ArgoApplicationReadService.go similarity index 98% rename from pkg/argoApplication/read/ArgoApplicationReadService.go rename to pkg/nucleus/argoApplication/read/ArgoApplicationReadService.go index b0f30ccb8f..d95521ba17 100644 --- a/pkg/argoApplication/read/ArgoApplicationReadService.go +++ b/pkg/nucleus/argoApplication/read/ArgoApplicationReadService.go @@ -24,11 +24,11 @@ import ( k8sCommonBean "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/api/helm-app/gRPC" "github.com/devtron-labs/devtron/api/helm-app/service" - "github.com/devtron-labs/devtron/pkg/argoApplication/bean" - "github.com/devtron-labs/devtron/pkg/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/cluster/adapter" clusterRepository "github.com/devtron-labs/devtron/pkg/cluster/repository" clientErrors "github.com/devtron-labs/devtron/pkg/errors" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" "go.uber.org/zap" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/pkg/argoApplication/read/config/ArgoApplicationConfigService.go b/pkg/nucleus/argoApplication/read/config/ArgoApplicationConfigService.go similarity index 97% rename from pkg/argoApplication/read/config/ArgoApplicationConfigService.go rename to pkg/nucleus/argoApplication/read/config/ArgoApplicationConfigService.go index 0c5a8bf819..95aee5d25c 100644 --- a/pkg/argoApplication/read/config/ArgoApplicationConfigService.go +++ b/pkg/nucleus/argoApplication/read/config/ArgoApplicationConfigService.go @@ -6,10 +6,10 @@ import ( "fmt" "github.com/devtron-labs/common-lib/utils/k8s" k8sCommonBean "github.com/devtron-labs/common-lib/utils/k8s/commonBean" - "github.com/devtron-labs/devtron/pkg/argoApplication/bean" - "github.com/devtron-labs/devtron/pkg/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/cluster/adapter" clusterRepository "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" "go.uber.org/zap" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/pkg/terminal/terminalSesion.go b/pkg/terminal/terminalSesion.go index b2b5f2841a..07549991f7 100644 --- a/pkg/terminal/terminalSesion.go +++ b/pkg/terminal/terminalSesion.go @@ -26,13 +26,13 @@ import ( "github.com/caarlos0/env" "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/internal/middleware" - "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/cluster" "github.com/devtron-labs/devtron/pkg/cluster/bean" "github.com/devtron-labs/devtron/pkg/cluster/environment" bean2 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" "github.com/devtron-labs/devtron/pkg/cluster/read" "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read/config" errors1 "github.com/juju/errors" "go.uber.org/zap" "io" diff --git a/wire_gen.go b/wire_gen.go index a59599dfb9..a1365d92a7 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -130,9 +130,6 @@ import ( "github.com/devtron-labs/devtron/pkg/appStore/values/repository" service4 "github.com/devtron-labs/devtron/pkg/appStore/values/service" appWorkflow2 "github.com/devtron-labs/devtron/pkg/appWorkflow" - "github.com/devtron-labs/devtron/pkg/argoApplication" - read22 "github.com/devtron-labs/devtron/pkg/argoApplication/read" - config2 "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/asyncProvider" "github.com/devtron-labs/devtron/pkg/attributes" "github.com/devtron-labs/devtron/pkg/auth/authentication" @@ -229,6 +226,9 @@ import ( "github.com/devtron-labs/devtron/pkg/module/repo" "github.com/devtron-labs/devtron/pkg/module/store" "github.com/devtron-labs/devtron/pkg/notifier" + "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" + read22 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read" + config2 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/history" From 414712aa5c1dcb163c80891332e8d67dbb2daf50 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Mon, 7 Apr 2025 09:36:27 +0530 Subject: [PATCH 08/17] cd handler and ci handle extracted, executor workflow service executed --- Wire.go | 6 +- .../configure/BuildPipelineRestHandler.go | 8 +- .../DeploymentPipelineRestHandler.go | 6 +- .../configure/PipelineConfigRestHandler.go | 8 +- pkg/build/trigger/Service.go | 440 ++++++++++++++++- .../devtronApps/PostStageTriggerService.go | 2 +- .../devtronApps/PrePostCommonService.go | 337 +++++++++++++ .../devtronApps/PreStageTriggerService.go | 2 +- .../trigger/devtronApps/TriggerService.go | 25 +- pkg/{pipeline => executor}/WorkflowService.go | 8 +- .../WorkflowServiceIT_test.go | 7 +- .../WorkflowService_test.go | 2 +- pkg/executor/wire_executor.go | 8 + pkg/pipeline/BlobStorageConfigService.go | 7 +- pkg/pipeline/CdHandler.go | 342 +------------- pkg/pipeline/CiHandler.go | 444 +----------------- pkg/pipeline/PipelineStageService.go | 5 + pkg/workflow/dag/WorkflowDagExecutor.go | 5 +- wire_gen.go | 23 +- 19 files changed, 864 insertions(+), 821 deletions(-) create mode 100644 pkg/deployment/trigger/devtronApps/PrePostCommonService.go rename pkg/{pipeline => executor}/WorkflowService.go (99%) rename pkg/{pipeline => executor}/WorkflowServiceIT_test.go (99%) rename pkg/{pipeline => executor}/WorkflowService_test.go (99%) create mode 100644 pkg/executor/wire_executor.go diff --git a/Wire.go b/Wire.go index 3401d2d623..1106c3e7dd 100644 --- a/Wire.go +++ b/Wire.go @@ -144,6 +144,7 @@ import ( "github.com/devtron-labs/devtron/pkg/deploymentGroup" "github.com/devtron-labs/devtron/pkg/dockerRegistry" "github.com/devtron-labs/devtron/pkg/eventProcessor" + "github.com/devtron-labs/devtron/pkg/executor" "github.com/devtron-labs/devtron/pkg/generateManifest" "github.com/devtron-labs/devtron/pkg/gitops" "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" @@ -218,7 +219,7 @@ func InitializeApp() (*App, error) { userResource.UserResourceWireSet, policyGovernance.PolicyGovernanceWireSet, resourceScan.ScanningResultWireSet, - + executor.ExecutorWireSet, // -------wireset end ---------- // ------- gitSensor.GetConfig, @@ -477,9 +478,6 @@ func InitializeApp() (*App, error) { util.IntValidator, types.GetCiCdConfig, - pipeline.NewWorkflowServiceImpl, - wire.Bind(new(pipeline.WorkflowService), new(*pipeline.WorkflowServiceImpl)), - pipeline.NewCiServiceImpl, wire.Bind(new(pipeline.CiService), new(*pipeline.CiServiceImpl)), diff --git a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go index 9e749ec207..78ddc57454 100644 --- a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go @@ -974,7 +974,7 @@ func (handler *PipelineConfigRestHandlerImpl) DownloadCiWorkflowArtifacts(w http } //RBAC - file, err := handler.ciHandler.DownloadCiWorkflowArtifacts(pipelineId, buildId) + file, err := handler.ciTriggerService.DownloadCiWorkflowArtifacts(pipelineId, buildId) defer file.Close() if err != nil { handler.Logger.Errorw("service err, DownloadCiWorkflowArtifacts", "err", err, "pipelineId", pipelineId, "buildId", buildId) @@ -1029,7 +1029,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetHistoricBuildLogs(w http.Respon return } //RBAC - resp, err := handler.ciHandler.GetHistoricBuildLogs(workflowId, nil) + resp, err := handler.ciTriggerService.GetHistoricBuildLogs(workflowId, nil) if err != nil { handler.Logger.Errorw("service err, GetHistoricBuildLogs", "err", err, "pipelineId", pipelineId, "workflowId", workflowId) common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) @@ -1168,7 +1168,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetBuildLogs(w http.ResponseWriter return } } - logsReader, cleanUp, err := handler.ciHandler.GetRunningWorkflowLogs(workflowId) + logsReader, cleanUp, err := handler.ciTriggerService.GetRunningWorkflowLogs(workflowId) if err != nil { handler.Logger.Errorw("service err, GetBuildLogs", "err", err, "pipelineId", pipelineId, "workflowId", workflowId, "lastEventId", lastEventId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) @@ -1611,7 +1611,7 @@ func (handler *PipelineConfigRestHandlerImpl) CancelWorkflow(w http.ResponseWrit //RBAC - resp, err := handler.ciHandler.CancelBuild(workflowId, forceAbort) + resp, err := handler.ciTriggerService.CancelBuild(workflowId, forceAbort) if err != nil { handler.Logger.Errorw("service err, CancelWorkflow", "err", err, "workflowId", workflowId, "pipelineId", pipelineId) if util.IsErrNoRows(err) { diff --git a/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go index 52728940d5..a856e40c56 100644 --- a/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go @@ -1643,7 +1643,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetPrePostDeploymentLogs(w http.Re } //RBAC CHECK - logsReader, cleanUp, err := handler.cdHandler.GetRunningWorkflowLogs(environmentId, pipelineId, workflowId) + logsReader, cleanUp, err := handler.cdTriggerService.GetRunningWorkflowLogs(environmentId, pipelineId, workflowId) if err != nil { handler.Logger.Errorw("service err, GetPrePostDeploymentLogs", "err", err, "appId", appId, "environmentId", environmentId, "pipelineId", pipelineId, "workflowId", workflowId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) @@ -1768,7 +1768,7 @@ func (handler *PipelineConfigRestHandlerImpl) DownloadArtifacts(w http.ResponseW } //RBAC CHECK - file, err := handler.cdHandler.DownloadCdWorkflowArtifacts(buildId) + file, err := handler.cdTriggerService.DownloadCdWorkflowArtifacts(buildId) defer file.Close() if err != nil { @@ -1952,7 +1952,7 @@ func (handler *PipelineConfigRestHandlerImpl) CancelStage(w http.ResponseWriter, } //RBAC - resp, err := handler.cdHandler.CancelStage(workflowRunnerId, forceAbort, userId) + resp, err := handler.cdTriggerService.CancelStage(workflowRunnerId, forceAbort, userId) if err != nil { handler.Logger.Errorw("service err, CancelStage", "err", err, "pipelineId", pipelineId, "workflowRunnerId", workflowRunnerId) if util.IsErrNoRows(err) { diff --git a/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go b/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go index 0980abae24..11de4523aa 100644 --- a/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go +++ b/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go @@ -33,6 +33,7 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" security2 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/read" read3 "github.com/devtron-labs/devtron/pkg/team/read" @@ -140,6 +141,7 @@ type PipelineConfigRestHandlerImpl struct { environmentRepository repository2.EnvironmentRepository chartReadService read5.ChartReadService ciTriggerService trigger.Service + cdTriggerService devtronApps.TriggerService } func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger *zap.SugaredLogger, @@ -174,7 +176,9 @@ func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger teamReadService read3.TeamReadService, EnvironmentRepository repository2.EnvironmentRepository, chartReadService read5.ChartReadService, - ciTriggerService trigger.Service) *PipelineConfigRestHandlerImpl { + ciTriggerService trigger.Service, + cdTriggerService devtronApps.TriggerService, +) *PipelineConfigRestHandlerImpl { envConfig := &PipelineRestHandlerEnvConfig{} err := env.Parse(envConfig) if err != nil { @@ -216,6 +220,8 @@ func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger teamReadService: teamReadService, environmentRepository: EnvironmentRepository, chartReadService: chartReadService, + ciTriggerService: ciTriggerService, + cdTriggerService: cdTriggerService, } } diff --git a/pkg/build/trigger/Service.go b/pkg/build/trigger/Service.go index e35f2a3058..0812c485ba 100644 --- a/pkg/build/trigger/Service.go +++ b/pkg/build/trigger/Service.go @@ -1,6 +1,7 @@ package trigger import ( + "bufio" "context" "errors" "fmt" @@ -9,6 +10,7 @@ import ( blob_storage "github.com/devtron-labs/common-lib/blob-storage" "github.com/devtron-labs/common-lib/utils" bean4 "github.com/devtron-labs/common-lib/utils/bean" + "github.com/devtron-labs/common-lib/utils/k8s" commonBean "github.com/devtron-labs/common-lib/workflow" client "github.com/devtron-labs/devtron/client/events" "github.com/devtron-labs/devtron/client/gitSensor" @@ -31,13 +33,20 @@ import ( "github.com/devtron-labs/devtron/pkg/build/pipeline" buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" + "github.com/devtron-labs/devtron/pkg/cluster" + adapter2 "github.com/devtron-labs/devtron/pkg/cluster/adapter" + clusterBean "github.com/devtron-labs/devtron/pkg/cluster/bean" + "github.com/devtron-labs/devtron/pkg/cluster/environment" repository6 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" + "github.com/devtron-labs/devtron/pkg/executor" pipeline2 "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/adapter" pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" + constants2 "github.com/devtron-labs/devtron/pkg/pipeline/constants" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" + util2 "github.com/devtron-labs/devtron/pkg/pipeline/util" "github.com/devtron-labs/devtron/pkg/plugin" bean2 "github.com/devtron-labs/devtron/pkg/plugin/bean" repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository" @@ -47,7 +56,10 @@ import ( "github.com/devtron-labs/devtron/util/sliceUtil" "github.com/go-pg/pg" "go.uber.org/zap" + "io/ioutil" + "k8s.io/client-go/rest" "net/http" + "os" "path" "path/filepath" "slices" @@ -61,9 +73,14 @@ type Service interface { CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) - TriggerCiPipeline(trigger types.Trigger) (int, error) + //TriggerCiPipeline(trigger types.Trigger) (int, error) StartCiWorkflowAndPrepareWfRequest(trigger types.Trigger) (*pipelineConfig.CiPipeline, map[string]string, *pipelineConfig.CiWorkflow, *types.WorkflowRequest, error) + + CancelBuild(workflowId int, forceAbort bool) (int, error) + GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, func() error, error) + GetHistoricBuildLogs(workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) + DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) } type BuildxCacheFlags struct { @@ -73,7 +90,7 @@ type BuildxCacheFlags struct { type ServiceImpl struct { Logger *zap.SugaredLogger - workflowService pipeline2.WorkflowService + workflowService executor.WorkflowService ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository ciPipelineRepository pipelineConfig.CiPipelineRepository ciArtifactRepository repository5.CiArtifactRepository @@ -94,9 +111,14 @@ type ServiceImpl struct { ciService pipeline2.CiService ciWorkflowRepository pipelineConfig.CiWorkflowRepository gitSensorClient gitSensor.Client + ciLogService pipeline2.CiLogService + blobConfigStorageService pipeline2.BlobStorageConfigService + clusterService cluster.ClusterService + envService environment.EnvironmentService + K8sUtil *k8s.K8sServiceImpl } -func NewServiceImpl(Logger *zap.SugaredLogger, workflowService pipeline2.WorkflowService, +func NewServiceImpl(Logger *zap.SugaredLogger, workflowService executor.WorkflowService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, ciPipelineRepository pipelineConfig.CiPipelineRepository, ciArtifactRepository repository5.CiArtifactRepository, @@ -114,6 +136,11 @@ func NewServiceImpl(Logger *zap.SugaredLogger, workflowService pipeline2.Workflo ciService pipeline2.CiService, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, gitSensorClient gitSensor.Client, + ciLogService pipeline2.CiLogService, + blobConfigStorageService pipeline2.BlobStorageConfigService, + clusterService cluster.ClusterService, + envService environment.EnvironmentService, + K8sUtil *k8s.K8sServiceImpl, ) *ServiceImpl { buildxCacheFlags := &BuildxCacheFlags{} err := env.Parse(buildxCacheFlags) @@ -142,6 +169,11 @@ func NewServiceImpl(Logger *zap.SugaredLogger, workflowService pipeline2.Workflo ciService: ciService, ciWorkflowRepository: ciWorkflowRepository, gitSensorClient: gitSensorClient, + ciLogService: ciLogService, + blobConfigStorageService: blobConfigStorageService, + clusterService: clusterService, + envService: envService, + K8sUtil: K8sUtil, } config, err := types.GetCiConfig() if err != nil { @@ -227,7 +259,7 @@ func (impl *ServiceImpl) reTriggerCi(retryCount int, refCiWorkflow *pipelineConf "runtimeParameters", trigger.RuntimeParameters, "err", err) return err } - _, err = impl.TriggerCiPipeline(trigger) + _, err = impl.triggerCiPipeline(trigger) if err != nil { impl.Logger.Errorw("error occurred in re-triggering ciWorkflow", "triggerDetails", trigger, "err", err) @@ -265,7 +297,7 @@ func (impl *ServiceImpl) HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) PipelineType: ciTriggerRequest.PipelineType, CiArtifactLastFetch: createdOn, } - id, err := impl.TriggerCiPipeline(trigger) + id, err := impl.triggerCiPipeline(trigger) if err != nil { return 0, err @@ -318,7 +350,7 @@ func (impl *ServiceImpl) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRe TriggeredBy: gitCiTriggerRequest.TriggeredBy, RuntimeParameters: runtimeParams, } - id, err := impl.TriggerCiPipeline(trigger) + id, err := impl.triggerCiPipeline(trigger) if err != nil { return 0, err } @@ -569,7 +601,7 @@ func SetGitCommitValuesForBuildingCommitHash(ciMaterial *pipelineConfig.CiPipeli return newGitCommit } -func (impl *ServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { +func (impl *ServiceImpl) triggerCiPipeline(trigger types.Trigger) (int, error) { pipeline, variableSnapshot, savedCiWf, workflowRequest, err := impl.StartCiWorkflowAndPrepareWfRequest(trigger) if err != nil { return 0, err @@ -1529,3 +1561,397 @@ func (impl *ServiceImpl) markCurrentCiWorkflowFailed(savedCiWf *pipelineConfig.C } return nil } + +func (impl *ServiceImpl) CancelBuild(workflowId int, forceAbort bool) (int, error) { + workflow, err := impl.ciWorkflowRepository.FindById(workflowId) + if err != nil { + impl.Logger.Errorw("error in finding ci-workflow by workflow id", "ciWorkflowId", workflowId, "err", err) + return 0, err + } + isExt := workflow.Namespace != constants2.DefaultCiWorkflowNamespace + var env *repository6.Environment + var restConfig *rest.Config + if isExt { + restConfig, err = impl.getRestConfig(workflow) + if err != nil { + return 0, err + } + } + // Terminate workflow + cancelWfDtoRequest := &types.CancelWfRequestDto{ + ExecutorType: workflow.ExecutorType, + WorkflowName: workflow.Name, + Namespace: workflow.Namespace, + RestConfig: restConfig, + IsExt: isExt, + Environment: env, + } + // Terminate workflow + err = impl.workflowService.TerminateWorkflow(cancelWfDtoRequest) + if err != nil && forceAbort { + impl.Logger.Errorw("error in terminating workflow, with force abort flag flag as true", "workflowName", workflow.Name, "err", err) + + cancelWfDtoRequest.WorkflowGenerateName = fmt.Sprintf("%d-%s", workflowId, workflow.Name) + err1 := impl.workflowService.TerminateDanglingWorkflows(cancelWfDtoRequest) + if err1 != nil { + impl.Logger.Errorw("error in terminating dangling workflows", "cancelWfDtoRequest", cancelWfDtoRequest, "err", err) + // ignoring error here in case of force abort, confirmed from product + } + } else if err != nil && strings.Contains(err.Error(), "cannot find workflow") { + return 0, &util.ApiError{Code: "200", HttpStatusCode: http.StatusBadRequest, UserMessage: err.Error()} + } else if err != nil { + impl.Logger.Errorw("cannot terminate wf", "err", err) + return 0, err + } + if forceAbort { + err = impl.handleForceAbortCaseForCi(workflow, forceAbort) + if err != nil { + impl.Logger.Errorw("error in handleForceAbortCaseForCi", "forceAbortFlag", forceAbort, "workflow", workflow, "err", err) + return 0, err + } + return workflow.Id, nil + } + + workflow.Status = cdWorkflow.WorkflowCancel + if workflow.ExecutorType == cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM { + workflow.PodStatus = "Failed" + workflow.Message = constants2.TERMINATE_MESSAGE + } + err = impl.ciService.UpdateCiWorkflowWithStage(workflow) + if err != nil { + impl.Logger.Errorw("cannot update deleted workflow status, but wf deleted", "err", err) + return 0, err + } + imagePathReservationId := workflow.ImagePathReservationId + err = impl.customTagService.DeactivateImagePathReservation(imagePathReservationId) + if err != nil { + impl.Logger.Errorw("error in marking image tag unreserved", "err", err) + return 0, err + } + imagePathReservationIds := workflow.ImagePathReservationIds + if len(imagePathReservationIds) > 0 { + err = impl.customTagService.DeactivateImagePathReservationByImageIds(imagePathReservationIds) + if err != nil { + impl.Logger.Errorw("error in marking image tag unreserved", "err", err) + return 0, err + } + } + return workflow.Id, nil +} + +func (impl *ServiceImpl) handleForceAbortCaseForCi(workflow *pipelineConfig.CiWorkflow, forceAbort bool) error { + isWorkflowInNonTerminalStage := workflow.Status == string(v1alpha1.NodePending) || workflow.Status == string(v1alpha1.NodeRunning) + if !isWorkflowInNonTerminalStage { + if forceAbort { + return impl.updateWorkflowForForceAbort(workflow) + } else { + return &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "cannot cancel build, build not in progress"} + } + } + //this arises when someone deletes the workflow in resource browser and wants to force abort a ci + if workflow.Status == string(v1alpha1.NodeRunning) && forceAbort { + return impl.updateWorkflowForForceAbort(workflow) + } + return nil +} + +func (impl *ServiceImpl) updateWorkflowForForceAbort(workflow *pipelineConfig.CiWorkflow) error { + workflow.Status = cdWorkflow.WorkflowCancel + workflow.PodStatus = string(bean.Failed) + workflow.Message = constants2.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE + err := impl.ciService.UpdateCiWorkflowWithStage(workflow) + if err != nil { + impl.Logger.Errorw("error in updating workflow status", "err", err) + return err + } + return nil +} + +func (impl *ServiceImpl) getRestConfig(workflow *pipelineConfig.CiWorkflow) (*rest.Config, error) { + env, err := impl.envRepository.FindById(workflow.EnvironmentId) + if err != nil { + impl.Logger.Errorw("could not fetch stage env", "err", err) + return nil, err + } + + clusterBean := adapter2.GetClusterBean(*env.Cluster) + + clusterConfig := clusterBean.GetClusterConfig() + restConfig, err := impl.K8sUtil.GetRestConfigByCluster(clusterConfig) + if err != nil { + impl.Logger.Errorw("error in getting rest config by cluster id", "err", err) + return nil, err + } + return restConfig, nil +} + +func (impl *ServiceImpl) GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, func() error, error) { + ciWorkflow, err := impl.ciWorkflowRepository.FindById(workflowId) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return nil, nil, err + } + return impl.getWorkflowLogs(ciWorkflow) +} + +func (impl *ServiceImpl) getWorkflowLogs(ciWorkflow *pipelineConfig.CiWorkflow) (*bufio.Reader, func() error, error) { + if string(v1alpha1.NodePending) == ciWorkflow.PodStatus { + return bufio.NewReader(strings.NewReader("")), func() error { return nil }, nil + } + ciLogRequest := types.BuildLogRequest{ + PodName: ciWorkflow.PodName, + Namespace: ciWorkflow.Namespace, + } + isExt := false + clusterConfig := &k8s.ClusterConfig{} + if ciWorkflow.EnvironmentId != 0 { + env, err := impl.envRepository.FindById(ciWorkflow.EnvironmentId) + if err != nil { + return nil, nil, err + } + var clusterBean clusterBean.ClusterBean + if env != nil && env.Cluster != nil { + clusterBean = adapter2.GetClusterBean(*env.Cluster) + } + clusterConfig = clusterBean.GetClusterConfig() + isExt = true + } + + logStream, cleanUp, err := impl.ciLogService.FetchRunningWorkflowLogs(ciLogRequest, clusterConfig, isExt) + if logStream == nil || err != nil { + if !ciWorkflow.BlobStorageEnabled { + return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "logs-not-stored-in-repository"} + } else if string(v1alpha1.NodeSucceeded) == ciWorkflow.Status || string(v1alpha1.NodeError) == ciWorkflow.Status || string(v1alpha1.NodeFailed) == ciWorkflow.Status || ciWorkflow.Status == cdWorkflow.WorkflowCancel { + impl.Logger.Debugw("pod is not live", "podName", ciWorkflow.PodName, "err", err) + return impl.getLogsFromRepository(ciWorkflow, clusterConfig, isExt) + } + if err != nil { + impl.Logger.Errorw("err on fetch workflow logs", "err", err) + return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: err.Error()} + } else if logStream == nil { + return nil, cleanUp, fmt.Errorf("no logs found for pod %s", ciWorkflow.PodName) + } + } + logReader := bufio.NewReader(logStream) + return logReader, cleanUp, err +} + +func (impl *ServiceImpl) getLogsFromRepository(ciWorkflow *pipelineConfig.CiWorkflow, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { + impl.Logger.Debug("getting historic logs", "ciWorkflowId", ciWorkflow.Id) + ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() + ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion + logsFilePath := impl.config.GetDefaultBuildLogsKeyPrefix() + "/" + ciWorkflow.Name + "/main.log" // this is for backward compatibilty + if strings.Contains(ciWorkflow.LogLocation, "main.log") { + logsFilePath = ciWorkflow.LogLocation + } + ciLogRequest := types.BuildLogRequest{ + PipelineId: ciWorkflow.CiPipelineId, + WorkflowId: ciWorkflow.Id, + PodName: ciWorkflow.PodName, + LogsFilePath: logsFilePath, + CloudProvider: impl.config.CloudProvider, + AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, + AccountName: impl.config.AzureAccountName, + BlobContainerName: impl.config.AzureBlobContainerCiLog, + AccountKey: impl.config.AzureAccountKey, + }, + AwsS3BaseConfig: &blob_storage.AwsS3BaseConfig{ + AccessKey: impl.config.BlobStorageS3AccessKey, + Passkey: impl.config.BlobStorageS3SecretKey, + EndpointUrl: impl.config.BlobStorageS3Endpoint, + IsInSecure: impl.config.BlobStorageS3EndpointInsecure, + BucketName: ciConfigLogsBucket, + Region: ciConfigCiCacheRegion, + VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, + }, + GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ + BucketName: ciConfigLogsBucket, + CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, + }, + } + useExternalBlobStorage := pipeline2.IsExternalBlobStorageEnabled(isExt, impl.config.UseBlobStorageConfigInCiWorkflow) + if useExternalBlobStorage { + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) + if err != nil { + impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, nil, err + } + rq := &ciLogRequest + rq.SetBuildLogRequest(cmConfig, secretConfig) + } + oldLogsStream, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, ciLogRequest) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return nil, nil, err + } + logReader := bufio.NewReader(oldLogsStream) + return logReader, cleanUp, err +} + +func (impl *ServiceImpl) GetHistoricBuildLogs(workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) { + var err error + if ciWorkflow == nil { + ciWorkflow, err = impl.ciWorkflowRepository.FindById(workflowId) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return nil, err + } + } + ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() + ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion + ciLogRequest := types.BuildLogRequest{ + PipelineId: ciWorkflow.CiPipelineId, + WorkflowId: ciWorkflow.Id, + PodName: ciWorkflow.PodName, + LogsFilePath: ciWorkflow.LogLocation, + CloudProvider: impl.config.CloudProvider, + AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, + AccountName: impl.config.AzureAccountName, + BlobContainerName: impl.config.AzureBlobContainerCiLog, + AccountKey: impl.config.AzureAccountKey, + }, + AwsS3BaseConfig: &blob_storage.AwsS3BaseConfig{ + AccessKey: impl.config.BlobStorageS3AccessKey, + Passkey: impl.config.BlobStorageS3SecretKey, + EndpointUrl: impl.config.BlobStorageS3Endpoint, + IsInSecure: impl.config.BlobStorageS3EndpointInsecure, + BucketName: ciConfigLogsBucket, + Region: ciConfigCiCacheRegion, + VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, + }, + GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ + BucketName: ciConfigLogsBucket, + CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, + }, + } + useExternalBlobStorage := pipeline2.IsExternalBlobStorageEnabled(ciWorkflow.IsExternalRunInJobType(), impl.config.UseBlobStorageConfigInCiWorkflow) + if useExternalBlobStorage { + envBean, err := impl.envService.FindById(ciWorkflow.EnvironmentId) + if err != nil { + impl.Logger.Errorw("error in getting envBean by envId", "err", err, "envId", ciWorkflow.EnvironmentId) + return nil, err + } + clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(envBean.ClusterId) + if err != nil { + impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig by clusterId", "err", err, "clusterId", envBean.ClusterId) + return nil, err + } + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) + if err != nil { + impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, err + } + rq := &ciLogRequest + rq.SetBuildLogRequest(cmConfig, secretConfig) + } + logsFile, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, ciLogRequest) + logs, err := ioutil.ReadFile(logsFile.Name()) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return map[string]string{}, err + } + logStr := string(logs) + resp := make(map[string]string) + resp["logs"] = logStr + defer cleanUp() + return resp, err +} + +func (impl *ServiceImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) { + ciWorkflow, err := impl.ciWorkflowRepository.FindById(buildId) + if err != nil { + impl.Logger.Errorw("unable to fetch ciWorkflow", "err", err) + return nil, err + } + useExternalBlobStorage := pipeline2.IsExternalBlobStorageEnabled(ciWorkflow.IsExternalRunInJobType(), impl.config.UseBlobStorageConfigInCiWorkflow) + if !ciWorkflow.BlobStorageEnabled { + return nil, errors.New("logs-not-stored-in-repository") + } + + if ciWorkflow.CiPipelineId != pipelineId { + impl.Logger.Error("invalid request, wf not in pipeline") + return nil, errors.New("invalid request, wf not in pipeline") + } + + ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() + item := strconv.Itoa(ciWorkflow.Id) + ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion + azureBlobConfig := &blob_storage.AzureBlobBaseConfig{ + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, + AccountName: impl.config.AzureAccountName, + BlobContainerName: impl.config.AzureBlobContainerCiLog, + AccountKey: impl.config.AzureAccountKey, + } + awsS3BaseConfig := &blob_storage.AwsS3BaseConfig{ + AccessKey: impl.config.BlobStorageS3AccessKey, + Passkey: impl.config.BlobStorageS3SecretKey, + EndpointUrl: impl.config.BlobStorageS3Endpoint, + IsInSecure: impl.config.BlobStorageS3EndpointInsecure, + BucketName: ciConfigLogsBucket, + Region: ciConfigCiCacheRegion, + VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, + } + gcpBlobBaseConfig := &blob_storage.GcpBlobBaseConfig{ + BucketName: ciConfigLogsBucket, + CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, + } + + ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() + key := fmt.Sprintf(ciArtifactLocationFormat, ciWorkflow.Id, ciWorkflow.Id) + if len(ciWorkflow.CiArtifactLocation) != 0 && util2.IsValidUrlSubPath(ciWorkflow.CiArtifactLocation) { + key = ciWorkflow.CiArtifactLocation + } else if util2.IsValidUrlSubPath(key) { + impl.ciWorkflowRepository.MigrateCiArtifactLocation(ciWorkflow.Id, key) + } + baseLogLocationPathConfig := impl.config.BaseLogLocationPath + blobStorageService := blob_storage.NewBlobStorageServiceImpl(nil) + destinationKey := filepath.Clean(filepath.Join(baseLogLocationPathConfig, item)) + request := &blob_storage.BlobStorageRequest{ + StorageType: impl.config.CloudProvider, + SourceKey: key, + DestinationKey: destinationKey, + AzureBlobBaseConfig: azureBlobConfig, + AwsS3BaseConfig: awsS3BaseConfig, + GcpBlobBaseConfig: gcpBlobBaseConfig, + } + if useExternalBlobStorage { + envBean, err := impl.envService.FindById(ciWorkflow.EnvironmentId) + if err != nil { + impl.Logger.Errorw("error in getting envBean by envId", "err", err, "envId", ciWorkflow.EnvironmentId) + return nil, err + } + clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(envBean.ClusterId) + if err != nil { + impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig by clusterId", "err", err, "clusterId", envBean.ClusterId) + return nil, err + } + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) + if err != nil { + impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, err + } + request = pipeline2.UpdateRequestWithExtClusterCmAndSecret(request, cmConfig, secretConfig) + } + _, numBytes, err := blobStorageService.Get(request) + if err != nil { + impl.Logger.Errorw("error occurred while downloading file", "request", request, "error", err) + return nil, errors.New("failed to download resource") + } + + file, err := os.Open(destinationKey) + if err != nil { + impl.Logger.Errorw("unable to open file", "file", item, "err", err) + return nil, errors.New("unable to open file") + } + + impl.Logger.Infow("Downloaded ", "filename", file.Name(), "bytes", numBytes) + return file, nil +} diff --git a/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go index 8a11b06a6b..defe63b75d 100644 --- a/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go @@ -123,7 +123,7 @@ func (impl *TriggerServiceImpl) TriggerPostStage(request bean.TriggerRequest) (* return nil, err } - _, jobHelmPackagePath, err := impl.cdWorkflowService.SubmitWorkflow(cdStageWorkflowRequest) + _, jobHelmPackagePath, err := impl.workflowService.SubmitWorkflow(cdStageWorkflowRequest) if err != nil { impl.logger.Errorw("error in submitting workflow", "err", err, "workflowId", cdStageWorkflowRequest.WorkflowId, "pipeline", pipeline, "env", env) runner.Status = cdWorkflow.WorkflowFailed diff --git a/pkg/deployment/trigger/devtronApps/PrePostCommonService.go b/pkg/deployment/trigger/devtronApps/PrePostCommonService.go new file mode 100644 index 0000000000..d1cf588032 --- /dev/null +++ b/pkg/deployment/trigger/devtronApps/PrePostCommonService.go @@ -0,0 +1,337 @@ +package devtronApps + +import ( + "bufio" + "errors" + "fmt" + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + blob_storage "github.com/devtron-labs/common-lib/blob-storage" + "github.com/devtron-labs/common-lib/utils/k8s" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + cdWorkflow2 "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" + "github.com/devtron-labs/devtron/internal/util" + bean2 "github.com/devtron-labs/devtron/pkg/bean" + "github.com/devtron-labs/devtron/pkg/cluster/adapter" + bean3 "github.com/devtron-labs/devtron/pkg/cluster/bean" + "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/pipeline/constants" + "github.com/devtron-labs/devtron/pkg/pipeline/types" + util2 "github.com/devtron-labs/devtron/pkg/pipeline/util" + "k8s.io/client-go/rest" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "time" +) + +func (impl *TriggerServiceImpl) CancelStage(workflowRunnerId int, forceAbort bool, userId int32) (int, error) { + workflowRunner, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(workflowRunnerId) + if err != nil { + impl.logger.Errorw("err", "err", err) + return 0, err + } + pipeline, err := impl.pipelineRepository.FindById(workflowRunner.CdWorkflow.PipelineId) + if err != nil { + impl.logger.Errorw("error while fetching cd pipeline", "err", err) + return 0, err + } + + env, err := impl.envRepository.FindById(pipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("could not fetch stage env", "err", err) + return 0, err + } + + var clusterBean bean3.ClusterBean + if env != nil && env.Cluster != nil { + clusterBean = adapter.GetClusterBean(*env.Cluster) + } + clusterConfig := clusterBean.GetClusterConfig() + var isExtCluster bool + if workflowRunner.WorkflowType == types.PRE { + isExtCluster = pipeline.RunPreStageInEnv + } else if workflowRunner.WorkflowType == types.POST { + isExtCluster = pipeline.RunPostStageInEnv + } + var restConfig *rest.Config + if isExtCluster { + restConfig, err = impl.K8sUtil.GetRestConfigByCluster(clusterConfig) + if err != nil { + impl.logger.Errorw("error in getting rest config by cluster id", "err", err) + return 0, err + } + } + // Terminate workflow + cancelWfDtoRequest := &types.CancelWfRequestDto{ + ExecutorType: workflowRunner.ExecutorType, + WorkflowName: workflowRunner.Name, + Namespace: workflowRunner.Namespace, + RestConfig: restConfig, + IsExt: isExtCluster, + Environment: nil, + } + err = impl.workflowService.TerminateWorkflow(cancelWfDtoRequest) + if err != nil && forceAbort { + impl.logger.Errorw("error in terminating workflow, with force abort flag as true", "workflowName", workflowRunner.Name, "err", err) + cancelWfDtoRequest.WorkflowGenerateName = fmt.Sprintf("%d-%s", workflowRunnerId, workflowRunner.Name) + err1 := impl.workflowService.TerminateDanglingWorkflows(cancelWfDtoRequest) + if err1 != nil { + impl.logger.Errorw("error in terminating dangling workflows", "cancelWfDtoRequest", cancelWfDtoRequest, "err", err) + // ignoring error here in case of force abort, confirmed from product + } + } else if err != nil && strings.Contains(err.Error(), "cannot find workflow") { + return 0, &util.ApiError{Code: "200", HttpStatusCode: http.StatusBadRequest, UserMessage: err.Error()} + } else if err != nil { + impl.logger.Error("cannot terminate wf runner", "err", err) + return 0, err + } + if forceAbort { + err = impl.handleForceAbortCaseForCdStage(workflowRunner, forceAbort) + if err != nil { + impl.logger.Errorw("error in handleForceAbortCaseForCdStage", "forceAbortFlag", forceAbort, "workflowRunner", workflowRunner, "err", err) + return 0, err + } + return workflowRunner.Id, nil + } + if len(workflowRunner.ImagePathReservationIds) > 0 { + err := impl.customTagService.DeactivateImagePathReservationByImageIds(workflowRunner.ImagePathReservationIds) + if err != nil { + impl.logger.Errorw("error in deactivating image path reservation ids", "err", err) + return 0, err + } + } + workflowRunner.Status = cdWorkflow2.WorkflowCancel + workflowRunner.UpdatedOn = time.Now() + workflowRunner.UpdatedBy = userId + err = impl.cdWorkflowRunnerService.UpdateCdWorkflowRunnerWithStage(workflowRunner) + if err != nil { + impl.logger.Error("cannot update deleted workflow runner status, but wf deleted", "err", err) + return 0, err + } + return workflowRunner.Id, nil +} + +func (impl *TriggerServiceImpl) updateWorkflowRunnerForForceAbort(workflowRunner *pipelineConfig.CdWorkflowRunner) error { + workflowRunner.Status = cdWorkflow2.WorkflowCancel + workflowRunner.PodStatus = string(bean2.Failed) + workflowRunner.Message = constants.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE + err := impl.cdWorkflowRunnerService.UpdateCdWorkflowRunnerWithStage(workflowRunner) + if err != nil { + impl.logger.Errorw("error in updating workflow status in cd workflow runner in force abort case", "err", err) + return err + } + return nil +} + +func (impl *TriggerServiceImpl) handleForceAbortCaseForCdStage(workflowRunner *pipelineConfig.CdWorkflowRunner, forceAbort bool) error { + isWorkflowInNonTerminalStage := workflowRunner.Status == string(v1alpha1.NodePending) || workflowRunner.Status == string(v1alpha1.NodeRunning) + if !isWorkflowInNonTerminalStage { + if forceAbort { + return impl.updateWorkflowRunnerForForceAbort(workflowRunner) + } else { + return &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "cannot cancel stage, stage not in progress"} + } + } + //this arises when someone deletes the workflow in resource browser and wants to force abort a cd stage(pre/post) + if workflowRunner.Status == string(v1alpha1.NodeRunning) && forceAbort { + return impl.updateWorkflowRunnerForForceAbort(workflowRunner) + } + return nil +} + +func (impl *TriggerServiceImpl) DownloadCdWorkflowArtifacts(buildId int) (*os.File, error) { + wfr, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(buildId) + if err != nil { + impl.logger.Errorw("unable to fetch ciWorkflow", "err", err) + return nil, err + } + useExternalBlobStorage := pipeline.IsExternalBlobStorageEnabled(wfr.IsExternalRun(), impl.config.UseBlobStorageConfigInCdWorkflow) + if !wfr.BlobStorageEnabled { + return nil, errors.New("logs-not-stored-in-repository") + } + + cdConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() + cdConfigCdCacheRegion := impl.config.GetDefaultCdLogsBucketRegion() + + item := strconv.Itoa(wfr.Id) + awsS3BaseConfig := &blob_storage.AwsS3BaseConfig{ + AccessKey: impl.config.BlobStorageS3AccessKey, + Passkey: impl.config.BlobStorageS3SecretKey, + EndpointUrl: impl.config.BlobStorageS3Endpoint, + IsInSecure: impl.config.BlobStorageS3EndpointInsecure, + BucketName: cdConfigLogsBucket, + Region: cdConfigCdCacheRegion, + VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, + } + azureBlobBaseConfig := &blob_storage.AzureBlobBaseConfig{ + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, + AccountKey: impl.config.AzureAccountKey, + AccountName: impl.config.AzureAccountName, + BlobContainerName: impl.config.AzureBlobContainerCiLog, + } + gcpBlobBaseConfig := &blob_storage.GcpBlobBaseConfig{ + BucketName: cdConfigLogsBucket, + CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, + } + cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() + key := fmt.Sprintf(cdArtifactLocationFormat, wfr.CdWorkflow.Id, wfr.Id) + if len(wfr.CdArtifactLocation) != 0 && util2.IsValidUrlSubPath(wfr.CdArtifactLocation) { + key = wfr.CdArtifactLocation + } else if util2.IsValidUrlSubPath(key) { + impl.cdWorkflowRepository.MigrateCdArtifactLocation(wfr.Id, key) + } + baseLogLocationPathConfig := impl.config.BaseLogLocationPath + blobStorageService := blob_storage.NewBlobStorageServiceImpl(nil) + destinationKey := filepath.Clean(filepath.Join(baseLogLocationPathConfig, item)) + request := &blob_storage.BlobStorageRequest{ + StorageType: impl.config.CloudProvider, + SourceKey: key, + DestinationKey: destinationKey, + AzureBlobBaseConfig: azureBlobBaseConfig, + AwsS3BaseConfig: awsS3BaseConfig, + GcpBlobBaseConfig: gcpBlobBaseConfig, + } + if useExternalBlobStorage { + clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(wfr.CdWorkflow.Pipeline.Environment.ClusterId) + if err != nil { + impl.logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig", "err", err, "clusterId", wfr.CdWorkflow.Pipeline.Environment.ClusterId) + return nil, err + } + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, wfr.Namespace) + if err != nil { + impl.logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, err + } + request = pipeline.UpdateRequestWithExtClusterCmAndSecret(request, cmConfig, secretConfig) + } + _, numBytes, err := blobStorageService.Get(request) + if err != nil { + impl.logger.Errorw("error occurred while downloading file", "request", request, "error", err) + return nil, errors.New("failed to download resource") + } + + file, err := os.Open(destinationKey) + if err != nil { + impl.logger.Errorw("unable to open file", "file", item, "err", err) + return nil, errors.New("unable to open file") + } + + impl.logger.Infow("Downloaded ", "name", file.Name(), "bytes", numBytes) + return file, nil +} + +func (impl *TriggerServiceImpl) GetRunningWorkflowLogs(environmentId int, pipelineId int, wfrId int) (*bufio.Reader, func() error, error) { + cdWorkflow, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(wfrId) + if err != nil { + impl.logger.Errorw("error on fetch wf runner", "err", err) + return nil, nil, err + } + + env, err := impl.envRepository.FindById(environmentId) + if err != nil { + impl.logger.Errorw("could not fetch stage env", "err", err) + return nil, nil, err + } + + pipeline, err := impl.pipelineRepository.FindById(cdWorkflow.CdWorkflow.PipelineId) + if err != nil { + impl.logger.Errorw("error while fetching cd pipeline", "err", err) + return nil, nil, err + } + var clusterBean bean3.ClusterBean + if env != nil && env.Cluster != nil { + clusterBean = adapter.GetClusterBean(*env.Cluster) + } + clusterConfig := clusterBean.GetClusterConfig() + var isExtCluster bool + if cdWorkflow.WorkflowType == types.PRE { + isExtCluster = pipeline.RunPreStageInEnv + } else if cdWorkflow.WorkflowType == types.POST { + isExtCluster = pipeline.RunPostStageInEnv + } + return impl.getWorkflowLogs(pipelineId, cdWorkflow, clusterConfig, isExtCluster) +} + +func (impl *TriggerServiceImpl) getWorkflowLogs(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, runStageInEnv bool) (*bufio.Reader, func() error, error) { + cdLogRequest := types.BuildLogRequest{ + PodName: cdWorkflow.PodName, + Namespace: cdWorkflow.Namespace, + } + + logStream, cleanUp, err := impl.ciLogService.FetchRunningWorkflowLogs(cdLogRequest, clusterConfig, runStageInEnv) + if logStream == nil || err != nil { + if !cdWorkflow.BlobStorageEnabled { + return nil, nil, errors.New("logs-not-stored-in-repository") + } else if string(v1alpha1.NodeSucceeded) == cdWorkflow.Status || string(v1alpha1.NodeError) == cdWorkflow.Status || string(v1alpha1.NodeFailed) == cdWorkflow.Status || cdWorkflow.Status == cdWorkflow2.WorkflowCancel { + impl.logger.Debugw("pod is not live", "podName", cdWorkflow.PodName, "err", err) + return impl.getLogsFromRepository(pipelineId, cdWorkflow, clusterConfig, runStageInEnv) + } + if err != nil { + impl.logger.Errorw("err on fetch workflow logs", "err", err) + return nil, nil, err + } else if logStream == nil { + return nil, cleanUp, fmt.Errorf("no logs found for pod %s", cdWorkflow.PodName) + } + } + logReader := bufio.NewReader(logStream) + return logReader, cleanUp, err +} + +func (impl *TriggerServiceImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { + impl.logger.Debug("getting historic logs", "pipelineId", pipelineId) + + cdConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() // TODO -fixme + cdConfigCdCacheRegion := impl.config.GetDefaultCdLogsBucketRegion() + + cdLogRequest := types.BuildLogRequest{ + PipelineId: cdWorkflow.CdWorkflow.PipelineId, + WorkflowId: cdWorkflow.Id, + PodName: cdWorkflow.PodName, + LogsFilePath: cdWorkflow.LogLocation, // impl.ciCdConfig.CiDefaultBuildLogsKeyPrefix + "/" + cdWorkflow.Name + "/main.log", //TODO - fixme + CloudProvider: impl.config.CloudProvider, + AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, + AccountName: impl.config.AzureAccountName, + BlobContainerName: impl.config.AzureBlobContainerCiLog, + AccountKey: impl.config.AzureAccountKey, + }, + AwsS3BaseConfig: &blob_storage.AwsS3BaseConfig{ + AccessKey: impl.config.BlobStorageS3AccessKey, + Passkey: impl.config.BlobStorageS3SecretKey, + EndpointUrl: impl.config.BlobStorageS3Endpoint, + IsInSecure: impl.config.BlobStorageS3EndpointInsecure, + BucketName: cdConfigLogsBucket, + Region: cdConfigCdCacheRegion, + VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, + }, + GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ + BucketName: cdConfigLogsBucket, + CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, + }, + } + useExternalBlobStorage := pipeline.IsExternalBlobStorageEnabled(isExt, impl.config.UseBlobStorageConfigInCdWorkflow) + if useExternalBlobStorage { + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, cdWorkflow.Namespace) + if err != nil { + impl.logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, nil, err + } + rq := &cdLogRequest + rq.SetBuildLogRequest(cmConfig, secretConfig) + } + + impl.logger.Debugw("s3 log req ", "pipelineId", pipelineId, "runnerId", cdWorkflow.Id) + oldLogsStream, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, cdLogRequest) + if err != nil { + impl.logger.Errorw("err", err) + return nil, nil, err + } + logReader := bufio.NewReader(oldLogsStream) + return logReader, cleanUp, err +} diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go index ef11bedf2c..64c373a235 100644 --- a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go @@ -147,7 +147,7 @@ func (impl *TriggerServiceImpl) TriggerPreStage(request bean.TriggerRequest) (*b cdStageWorkflowRequest.Pipeline = pipeline cdStageWorkflowRequest.Env = env cdStageWorkflowRequest.Type = pipelineConfigBean.CD_WORKFLOW_PIPELINE_TYPE - _, jobHelmPackagePath, err := impl.cdWorkflowService.SubmitWorkflow(cdStageWorkflowRequest) + _, jobHelmPackagePath, err := impl.workflowService.SubmitWorkflow(cdStageWorkflowRequest) span.End() if err != nil { runner.Status = cdWorkflow.WorkflowFailed diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/TriggerService.go index 7a5d135045..942761fc47 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService.go @@ -17,6 +17,7 @@ package devtronApps import ( + "bufio" "context" "errors" "fmt" @@ -52,6 +53,7 @@ import ( "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/read" pipeline2 "github.com/devtron-labs/devtron/pkg/build/pipeline" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" + "github.com/devtron-labs/devtron/pkg/cluster" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" repository5 "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/deployment/common" @@ -68,6 +70,7 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/userDeploymentRequest/service" clientErrors "github.com/devtron-labs/devtron/pkg/errors" "github.com/devtron-labs/devtron/pkg/eventProcessor/out" + "github.com/devtron-labs/devtron/pkg/executor" "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" k8s2 "github.com/devtron-labs/devtron/pkg/k8s" "github.com/devtron-labs/devtron/pkg/pipeline" @@ -93,6 +96,7 @@ import ( status2 "google.golang.org/grpc/status" "helm.sh/helm/v3/pkg/chart" "net/http" + "os" "path" "regexp" "strconv" @@ -112,6 +116,10 @@ type TriggerService interface { TriggerAutomaticDeployment(request bean.TriggerRequest) error TriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) + + CancelStage(workflowRunnerId int, forceAbort bool, userId int32) (int, error) + DownloadCdWorkflowArtifacts(buildId int) (*os.File, error) + GetRunningWorkflowLogs(environmentId int, pipelineId int, workflowId int) (*bufio.Reader, func() error, error) } type TriggerServiceImpl struct { @@ -136,7 +144,6 @@ type TriggerServiceImpl struct { pluginInputVariableParser pipeline.PluginInputVariableParser prePostCdScriptHistoryService history.PrePostCdScriptHistoryService scopedVariableManager variables.ScopedVariableCMCSManager - cdWorkflowService pipeline.WorkflowService imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService userService user.UserService gitSensorClient gitSensorClient.Client @@ -174,6 +181,10 @@ type TriggerServiceImpl struct { attributeService attributes.AttributesService clusterRepository repository5.ClusterRepository cdWorkflowRunnerService cd.CdWorkflowRunnerService + clusterService cluster.ClusterService + ciLogService pipeline.CiLogService + workflowService executor.WorkflowService + blobConfigStorageService pipeline.BlobStorageConfigService } func NewTriggerServiceImpl(logger *zap.SugaredLogger, @@ -194,7 +205,6 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, pluginInputVariableParser pipeline.PluginInputVariableParser, prePostCdScriptHistoryService history.PrePostCdScriptHistoryService, scopedVariableManager variables.ScopedVariableCMCSManager, - cdWorkflowService pipeline.WorkflowService, imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService, userService user.UserService, gitSensorClient gitSensorClient.Client, @@ -233,6 +243,10 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, attributeService attributes.AttributesService, clusterRepository repository5.ClusterRepository, cdWorkflowRunnerService cd.CdWorkflowRunnerService, + clusterService cluster.ClusterService, + ciLogService pipeline.CiLogService, + workflowService executor.WorkflowService, + blobConfigStorageService pipeline.BlobStorageConfigService, ) (*TriggerServiceImpl, error) { impl := &TriggerServiceImpl{ logger: logger, @@ -253,7 +267,6 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, pluginInputVariableParser: pluginInputVariableParser, prePostCdScriptHistoryService: prePostCdScriptHistoryService, scopedVariableManager: scopedVariableManager, - cdWorkflowService: cdWorkflowService, imageDigestPolicyService: imageDigestPolicyService, userService: userService, gitSensorClient: gitSensorClient, @@ -297,7 +310,11 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, attributeService: attributeService, cdWorkflowRunnerService: cdWorkflowRunnerService, - clusterRepository: clusterRepository, + clusterRepository: clusterRepository, + clusterService: clusterService, + ciLogService: ciLogService, + workflowService: workflowService, + blobConfigStorageService: blobConfigStorageService, } config, err := types.GetCdConfig() if err != nil { diff --git a/pkg/pipeline/WorkflowService.go b/pkg/executor/WorkflowService.go similarity index 99% rename from pkg/pipeline/WorkflowService.go rename to pkg/executor/WorkflowService.go index 191b973c75..339c566113 100644 --- a/pkg/pipeline/WorkflowService.go +++ b/pkg/executor/WorkflowService.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package pipeline +package executor import ( "context" @@ -33,6 +33,7 @@ import ( "github.com/devtron-labs/devtron/pkg/config/read" v1 "github.com/devtron-labs/devtron/pkg/infraConfig/bean/v1" k8s2 "github.com/devtron-labs/devtron/pkg/k8s" + "github.com/devtron-labs/devtron/pkg/pipeline" bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders" @@ -66,7 +67,7 @@ type WorkflowServiceImpl struct { ciCdConfig *types.CiCdConfig configMapService read.ConfigReadService envRepository repository2.EnvironmentRepository - globalCMCSService GlobalCMCSService + globalCMCSService pipeline.GlobalCMCSService argoWorkflowExecutor executors.ArgoWorkflowExecutor systemWorkflowExecutor executors.SystemWorkflowExecutor k8sUtil *k8s.K8sServiceImpl @@ -80,7 +81,7 @@ func NewWorkflowServiceImpl(Logger *zap.SugaredLogger, envRepository repository2.EnvironmentRepository, ciCdConfig *types.CiCdConfig, configMapService read.ConfigReadService, - globalCMCSService GlobalCMCSService, + globalCMCSService pipeline.GlobalCMCSService, argoWorkflowExecutor executors.ArgoWorkflowExecutor, k8sUtil *k8s.K8sServiceImpl, systemWorkflowExecutor executors.SystemWorkflowExecutor, @@ -378,6 +379,7 @@ func (impl *WorkflowServiceImpl) getWorkflowExecutor(executorType cdWorkflow.Wor impl.Logger.Warnw("workflow executor not found", "type", executorType) return nil } + func (impl *WorkflowServiceImpl) GetWorkflow(executorType cdWorkflow.WorkflowExecutorType, name string, namespace string, restConfig *rest.Config) (*unstructured.UnstructuredList, error) { impl.Logger.Debug("getting wf", name) workflowExecutor := impl.getWorkflowExecutor(executorType) diff --git a/pkg/pipeline/WorkflowServiceIT_test.go b/pkg/executor/WorkflowServiceIT_test.go similarity index 99% rename from pkg/pipeline/WorkflowServiceIT_test.go rename to pkg/executor/WorkflowServiceIT_test.go index c025abe1fe..b1d569487f 100644 --- a/pkg/pipeline/WorkflowServiceIT_test.go +++ b/pkg/executor/WorkflowServiceIT_test.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package pipeline +package executor import ( "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" @@ -35,6 +35,7 @@ import ( "github.com/devtron-labs/devtron/pkg/commonService" k8s2 "github.com/devtron-labs/devtron/pkg/k8s" "github.com/devtron-labs/devtron/pkg/k8s/informer" + "github.com/devtron-labs/devtron/pkg/pipeline" bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/types" @@ -59,10 +60,10 @@ var cmManifest3 = "{\"kind\":\"ConfigMap\",\"apiVersion\":\"v1\",\"metadata\":{\ var cmManifest4 = "{\"kind\":\"ConfigMap\",\"apiVersion\":\"v1\",\"metadata\":{\"name\":\"cm4-5-ci\",\"creationTimestamp\":null,\"ownerReferences\":[{\"apiVersion\":\"argoproj.io/v1alpha1\",\"kind\":\"Workflow\",\"name\":\"{{workflow.name}}\",\"uid\":\"{{workflow.uid}}\",\"blockOwnerDeletion\":true}]},\"data\":{\"key4\":\"value4\"}}" func getWorkflowServiceImpl(t *testing.T) *WorkflowServiceImpl { - logger, dbConnection := getDbConnAndLoggerService(t) + logger, dbConnection := pipeline.getDbConnAndLoggerService(t) ciCdConfig, _ := types.GetCiCdConfig() newGlobalCMCSRepositoryImpl := repository.NewGlobalCMCSRepositoryImpl(logger, dbConnection) - globalCMCSServiceImpl := NewGlobalCMCSServiceImpl(logger, newGlobalCMCSRepositoryImpl) + globalCMCSServiceImpl := pipeline.NewGlobalCMCSServiceImpl(logger, newGlobalCMCSRepositoryImpl) newEnvConfigOverrideRepository := chartConfig.NewEnvConfigOverrideRepository(dbConnection) newConfigMapRepositoryImpl := chartConfig.NewConfigMapRepositoryImpl(logger, dbConnection) newChartRepository := chartRepoRepository.NewChartRepository(dbConnection) diff --git a/pkg/pipeline/WorkflowService_test.go b/pkg/executor/WorkflowService_test.go similarity index 99% rename from pkg/pipeline/WorkflowService_test.go rename to pkg/executor/WorkflowService_test.go index 4b4e937d68..1ebcb9639c 100644 --- a/pkg/pipeline/WorkflowService_test.go +++ b/pkg/executor/WorkflowService_test.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package pipeline +package executor import ( "fmt" diff --git a/pkg/executor/wire_executor.go b/pkg/executor/wire_executor.go new file mode 100644 index 0000000000..a99bf31f61 --- /dev/null +++ b/pkg/executor/wire_executor.go @@ -0,0 +1,8 @@ +package executor + +import "github.com/google/wire" + +var ExecutorWireSet = wire.NewSet( + NewWorkflowServiceImpl, + wire.Bind(new(WorkflowService), new(*WorkflowServiceImpl)), +) diff --git a/pkg/pipeline/BlobStorageConfigService.go b/pkg/pipeline/BlobStorageConfigService.go index fd6141c111..53ea2efde6 100644 --- a/pkg/pipeline/BlobStorageConfigService.go +++ b/pkg/pipeline/BlobStorageConfigService.go @@ -91,7 +91,7 @@ func (impl *BlobStorageConfigServiceImpl) FetchCmAndSecretBlobConfigFromExternal return cmConfig, secretConfig, nil } -func updateRequestWithExtClusterCmAndSecret(request *blob_storage.BlobStorageRequest, cmConfig *bean2.CmBlobStorageConfig, secretConfig *bean2.SecretBlobStorageConfig) *blob_storage.BlobStorageRequest { +func UpdateRequestWithExtClusterCmAndSecret(request *blob_storage.BlobStorageRequest, cmConfig *bean2.CmBlobStorageConfig, secretConfig *bean2.SecretBlobStorageConfig) *blob_storage.BlobStorageRequest { request.StorageType = cmConfig.CloudProvider request.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey @@ -113,3 +113,8 @@ func updateRequestWithExtClusterCmAndSecret(request *blob_storage.BlobStorageReq return request } + +func IsExternalBlobStorageEnabled(isExternalRun bool, useBlobStorageConfigInCdWorkflow bool) bool { + // TODO impl.config.UseBlobStorageConfigInCdWorkflow fetches the live status, we need to check from db as well, we should put useExternalBlobStorage in db + return isExternalRun && !useBlobStorageConfigInCdWorkflow +} diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 230d35c522..6816db9067 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -17,41 +17,30 @@ package pipeline import ( - "bufio" "errors" "fmt" "github.com/devtron-labs/common-lib/utils" bean4 "github.com/devtron-labs/common-lib/utils/bean" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/adapter/cdWorkflow" cdWorkflow2 "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" - bean2 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/build/artifacts/imageTagging" buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" - "github.com/devtron-labs/devtron/pkg/cluster/adapter" - bean3 "github.com/devtron-labs/devtron/pkg/cluster/bean" repository3 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" - "github.com/devtron-labs/devtron/pkg/pipeline/constants" - util2 "github.com/devtron-labs/devtron/pkg/pipeline/util" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" bean5 "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus/bean" "github.com/devtron-labs/devtron/pkg/workflow/cd" - "net/http" - "os" - "path/filepath" "strconv" "strings" "time" "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" - blob_storage "github.com/devtron-labs/common-lib/blob-storage" "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/auth/user" - "github.com/devtron-labs/devtron/pkg/cluster" pipelineBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/types" resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup" @@ -60,7 +49,6 @@ import ( "github.com/go-pg/pg" "go.opentelemetry.io/otel" "go.uber.org/zap" - "k8s.io/client-go/rest" ) const ( @@ -72,11 +60,8 @@ const ( type CdHandler interface { UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, string, bool, error) GetCdBuildHistory(appId int, environmentId int, pipelineId int, offset int, size int) ([]pipelineBean.CdWorkflowWithArtifact, error) - GetRunningWorkflowLogs(environmentId int, pipelineId int, workflowId int) (*bufio.Reader, func() error, error) FetchCdWorkflowDetails(appId int, environmentId int, pipelineId int, buildId int) (types.WorkflowResponse, error) - DownloadCdWorkflowArtifacts(buildId int) (*os.File, error) FetchCdPrePostStageStatus(pipelineId int) ([]pipelineBean.CdWorkflowWithArtifact, error) - CancelStage(workflowRunnerId int, forceAbort bool, userId int32) (int, error) FetchAppWorkflowStatusForTriggerView(appId int) ([]*pipelineConfig.CdWorkflowStatus, error) FetchAppWorkflowStatusForTriggerViewForEnvironment(request resourceGroup2.ResourceGroupingRequest, token string) ([]*pipelineConfig.CdWorkflowStatus, error) FetchAppDeploymentStatusForEnvironments(request resourceGroup2.ResourceGroupingRequest, token string) ([]*pipelineConfig.AppDeploymentStatus, error) @@ -86,7 +71,6 @@ type CdHandler interface { type CdHandlerImpl struct { Logger *zap.SugaredLogger userService user.UserService - ciLogService CiLogService ciArtifactRepository repository.CiArtifactRepository ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository cdWorkflowRepository pipelineConfig.CdWorkflowRepository @@ -97,10 +81,7 @@ type CdHandlerImpl struct { resourceGroupService resourceGroup2.ResourceGroupService imageTaggingService imageTagging.ImageTaggingService k8sUtil *k8s.K8sServiceImpl - workflowService WorkflowService config *types.CdConfig - clusterService cluster.ClusterService - blobConfigStorageService BlobStorageConfigService customTagService CustomTagService deploymentConfigService common2.DeploymentConfigService workflowStageStatusService workflowStatus.WorkFlowStageStatusService @@ -108,15 +89,14 @@ type CdHandlerImpl struct { } func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, - cdWorkflowRepository pipelineConfig.CdWorkflowRepository, ciLogService CiLogService, + cdWorkflowRepository pipelineConfig.CdWorkflowRepository, ciArtifactRepository repository.CiArtifactRepository, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, pipelineRepository pipelineConfig.PipelineRepository, envRepository repository3.EnvironmentRepository, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, enforcerUtil rbac.EnforcerUtil, resourceGroupService resourceGroup2.ResourceGroupService, imageTaggingService imageTagging.ImageTaggingService, k8sUtil *k8s.K8sServiceImpl, - workflowService WorkflowService, clusterService cluster.ClusterService, - blobConfigStorageService BlobStorageConfigService, customTagService CustomTagService, + customTagService CustomTagService, deploymentConfigService common2.DeploymentConfigService, workflowStageStatusService workflowStatus.WorkFlowStageStatusService, cdWorkflowRunnerService cd.CdWorkflowRunnerService, @@ -124,7 +104,6 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, cdh := &CdHandlerImpl{ Logger: Logger, userService: userService, - ciLogService: ciLogService, cdWorkflowRepository: cdWorkflowRepository, ciArtifactRepository: ciArtifactRepository, ciPipelineMaterialRepository: ciPipelineMaterialRepository, @@ -135,9 +114,6 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, resourceGroupService: resourceGroupService, imageTaggingService: imageTaggingService, k8sUtil: k8sUtil, - workflowService: workflowService, - clusterService: clusterService, - blobConfigStorageService: blobConfigStorageService, customTagService: customTagService, deploymentConfigService: deploymentConfigService, workflowStageStatusService: workflowStageStatusService, @@ -151,121 +127,6 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, return cdh } -func (impl *CdHandlerImpl) CancelStage(workflowRunnerId int, forceAbort bool, userId int32) (int, error) { - workflowRunner, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(workflowRunnerId) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return 0, err - } - pipeline, err := impl.pipelineRepository.FindById(workflowRunner.CdWorkflow.PipelineId) - if err != nil { - impl.Logger.Errorw("error while fetching cd pipeline", "err", err) - return 0, err - } - - env, err := impl.envRepository.FindById(pipeline.EnvironmentId) - if err != nil { - impl.Logger.Errorw("could not fetch stage env", "err", err) - return 0, err - } - - var clusterBean bean3.ClusterBean - if env != nil && env.Cluster != nil { - clusterBean = adapter.GetClusterBean(*env.Cluster) - } - clusterConfig := clusterBean.GetClusterConfig() - var isExtCluster bool - if workflowRunner.WorkflowType == types.PRE { - isExtCluster = pipeline.RunPreStageInEnv - } else if workflowRunner.WorkflowType == types.POST { - isExtCluster = pipeline.RunPostStageInEnv - } - var restConfig *rest.Config - if isExtCluster { - restConfig, err = impl.k8sUtil.GetRestConfigByCluster(clusterConfig) - if err != nil { - impl.Logger.Errorw("error in getting rest config by cluster id", "err", err) - return 0, err - } - } - // Terminate workflow - cancelWfDtoRequest := &types.CancelWfRequestDto{ - ExecutorType: workflowRunner.ExecutorType, - WorkflowName: workflowRunner.Name, - Namespace: workflowRunner.Namespace, - RestConfig: restConfig, - IsExt: isExtCluster, - Environment: nil, - } - err = impl.workflowService.TerminateWorkflow(cancelWfDtoRequest) - if err != nil && forceAbort { - impl.Logger.Errorw("error in terminating workflow, with force abort flag as true", "workflowName", workflowRunner.Name, "err", err) - cancelWfDtoRequest.WorkflowGenerateName = fmt.Sprintf("%d-%s", workflowRunnerId, workflowRunner.Name) - err1 := impl.workflowService.TerminateDanglingWorkflows(cancelWfDtoRequest) - if err1 != nil { - impl.Logger.Errorw("error in terminating dangling workflows", "cancelWfDtoRequest", cancelWfDtoRequest, "err", err) - // ignoring error here in case of force abort, confirmed from product - } - } else if err != nil && strings.Contains(err.Error(), "cannot find workflow") { - return 0, &util.ApiError{Code: "200", HttpStatusCode: http.StatusBadRequest, UserMessage: err.Error()} - } else if err != nil { - impl.Logger.Error("cannot terminate wf runner", "err", err) - return 0, err - } - if forceAbort { - err = impl.handleForceAbortCaseForCdStage(workflowRunner, forceAbort) - if err != nil { - impl.Logger.Errorw("error in handleForceAbortCaseForCdStage", "forceAbortFlag", forceAbort, "workflowRunner", workflowRunner, "err", err) - return 0, err - } - return workflowRunner.Id, nil - } - if len(workflowRunner.ImagePathReservationIds) > 0 { - err := impl.customTagService.DeactivateImagePathReservationByImageIds(workflowRunner.ImagePathReservationIds) - if err != nil { - impl.Logger.Errorw("error in deactivating image path reservation ids", "err", err) - return 0, err - } - } - workflowRunner.Status = cdWorkflow2.WorkflowCancel - workflowRunner.UpdatedOn = time.Now() - workflowRunner.UpdatedBy = userId - err = impl.cdWorkflowRunnerService.UpdateCdWorkflowRunnerWithStage(workflowRunner) - if err != nil { - impl.Logger.Error("cannot update deleted workflow runner status, but wf deleted", "err", err) - return 0, err - } - return workflowRunner.Id, nil -} - -func (impl *CdHandlerImpl) updateWorkflowRunnerForForceAbort(workflowRunner *pipelineConfig.CdWorkflowRunner) error { - workflowRunner.Status = cdWorkflow2.WorkflowCancel - workflowRunner.PodStatus = string(bean2.Failed) - workflowRunner.Message = constants.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE - err := impl.cdWorkflowRunnerService.UpdateCdWorkflowRunnerWithStage(workflowRunner) - if err != nil { - impl.Logger.Errorw("error in updating workflow status in cd workflow runner in force abort case", "err", err) - return err - } - return nil -} - -func (impl *CdHandlerImpl) handleForceAbortCaseForCdStage(workflowRunner *pipelineConfig.CdWorkflowRunner, forceAbort bool) error { - isWorkflowInNonTerminalStage := workflowRunner.Status == string(v1alpha1.NodePending) || workflowRunner.Status == string(v1alpha1.NodeRunning) - if !isWorkflowInNonTerminalStage { - if forceAbort { - return impl.updateWorkflowRunnerForForceAbort(workflowRunner) - } else { - return &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "cannot cancel stage, stage not in progress"} - } - } - //this arises when someone deletes the workflow in resource browser and wants to force abort a cd stage(pre/post) - if workflowRunner.Status == string(v1alpha1.NodeRunning) && forceAbort { - return impl.updateWorkflowRunnerForForceAbort(workflowRunner) - } - return nil -} - func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, string, bool, error) { wfStatusRs := impl.extractWorkfowStatus(workflowStatus) workflowName, status, podStatus, message, podName := wfStatusRs.WorkflowName, wfStatusRs.Status, wfStatusRs.PodStatus, wfStatusRs.Message, wfStatusRs.PodName @@ -516,122 +377,6 @@ func (impl *CdHandlerImpl) GetCdBuildHistory(appId int, environmentId int, pipel return cdWorkflowArtifact, nil } -func (impl *CdHandlerImpl) GetRunningWorkflowLogs(environmentId int, pipelineId int, wfrId int) (*bufio.Reader, func() error, error) { - cdWorkflow, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(wfrId) - if err != nil { - impl.Logger.Errorw("error on fetch wf runner", "err", err) - return nil, nil, err - } - - env, err := impl.envRepository.FindById(environmentId) - if err != nil { - impl.Logger.Errorw("could not fetch stage env", "err", err) - return nil, nil, err - } - - pipeline, err := impl.pipelineRepository.FindById(cdWorkflow.CdWorkflow.PipelineId) - if err != nil { - impl.Logger.Errorw("error while fetching cd pipeline", "err", err) - return nil, nil, err - } - var clusterBean bean3.ClusterBean - if env != nil && env.Cluster != nil { - clusterBean = adapter.GetClusterBean(*env.Cluster) - } - clusterConfig := clusterBean.GetClusterConfig() - var isExtCluster bool - if cdWorkflow.WorkflowType == types.PRE { - isExtCluster = pipeline.RunPreStageInEnv - } else if cdWorkflow.WorkflowType == types.POST { - isExtCluster = pipeline.RunPostStageInEnv - } - return impl.getWorkflowLogs(pipelineId, cdWorkflow, clusterConfig, isExtCluster) -} - -func (impl *CdHandlerImpl) getWorkflowLogs(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, runStageInEnv bool) (*bufio.Reader, func() error, error) { - cdLogRequest := types.BuildLogRequest{ - PodName: cdWorkflow.PodName, - Namespace: cdWorkflow.Namespace, - } - - logStream, cleanUp, err := impl.ciLogService.FetchRunningWorkflowLogs(cdLogRequest, clusterConfig, runStageInEnv) - if logStream == nil || err != nil { - if !cdWorkflow.BlobStorageEnabled { - return nil, nil, errors.New("logs-not-stored-in-repository") - } else if string(v1alpha1.NodeSucceeded) == cdWorkflow.Status || string(v1alpha1.NodeError) == cdWorkflow.Status || string(v1alpha1.NodeFailed) == cdWorkflow.Status || cdWorkflow.Status == cdWorkflow2.WorkflowCancel { - impl.Logger.Debugw("pod is not live", "podName", cdWorkflow.PodName, "err", err) - return impl.getLogsFromRepository(pipelineId, cdWorkflow, clusterConfig, runStageInEnv) - } - if err != nil { - impl.Logger.Errorw("err on fetch workflow logs", "err", err) - return nil, nil, err - } else if logStream == nil { - return nil, cleanUp, fmt.Errorf("no logs found for pod %s", cdWorkflow.PodName) - } - } - logReader := bufio.NewReader(logStream) - return logReader, cleanUp, err -} - -func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { - impl.Logger.Debug("getting historic logs", "pipelineId", pipelineId) - - cdConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() // TODO -fixme - cdConfigCdCacheRegion := impl.config.GetDefaultCdLogsBucketRegion() - - cdLogRequest := types.BuildLogRequest{ - PipelineId: cdWorkflow.CdWorkflow.PipelineId, - WorkflowId: cdWorkflow.Id, - PodName: cdWorkflow.PodName, - LogsFilePath: cdWorkflow.LogLocation, // impl.ciCdConfig.CiDefaultBuildLogsKeyPrefix + "/" + cdWorkflow.Name + "/main.log", //TODO - fixme - CloudProvider: impl.config.CloudProvider, - AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ - Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, - AccountName: impl.config.AzureAccountName, - BlobContainerName: impl.config.AzureBlobContainerCiLog, - AccountKey: impl.config.AzureAccountKey, - }, - AwsS3BaseConfig: &blob_storage.AwsS3BaseConfig{ - AccessKey: impl.config.BlobStorageS3AccessKey, - Passkey: impl.config.BlobStorageS3SecretKey, - EndpointUrl: impl.config.BlobStorageS3Endpoint, - IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - BucketName: cdConfigLogsBucket, - Region: cdConfigCdCacheRegion, - VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, - }, - GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ - BucketName: cdConfigLogsBucket, - CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - }, - } - useExternalBlobStorage := isExternalBlobStorageEnabled(isExt, impl.config.UseBlobStorageConfigInCdWorkflow) - if useExternalBlobStorage { - // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - // from them else return. - cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, cdWorkflow.Namespace) - if err != nil { - impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) - return nil, nil, err - } - rq := &cdLogRequest - rq.SetBuildLogRequest(cmConfig, secretConfig) - } - - impl.Logger.Debugw("s3 log req ", "pipelineId", pipelineId, "runnerId", cdWorkflow.Id) - oldLogsStream, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, cdLogRequest) - if err != nil { - impl.Logger.Errorw("err", err) - return nil, nil, err - } - logReader := bufio.NewReader(oldLogsStream) - return logReader, cleanUp, err -} -func isExternalBlobStorageEnabled(isExternalRun bool, useBlobStorageConfigInCdWorkflow bool) bool { - // TODO impl.config.UseBlobStorageConfigInCdWorkflow fetches the live status, we need to check from db as well, we should put useExternalBlobStorage in db - return isExternalRun && !useBlobStorageConfigInCdWorkflow -} - func (impl *CdHandlerImpl) FetchCdWorkflowDetails(appId int, environmentId int, pipelineId int, buildId int) (types.WorkflowResponse, error) { workflowR, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(buildId) if err != nil && err != pg.ErrNoRows { @@ -736,89 +481,6 @@ func (impl *CdHandlerImpl) FetchCdWorkflowDetails(appId int, environmentId int, } -func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(buildId int) (*os.File, error) { - wfr, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(buildId) - if err != nil { - impl.Logger.Errorw("unable to fetch ciWorkflow", "err", err) - return nil, err - } - useExternalBlobStorage := isExternalBlobStorageEnabled(wfr.IsExternalRun(), impl.config.UseBlobStorageConfigInCdWorkflow) - if !wfr.BlobStorageEnabled { - return nil, errors.New("logs-not-stored-in-repository") - } - - cdConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() - cdConfigCdCacheRegion := impl.config.GetDefaultCdLogsBucketRegion() - - item := strconv.Itoa(wfr.Id) - awsS3BaseConfig := &blob_storage.AwsS3BaseConfig{ - AccessKey: impl.config.BlobStorageS3AccessKey, - Passkey: impl.config.BlobStorageS3SecretKey, - EndpointUrl: impl.config.BlobStorageS3Endpoint, - IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - BucketName: cdConfigLogsBucket, - Region: cdConfigCdCacheRegion, - VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, - } - azureBlobBaseConfig := &blob_storage.AzureBlobBaseConfig{ - Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, - AccountKey: impl.config.AzureAccountKey, - AccountName: impl.config.AzureAccountName, - BlobContainerName: impl.config.AzureBlobContainerCiLog, - } - gcpBlobBaseConfig := &blob_storage.GcpBlobBaseConfig{ - BucketName: cdConfigLogsBucket, - CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - } - cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() - key := fmt.Sprintf(cdArtifactLocationFormat, wfr.CdWorkflow.Id, wfr.Id) - if len(wfr.CdArtifactLocation) != 0 && util2.IsValidUrlSubPath(wfr.CdArtifactLocation) { - key = wfr.CdArtifactLocation - } else if util2.IsValidUrlSubPath(key) { - impl.cdWorkflowRepository.MigrateCdArtifactLocation(wfr.Id, key) - } - baseLogLocationPathConfig := impl.config.BaseLogLocationPath - blobStorageService := blob_storage.NewBlobStorageServiceImpl(nil) - destinationKey := filepath.Clean(filepath.Join(baseLogLocationPathConfig, item)) - request := &blob_storage.BlobStorageRequest{ - StorageType: impl.config.CloudProvider, - SourceKey: key, - DestinationKey: destinationKey, - AzureBlobBaseConfig: azureBlobBaseConfig, - AwsS3BaseConfig: awsS3BaseConfig, - GcpBlobBaseConfig: gcpBlobBaseConfig, - } - if useExternalBlobStorage { - clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(wfr.CdWorkflow.Pipeline.Environment.ClusterId) - if err != nil { - impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig", "err", err, "clusterId", wfr.CdWorkflow.Pipeline.Environment.ClusterId) - return nil, err - } - // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - // from them else return. - cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, wfr.Namespace) - if err != nil { - impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) - return nil, err - } - request = updateRequestWithExtClusterCmAndSecret(request, cmConfig, secretConfig) - } - _, numBytes, err := blobStorageService.Get(request) - if err != nil { - impl.Logger.Errorw("error occurred while downloading file", "request", request, "error", err) - return nil, errors.New("failed to download resource") - } - - file, err := os.Open(destinationKey) - if err != nil { - impl.Logger.Errorw("unable to open file", "file", item, "err", err) - return nil, errors.New("unable to open file") - } - - impl.Logger.Infow("Downloaded ", "name", file.Name(), "bytes", numBytes) - return file, nil -} - func (impl *CdHandlerImpl) converterWFR(wfr pipelineConfig.CdWorkflowRunner) pipelineBean.CdWorkflowWithArtifact { workflow := pipelineBean.CdWorkflowWithArtifact{} if wfr.Id > 0 { diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 20d18c93eb..70cb749398 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -17,7 +17,6 @@ package pipeline import ( - "bufio" "context" "errors" "fmt" @@ -26,43 +25,29 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/pkg/build/artifacts/imageTagging" buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" - "github.com/devtron-labs/devtron/pkg/cluster/adapter" - clusterBean "github.com/devtron-labs/devtron/pkg/cluster/bean" - "github.com/devtron-labs/devtron/pkg/cluster/environment" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/pipeline/constants" - util3 "github.com/devtron-labs/devtron/pkg/pipeline/util" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" - "io/ioutil" - "net/http" - "os" - "path/filepath" "regexp" "strconv" "strings" "time" - blob_storage "github.com/devtron-labs/common-lib/blob-storage" - "github.com/devtron-labs/common-lib/utils/k8s" + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" bean2 "github.com/devtron-labs/devtron/api/bean" + client "github.com/devtron-labs/devtron/client/events" "github.com/devtron-labs/devtron/client/gitSensor" + "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/auth/user" - "github.com/devtron-labs/devtron/pkg/cluster" k8sPkg "github.com/devtron-labs/devtron/pkg/k8s" pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/util/rbac" - "k8s.io/client-go/rest" - - "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" - client "github.com/devtron-labs/devtron/client/events" - "github.com/devtron-labs/devtron/internal/sql/repository" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/internal/util" - "github.com/devtron-labs/devtron/pkg/bean" "github.com/go-pg/pg" "go.uber.org/zap" ) @@ -75,13 +60,8 @@ type CiHandler interface { FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId int, gitMaterialId int, showAll bool) ([]buildBean.CiPipelineMaterialResponse, error) FetchWorkflowDetails(appId int, pipelineId int, buildId int) (types.WorkflowResponse, error) FetchArtifactsForCiJob(buildId int) (*types.ArtifactsForCiJob, error) - CancelBuild(workflowId int, forceAbort bool) (int, error) - - GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, func() error, error) - GetHistoricBuildLogs(workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) GetBuildHistory(pipelineId int, appId int, offset int, size int) ([]types.WorkflowResponse, error) - DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, error) FetchCiStatusForTriggerView(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) @@ -99,15 +79,12 @@ type CiHandlerImpl struct { ciService CiService gitSensorClient gitSensor.Client ciWorkflowRepository pipelineConfig.CiWorkflowRepository - workflowService WorkflowService - ciLogService CiLogService ciArtifactRepository repository.CiArtifactRepository userService user.UserService eventClient client.EventClient eventFactory client.EventFactory ciPipelineRepository pipelineConfig.CiPipelineRepository appListingRepository repository.AppListingRepository - K8sUtil *k8s.K8sServiceImpl cdPipelineRepository pipelineConfig.PipelineRepository enforcerUtil rbac.EnforcerUtil resourceGroupService resourceGroup.ResourceGroupService @@ -117,18 +94,14 @@ type CiHandlerImpl struct { appWorkflowRepository appWorkflow.AppWorkflowRepository config *types.CiConfig k8sCommonService k8sPkg.K8sCommonService - clusterService cluster.ClusterService - blobConfigStorageService BlobStorageConfigService - envService environment.EnvironmentService workFlowStageStatusService workflowStatus.WorkFlowStageStatusService } -func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, gitSensorClient gitSensor.Client, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, workflowService WorkflowService, - ciLogService CiLogService, ciArtifactRepository repository.CiArtifactRepository, userService user.UserService, eventClient client.EventClient, eventFactory client.EventFactory, ciPipelineRepository pipelineConfig.CiPipelineRepository, +func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, gitSensorClient gitSensor.Client, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, + ciArtifactRepository repository.CiArtifactRepository, userService user.UserService, eventClient client.EventClient, eventFactory client.EventFactory, ciPipelineRepository pipelineConfig.CiPipelineRepository, appListingRepository repository.AppListingRepository, cdPipelineRepository pipelineConfig.PipelineRepository, enforcerUtil rbac.EnforcerUtil, resourceGroupService resourceGroup.ResourceGroupService, envRepository repository2.EnvironmentRepository, - imageTaggingService imageTagging.ImageTaggingService, k8sCommonService k8sPkg.K8sCommonService, clusterService cluster.ClusterService, blobConfigStorageService BlobStorageConfigService, appWorkflowRepository appWorkflow.AppWorkflowRepository, customTagService CustomTagService, - envService environment.EnvironmentService, workFlowStageStatusService workflowStatus.WorkFlowStageStatusService, - K8sUtil *k8s.K8sServiceImpl, + imageTaggingService imageTagging.ImageTaggingService, k8sCommonService k8sPkg.K8sCommonService, appWorkflowRepository appWorkflow.AppWorkflowRepository, customTagService CustomTagService, + workFlowStageStatusService workflowStatus.WorkFlowStageStatusService, ) *CiHandlerImpl { cih := &CiHandlerImpl{ Logger: Logger, @@ -136,15 +109,12 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline ciPipelineMaterialRepository: ciPipelineMaterialRepository, gitSensorClient: gitSensorClient, ciWorkflowRepository: ciWorkflowRepository, - workflowService: workflowService, - ciLogService: ciLogService, ciArtifactRepository: ciArtifactRepository, userService: userService, eventClient: eventClient, eventFactory: eventFactory, ciPipelineRepository: ciPipelineRepository, appListingRepository: appListingRepository, - K8sUtil: K8sUtil, cdPipelineRepository: cdPipelineRepository, enforcerUtil: enforcerUtil, resourceGroupService: resourceGroupService, @@ -153,9 +123,6 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline customTagService: customTagService, appWorkflowRepository: appWorkflowRepository, k8sCommonService: k8sCommonService, - clusterService: clusterService, - blobConfigStorageService: blobConfigStorageService, - envService: envService, workFlowStageStatusService: workFlowStageStatusService, } config, err := types.GetCiConfig() @@ -443,129 +410,6 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int return ciWorkLowResponses, nil } -func (impl *CiHandlerImpl) CancelBuild(workflowId int, forceAbort bool) (int, error) { - workflow, err := impl.ciWorkflowRepository.FindById(workflowId) - if err != nil { - impl.Logger.Errorw("error in finding ci-workflow by workflow id", "ciWorkflowId", workflowId, "err", err) - return 0, err - } - isExt := workflow.Namespace != constants.DefaultCiWorkflowNamespace - var env *repository2.Environment - var restConfig *rest.Config - if isExt { - restConfig, err = impl.getRestConfig(workflow) - if err != nil { - return 0, err - } - } - // Terminate workflow - cancelWfDtoRequest := &types.CancelWfRequestDto{ - ExecutorType: workflow.ExecutorType, - WorkflowName: workflow.Name, - Namespace: workflow.Namespace, - RestConfig: restConfig, - IsExt: isExt, - Environment: env, - } - // Terminate workflow - err = impl.workflowService.TerminateWorkflow(cancelWfDtoRequest) - if err != nil && forceAbort { - impl.Logger.Errorw("error in terminating workflow, with force abort flag flag as true", "workflowName", workflow.Name, "err", err) - - cancelWfDtoRequest.WorkflowGenerateName = fmt.Sprintf("%d-%s", workflowId, workflow.Name) - err1 := impl.workflowService.TerminateDanglingWorkflows(cancelWfDtoRequest) - if err1 != nil { - impl.Logger.Errorw("error in terminating dangling workflows", "cancelWfDtoRequest", cancelWfDtoRequest, "err", err) - // ignoring error here in case of force abort, confirmed from product - } - } else if err != nil && strings.Contains(err.Error(), "cannot find workflow") { - return 0, &util.ApiError{Code: "200", HttpStatusCode: http.StatusBadRequest, UserMessage: err.Error()} - } else if err != nil { - impl.Logger.Errorw("cannot terminate wf", "err", err) - return 0, err - } - if forceAbort { - err = impl.handleForceAbortCaseForCi(workflow, forceAbort) - if err != nil { - impl.Logger.Errorw("error in handleForceAbortCaseForCi", "forceAbortFlag", forceAbort, "workflow", workflow, "err", err) - return 0, err - } - return workflow.Id, nil - } - - workflow.Status = cdWorkflow.WorkflowCancel - if workflow.ExecutorType == cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM { - workflow.PodStatus = "Failed" - workflow.Message = constants.TERMINATE_MESSAGE - } - err = impl.ciService.UpdateCiWorkflowWithStage(workflow) - if err != nil { - impl.Logger.Errorw("cannot update deleted workflow status, but wf deleted", "err", err) - return 0, err - } - imagePathReservationId := workflow.ImagePathReservationId - err = impl.customTagService.DeactivateImagePathReservation(imagePathReservationId) - if err != nil { - impl.Logger.Errorw("error in marking image tag unreserved", "err", err) - return 0, err - } - imagePathReservationIds := workflow.ImagePathReservationIds - if len(imagePathReservationIds) > 0 { - err = impl.customTagService.DeactivateImagePathReservationByImageIds(imagePathReservationIds) - if err != nil { - impl.Logger.Errorw("error in marking image tag unreserved", "err", err) - return 0, err - } - } - return workflow.Id, nil -} - -func (impl *CiHandlerImpl) handleForceAbortCaseForCi(workflow *pipelineConfig.CiWorkflow, forceAbort bool) error { - isWorkflowInNonTerminalStage := workflow.Status == string(v1alpha1.NodePending) || workflow.Status == string(v1alpha1.NodeRunning) - if !isWorkflowInNonTerminalStage { - if forceAbort { - return impl.updateWorkflowForForceAbort(workflow) - } else { - return &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "cannot cancel build, build not in progress"} - } - } - //this arises when someone deletes the workflow in resource browser and wants to force abort a ci - if workflow.Status == string(v1alpha1.NodeRunning) && forceAbort { - return impl.updateWorkflowForForceAbort(workflow) - } - return nil -} - -func (impl *CiHandlerImpl) updateWorkflowForForceAbort(workflow *pipelineConfig.CiWorkflow) error { - workflow.Status = cdWorkflow.WorkflowCancel - workflow.PodStatus = string(bean.Failed) - workflow.Message = constants.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE - err := impl.ciService.UpdateCiWorkflowWithStage(workflow) - if err != nil { - impl.Logger.Errorw("error in updating workflow status", "err", err) - return err - } - return nil -} - -func (impl *CiHandlerImpl) getRestConfig(workflow *pipelineConfig.CiWorkflow) (*rest.Config, error) { - env, err := impl.envRepository.FindById(workflow.EnvironmentId) - if err != nil { - impl.Logger.Errorw("could not fetch stage env", "err", err) - return nil, err - } - - clusterBean := adapter.GetClusterBean(*env.Cluster) - - clusterConfig := clusterBean.GetClusterConfig() - restConfig, err := impl.K8sUtil.GetRestConfigByCluster(clusterConfig) - if err != nil { - impl.Logger.Errorw("error in getting rest config by cluster id", "err", err) - return nil, err - } - return restConfig, nil -} - func (impl *CiHandlerImpl) FetchWorkflowDetails(appId int, pipelineId int, buildId int) (types.WorkflowResponse, error) { workflow, err := impl.ciWorkflowRepository.FindById(buildId) if err != nil { @@ -670,276 +514,6 @@ func (impl *CiHandlerImpl) FetchArtifactsForCiJob(buildId int) (*types.Artifacts } return artifactsResponse, nil } -func (impl *CiHandlerImpl) GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, func() error, error) { - ciWorkflow, err := impl.ciWorkflowRepository.FindById(workflowId) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return nil, nil, err - } - return impl.getWorkflowLogs(ciWorkflow) -} - -func (impl *CiHandlerImpl) getWorkflowLogs(ciWorkflow *pipelineConfig.CiWorkflow) (*bufio.Reader, func() error, error) { - if string(v1alpha1.NodePending) == ciWorkflow.PodStatus { - return bufio.NewReader(strings.NewReader("")), func() error { return nil }, nil - } - ciLogRequest := types.BuildLogRequest{ - PodName: ciWorkflow.PodName, - Namespace: ciWorkflow.Namespace, - } - isExt := false - clusterConfig := &k8s.ClusterConfig{} - if ciWorkflow.EnvironmentId != 0 { - env, err := impl.envRepository.FindById(ciWorkflow.EnvironmentId) - if err != nil { - return nil, nil, err - } - var clusterBean clusterBean.ClusterBean - if env != nil && env.Cluster != nil { - clusterBean = adapter.GetClusterBean(*env.Cluster) - } - clusterConfig = clusterBean.GetClusterConfig() - isExt = true - } - - logStream, cleanUp, err := impl.ciLogService.FetchRunningWorkflowLogs(ciLogRequest, clusterConfig, isExt) - if logStream == nil || err != nil { - if !ciWorkflow.BlobStorageEnabled { - return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "logs-not-stored-in-repository"} - } else if string(v1alpha1.NodeSucceeded) == ciWorkflow.Status || string(v1alpha1.NodeError) == ciWorkflow.Status || string(v1alpha1.NodeFailed) == ciWorkflow.Status || ciWorkflow.Status == cdWorkflow.WorkflowCancel { - impl.Logger.Debugw("pod is not live", "podName", ciWorkflow.PodName, "err", err) - return impl.getLogsFromRepository(ciWorkflow, clusterConfig, isExt) - } - if err != nil { - impl.Logger.Errorw("err on fetch workflow logs", "err", err) - return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: err.Error()} - } else if logStream == nil { - return nil, cleanUp, fmt.Errorf("no logs found for pod %s", ciWorkflow.PodName) - } - } - logReader := bufio.NewReader(logStream) - return logReader, cleanUp, err -} - -func (impl *CiHandlerImpl) getLogsFromRepository(ciWorkflow *pipelineConfig.CiWorkflow, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { - impl.Logger.Debug("getting historic logs", "ciWorkflowId", ciWorkflow.Id) - ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() - ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion - logsFilePath := impl.config.GetDefaultBuildLogsKeyPrefix() + "/" + ciWorkflow.Name + "/main.log" // this is for backward compatibilty - if strings.Contains(ciWorkflow.LogLocation, "main.log") { - logsFilePath = ciWorkflow.LogLocation - } - ciLogRequest := types.BuildLogRequest{ - PipelineId: ciWorkflow.CiPipelineId, - WorkflowId: ciWorkflow.Id, - PodName: ciWorkflow.PodName, - LogsFilePath: logsFilePath, - CloudProvider: impl.config.CloudProvider, - AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ - Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, - AccountName: impl.config.AzureAccountName, - BlobContainerName: impl.config.AzureBlobContainerCiLog, - AccountKey: impl.config.AzureAccountKey, - }, - AwsS3BaseConfig: &blob_storage.AwsS3BaseConfig{ - AccessKey: impl.config.BlobStorageS3AccessKey, - Passkey: impl.config.BlobStorageS3SecretKey, - EndpointUrl: impl.config.BlobStorageS3Endpoint, - IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - BucketName: ciConfigLogsBucket, - Region: ciConfigCiCacheRegion, - VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, - }, - GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ - BucketName: ciConfigLogsBucket, - CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - }, - } - useExternalBlobStorage := isExternalBlobStorageEnabled(isExt, impl.config.UseBlobStorageConfigInCiWorkflow) - if useExternalBlobStorage { - // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - // from them else return. - cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) - if err != nil { - impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) - return nil, nil, err - } - rq := &ciLogRequest - rq.SetBuildLogRequest(cmConfig, secretConfig) - } - oldLogsStream, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, ciLogRequest) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return nil, nil, err - } - logReader := bufio.NewReader(oldLogsStream) - return logReader, cleanUp, err -} - -func (impl *CiHandlerImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) { - ciWorkflow, err := impl.ciWorkflowRepository.FindById(buildId) - if err != nil { - impl.Logger.Errorw("unable to fetch ciWorkflow", "err", err) - return nil, err - } - useExternalBlobStorage := isExternalBlobStorageEnabled(ciWorkflow.IsExternalRunInJobType(), impl.config.UseBlobStorageConfigInCiWorkflow) - if !ciWorkflow.BlobStorageEnabled { - return nil, errors.New("logs-not-stored-in-repository") - } - - if ciWorkflow.CiPipelineId != pipelineId { - impl.Logger.Error("invalid request, wf not in pipeline") - return nil, errors.New("invalid request, wf not in pipeline") - } - - ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() - item := strconv.Itoa(ciWorkflow.Id) - ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion - azureBlobConfig := &blob_storage.AzureBlobBaseConfig{ - Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, - AccountName: impl.config.AzureAccountName, - BlobContainerName: impl.config.AzureBlobContainerCiLog, - AccountKey: impl.config.AzureAccountKey, - } - awsS3BaseConfig := &blob_storage.AwsS3BaseConfig{ - AccessKey: impl.config.BlobStorageS3AccessKey, - Passkey: impl.config.BlobStorageS3SecretKey, - EndpointUrl: impl.config.BlobStorageS3Endpoint, - IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - BucketName: ciConfigLogsBucket, - Region: ciConfigCiCacheRegion, - VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, - } - gcpBlobBaseConfig := &blob_storage.GcpBlobBaseConfig{ - BucketName: ciConfigLogsBucket, - CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - } - - ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() - key := fmt.Sprintf(ciArtifactLocationFormat, ciWorkflow.Id, ciWorkflow.Id) - if len(ciWorkflow.CiArtifactLocation) != 0 && util3.IsValidUrlSubPath(ciWorkflow.CiArtifactLocation) { - key = ciWorkflow.CiArtifactLocation - } else if util3.IsValidUrlSubPath(key) { - impl.ciWorkflowRepository.MigrateCiArtifactLocation(ciWorkflow.Id, key) - } - baseLogLocationPathConfig := impl.config.BaseLogLocationPath - blobStorageService := blob_storage.NewBlobStorageServiceImpl(nil) - destinationKey := filepath.Clean(filepath.Join(baseLogLocationPathConfig, item)) - request := &blob_storage.BlobStorageRequest{ - StorageType: impl.config.CloudProvider, - SourceKey: key, - DestinationKey: destinationKey, - AzureBlobBaseConfig: azureBlobConfig, - AwsS3BaseConfig: awsS3BaseConfig, - GcpBlobBaseConfig: gcpBlobBaseConfig, - } - if useExternalBlobStorage { - envBean, err := impl.envService.FindById(ciWorkflow.EnvironmentId) - if err != nil { - impl.Logger.Errorw("error in getting envBean by envId", "err", err, "envId", ciWorkflow.EnvironmentId) - return nil, err - } - clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(envBean.ClusterId) - if err != nil { - impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig by clusterId", "err", err, "clusterId", envBean.ClusterId) - return nil, err - } - // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - // from them else return. - cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) - if err != nil { - impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) - return nil, err - } - request = updateRequestWithExtClusterCmAndSecret(request, cmConfig, secretConfig) - } - _, numBytes, err := blobStorageService.Get(request) - if err != nil { - impl.Logger.Errorw("error occurred while downloading file", "request", request, "error", err) - return nil, errors.New("failed to download resource") - } - - file, err := os.Open(destinationKey) - if err != nil { - impl.Logger.Errorw("unable to open file", "file", item, "err", err) - return nil, errors.New("unable to open file") - } - - impl.Logger.Infow("Downloaded ", "filename", file.Name(), "bytes", numBytes) - return file, nil -} - -func (impl *CiHandlerImpl) GetHistoricBuildLogs(workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) { - var err error - if ciWorkflow == nil { - ciWorkflow, err = impl.ciWorkflowRepository.FindById(workflowId) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return nil, err - } - } - ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() - ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion - ciLogRequest := types.BuildLogRequest{ - PipelineId: ciWorkflow.CiPipelineId, - WorkflowId: ciWorkflow.Id, - PodName: ciWorkflow.PodName, - LogsFilePath: ciWorkflow.LogLocation, - CloudProvider: impl.config.CloudProvider, - AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ - Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, - AccountName: impl.config.AzureAccountName, - BlobContainerName: impl.config.AzureBlobContainerCiLog, - AccountKey: impl.config.AzureAccountKey, - }, - AwsS3BaseConfig: &blob_storage.AwsS3BaseConfig{ - AccessKey: impl.config.BlobStorageS3AccessKey, - Passkey: impl.config.BlobStorageS3SecretKey, - EndpointUrl: impl.config.BlobStorageS3Endpoint, - IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - BucketName: ciConfigLogsBucket, - Region: ciConfigCiCacheRegion, - VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, - }, - GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ - BucketName: ciConfigLogsBucket, - CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - }, - } - useExternalBlobStorage := isExternalBlobStorageEnabled(ciWorkflow.IsExternalRunInJobType(), impl.config.UseBlobStorageConfigInCiWorkflow) - if useExternalBlobStorage { - envBean, err := impl.envService.FindById(ciWorkflow.EnvironmentId) - if err != nil { - impl.Logger.Errorw("error in getting envBean by envId", "err", err, "envId", ciWorkflow.EnvironmentId) - return nil, err - } - clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(envBean.ClusterId) - if err != nil { - impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig by clusterId", "err", err, "clusterId", envBean.ClusterId) - return nil, err - } - // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - // from them else return. - cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) - if err != nil { - impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) - return nil, err - } - rq := &ciLogRequest - rq.SetBuildLogRequest(cmConfig, secretConfig) - } - logsFile, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, ciLogRequest) - logs, err := ioutil.ReadFile(logsFile.Name()) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return map[string]string{}, err - } - logStr := string(logs) - resp := make(map[string]string) - resp["logs"] = logStr - defer cleanUp() - return resp, err -} func ExtractWorkflowStatus(workflowStatus v1alpha1.WorkflowStatus) (string, string, string, string, string, string) { workflowName := "" diff --git a/pkg/pipeline/PipelineStageService.go b/pkg/pipeline/PipelineStageService.go index 2ff4123f0f..2a538b7282 100644 --- a/pkg/pipeline/PipelineStageService.go +++ b/pkg/pipeline/PipelineStageService.go @@ -74,6 +74,11 @@ func NewPipelineStageService(logger *zap.SugaredLogger, } } +const ( + preCdStage = "preCD" + postCdStage = "postCD" +) + type PipelineStageServiceImpl struct { logger *zap.SugaredLogger pipelineStageRepository repository.PipelineStageRepository diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index 2a70ebe51e..affa658f1d 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -50,6 +50,7 @@ import ( triggerBean "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/userDeploymentRequest/service" eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" + "github.com/devtron-labs/devtron/pkg/executor" k8sPkg "github.com/devtron-labs/devtron/pkg/k8s" "github.com/devtron-labs/devtron/pkg/pipeline" constants2 "github.com/devtron-labs/devtron/pkg/pipeline/constants" @@ -148,7 +149,7 @@ type WorkflowDagExecutorImpl struct { K8sUtil *k8s.K8sServiceImpl envRepository repository5.EnvironmentRepository k8sCommonService k8sPkg.K8sCommonService - workflowService pipeline.WorkflowService + workflowService executor.WorkflowService ciTriggerService trigger.Service } @@ -181,7 +182,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi K8sUtil *k8s.K8sServiceImpl, envRepository repository5.EnvironmentRepository, k8sCommonService k8sPkg.K8sCommonService, - workflowService pipeline.WorkflowService, + workflowService executor.WorkflowService, ciTriggerService trigger.Service, ) *WorkflowDagExecutorImpl { wde := &WorkflowDagExecutorImpl{logger: Logger, diff --git a/wire_gen.go b/wire_gen.go index a1365d92a7..4f51c1ac3c 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -202,6 +202,7 @@ import ( "github.com/devtron-labs/devtron/pkg/eventProcessor/celEvaluator" "github.com/devtron-labs/devtron/pkg/eventProcessor/in" "github.com/devtron-labs/devtron/pkg/eventProcessor/out" + "github.com/devtron-labs/devtron/pkg/executor" "github.com/devtron-labs/devtron/pkg/externalLink" "github.com/devtron-labs/devtron/pkg/fluxApplication" "github.com/devtron-labs/devtron/pkg/generateManifest" @@ -625,7 +626,7 @@ func InitializeApp() (*App, error) { } ciInfraGetter := ci.NewCiInfraGetter(sugaredLogger, infraConfigServiceImpl, infraConfigAuditServiceImpl) infraProviderImpl := infraProviders.NewInfraProviderImpl(sugaredLogger, infraGetter, ciInfraGetter) - workflowServiceImpl, err := pipeline.NewWorkflowServiceImpl(sugaredLogger, environmentRepositoryImpl, ciCdConfig, configReadServiceImpl, globalCMCSServiceImpl, argoWorkflowExecutorImpl, k8sServiceImpl, systemWorkflowExecutorImpl, k8sCommonServiceImpl, infraProviderImpl) + workflowServiceImpl, err := executor.NewWorkflowServiceImpl(sugaredLogger, environmentRepositoryImpl, ciCdConfig, configReadServiceImpl, globalCMCSServiceImpl, argoWorkflowExecutorImpl, k8sServiceImpl, systemWorkflowExecutorImpl, k8sCommonServiceImpl, infraProviderImpl) if err != nil { return nil, err } @@ -673,7 +674,12 @@ func InitializeApp() (*App, error) { ciCdPipelineOrchestratorImpl := pipeline.NewCiCdPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, clientImpl, ciCdConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, pipelineStageServiceImpl, gitMaterialHistoryServiceImpl, ciPipelineHistoryServiceImpl, ciTemplateReadServiceImpl, ciTemplateServiceImpl, dockerArtifactStoreRepositoryImpl, ciArtifactRepositoryImpl, configMapServiceImpl, customTagServiceImpl, genericNoteServiceImpl, chartServiceImpl, transactionUtilImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl, deploymentConfigReadServiceImpl, chartReadServiceImpl) pluginInputVariableParserImpl := pipeline.NewPluginInputVariableParserImpl(sugaredLogger, dockerRegistryConfigImpl, customTagServiceImpl) ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workFlowStageStatusServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciWorkflowRepositoryImpl, transactionUtilImpl) - serviceImpl := trigger.NewServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciPipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateReadServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, ciCdPipelineOrchestratorImpl, attributesServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl, ciServiceImpl, ciWorkflowRepositoryImpl, clientImpl) + ciLogServiceImpl, err := pipeline.NewCiLogServiceImpl(sugaredLogger, k8sServiceImpl) + if err != nil { + return nil, err + } + blobStorageConfigServiceImpl := pipeline.NewBlobStorageConfigServiceImpl(sugaredLogger, k8sServiceImpl, ciCdConfig) + serviceImpl := trigger.NewServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciPipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateReadServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, ciCdPipelineOrchestratorImpl, attributesServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl, ciServiceImpl, ciWorkflowRepositoryImpl, clientImpl, ciLogServiceImpl, blobStorageConfigServiceImpl, clusterServiceImplExtended, environmentServiceImpl, k8sServiceImpl) gitWebhookServiceImpl := gitWebhook.NewGitWebhookServiceImpl(sugaredLogger, gitWebhookRepositoryImpl, serviceImpl) gitWebhookRestHandlerImpl := restHandler.NewGitWebhookRestHandlerImpl(sugaredLogger, gitWebhookServiceImpl) ecrConfig, err := pipeline.GetEcrConfig() @@ -726,13 +732,8 @@ func InitializeApp() (*App, error) { pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, gitMaterialReadServiceImpl, chartRepositoryImpl, ciPipelineConfigServiceImpl, ciMaterialConfigServiceImpl, appArtifactManagerImpl, devtronAppCMCSServiceImpl, devtronAppStrategyServiceImpl, appDeploymentTypeChangeManagerImpl, cdPipelineConfigServiceImpl, devtronAppConfigServiceImpl) deploymentTemplateValidationServiceImpl := deploymentTemplate.NewDeploymentTemplateValidationServiceImpl(sugaredLogger, chartRefServiceImpl, scopedVariableManagerImpl) devtronAppGitOpConfigServiceImpl := gitOpsConfig.NewDevtronAppGitOpConfigServiceImpl(sugaredLogger, chartRepositoryImpl, chartServiceImpl, gitOpsConfigReadServiceImpl, gitOpsValidationServiceImpl, argoClientWrapperServiceImpl, deploymentConfigServiceImpl, chartReadServiceImpl) - ciLogServiceImpl, err := pipeline.NewCiLogServiceImpl(sugaredLogger, k8sServiceImpl) - if err != nil { - return nil, err - } - blobStorageConfigServiceImpl := pipeline.NewBlobStorageConfigServiceImpl(sugaredLogger, k8sServiceImpl, ciCdConfig) - ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, workflowServiceImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, environmentServiceImpl, workFlowStageStatusServiceImpl, k8sServiceImpl) - cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, workflowServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, customTagServiceImpl, deploymentConfigServiceImpl, workFlowStageStatusServiceImpl, cdWorkflowRunnerServiceImpl) + ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, workFlowStageStatusServiceImpl) + cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, customTagServiceImpl, deploymentConfigServiceImpl, workFlowStageStatusServiceImpl, cdWorkflowRunnerServiceImpl) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl, chartServiceImpl, deploymentConfigServiceImpl, pipelineBuilderImpl) appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, attributesServiceImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, pipelineStageServiceImpl, ciTemplateReadServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigServiceImpl, gitOpsConfigReadServiceImpl, chartReadServiceImpl) deploymentTemplateRepositoryImpl := repository2.NewDeploymentTemplateRepositoryImpl(db, sugaredLogger) @@ -750,7 +751,6 @@ func InitializeApp() (*App, error) { cveStoreRepositoryImpl := repository24.NewCveStoreRepositoryImpl(db, sugaredLogger) policyServiceImpl := imageScanning.NewPolicyServiceImpl(environmentServiceImpl, sugaredLogger, appRepositoryImpl, pipelineOverrideRepositoryImpl, cvePolicyRepositoryImpl, clusterServiceImplExtended, pipelineRepositoryImpl, imageScanResultRepositoryImpl, imageScanDeployInfoRepositoryImpl, imageScanObjectMetaRepositoryImpl, httpClient, ciArtifactRepositoryImpl, ciCdConfig, imageScanHistoryReadServiceImpl, cveStoreRepositoryImpl, ciTemplateRepositoryImpl, clusterReadServiceImpl, transactionUtilImpl) imageScanResultReadServiceImpl := read18.NewImageScanResultReadServiceImpl(sugaredLogger, imageScanResultRepositoryImpl) - pipelineConfigRestHandlerImpl := configure.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, deploymentTemplateValidationServiceImpl, chartServiceImpl, devtronAppGitOpConfigServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, generateManifestDeploymentTemplateServiceImpl, appWorkflowServiceImpl, gitMaterialReadServiceImpl, policyServiceImpl, imageScanResultReadServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingReadServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, ciCdPipelineOrchestratorImpl, gitProviderReadServiceImpl, teamReadServiceImpl, environmentRepositoryImpl, chartReadServiceImpl, serviceImpl) gitOpsManifestPushServiceImpl := publish.NewGitOpsManifestPushServiceImpl(sugaredLogger, pipelineStatusTimelineServiceImpl, pipelineOverrideRepositoryImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, chartServiceImpl, gitOperationServiceImpl, argoClientWrapperServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, chartTemplateServiceImpl) manifestCreationServiceImpl := manifest.NewManifestCreationServiceImpl(sugaredLogger, dockerRegistryIpsConfigServiceImpl, chartRefServiceImpl, scopedVariableCMCSManagerImpl, k8sCommonServiceImpl, deployedAppMetricsServiceImpl, imageDigestPolicyServiceImpl, utilMergeUtil, appCrudOperationServiceImpl, deploymentTemplateServiceImpl, argoClientWrapperServiceImpl, configMapHistoryRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineOverrideRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, pipelineConfigRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, deploymentConfigServiceImpl, envConfigOverrideReadServiceImpl) configMapHistoryReadServiceImpl := read19.NewConfigMapHistoryReadService(sugaredLogger, configMapHistoryRepositoryImpl, scopedVariableCMCSManagerImpl) @@ -763,10 +763,11 @@ func InitializeApp() (*App, error) { scanToolExecutionHistoryMappingRepositoryImpl := repository24.NewScanToolExecutionHistoryMappingRepositoryImpl(db, sugaredLogger) cdWorkflowReadServiceImpl := read20.NewCdWorkflowReadServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) imageScanServiceImpl := imageScanning.NewImageScanServiceImpl(sugaredLogger, imageScanHistoryRepositoryImpl, imageScanResultRepositoryImpl, imageScanObjectMetaRepositoryImpl, cveStoreRepositoryImpl, imageScanDeployInfoRepositoryImpl, userServiceImpl, appRepositoryImpl, environmentServiceImpl, ciArtifactRepositoryImpl, policyServiceImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, scanToolMetadataRepositoryImpl, scanToolExecutionHistoryMappingRepositoryImpl, cvePolicyRepositoryImpl, cdWorkflowReadServiceImpl) - triggerServiceImpl, err := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, workflowServiceImpl, imageDigestPolicyServiceImpl, userServiceImpl, clientImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryReadServiceImpl, imageScanDeployInfoReadServiceImpl, imageScanDeployInfoServiceImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateReadServiceImpl, gitMaterialReadServiceImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, ciCdPipelineOrchestratorImpl, gitOperationServiceImpl, attributesServiceImpl, clusterRepositoryImpl, cdWorkflowRunnerServiceImpl) + triggerServiceImpl, err := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, imageDigestPolicyServiceImpl, userServiceImpl, clientImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryReadServiceImpl, imageScanDeployInfoReadServiceImpl, imageScanDeployInfoServiceImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateReadServiceImpl, gitMaterialReadServiceImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, ciCdPipelineOrchestratorImpl, gitOperationServiceImpl, attributesServiceImpl, clusterRepositoryImpl, cdWorkflowRunnerServiceImpl, clusterServiceImplExtended, ciLogServiceImpl, workflowServiceImpl, blobStorageConfigServiceImpl) if err != nil { return nil, err } + pipelineConfigRestHandlerImpl := configure.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, deploymentTemplateValidationServiceImpl, chartServiceImpl, devtronAppGitOpConfigServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, generateManifestDeploymentTemplateServiceImpl, appWorkflowServiceImpl, gitMaterialReadServiceImpl, policyServiceImpl, imageScanResultReadServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingReadServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, ciCdPipelineOrchestratorImpl, gitProviderReadServiceImpl, teamReadServiceImpl, environmentRepositoryImpl, chartReadServiceImpl, serviceImpl, triggerServiceImpl) commonArtifactServiceImpl := artifacts.NewCommonArtifactServiceImpl(sugaredLogger, ciArtifactRepositoryImpl) workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, cdWorkflowRunnerServiceImpl, ciServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl, runnable, imageScanHistoryRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, environmentRepositoryImpl, k8sCommonServiceImpl, workflowServiceImpl, serviceImpl) externalCiRestHandlerImpl := restHandler.NewExternalCiRestHandlerImpl(sugaredLogger, validate, userServiceImpl, enforcerImpl, workflowDagExecutorImpl) From 1a2de648a2b5566e3ed017506a4257b6ac1575ea Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Mon, 7 Apr 2025 10:07:26 +0530 Subject: [PATCH 09/17] workflow service ent change --- pkg/executor/WorkflowService.go | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/pkg/executor/WorkflowService.go b/pkg/executor/WorkflowService.go index 339c566113..3db6887c1f 100644 --- a/pkg/executor/WorkflowService.go +++ b/pkg/executor/WorkflowService.go @@ -70,9 +70,9 @@ type WorkflowServiceImpl struct { globalCMCSService pipeline.GlobalCMCSService argoWorkflowExecutor executors.ArgoWorkflowExecutor systemWorkflowExecutor executors.SystemWorkflowExecutor - k8sUtil *k8s.K8sServiceImpl k8sCommonService k8s2.K8sCommonService infraProvider infraProviders.InfraProvider + k8sUtil *k8s.K8sServiceImpl } // TODO: Move to bean @@ -83,10 +83,11 @@ func NewWorkflowServiceImpl(Logger *zap.SugaredLogger, configMapService read.ConfigReadService, globalCMCSService pipeline.GlobalCMCSService, argoWorkflowExecutor executors.ArgoWorkflowExecutor, - k8sUtil *k8s.K8sServiceImpl, systemWorkflowExecutor executors.SystemWorkflowExecutor, k8sCommonService k8s2.K8sCommonService, - infraProvider infraProviders.InfraProvider) (*WorkflowServiceImpl, error) { + infraProvider infraProviders.InfraProvider, + k8sUtil *k8s.K8sServiceImpl, +) (*WorkflowServiceImpl, error) { commonWorkflowService := &WorkflowServiceImpl{ Logger: Logger, ciCdConfig: ciCdConfig, @@ -110,9 +111,6 @@ func NewWorkflowServiceImpl(Logger *zap.SugaredLogger, const ( CI_NODE_SELECTOR_APP_LABEL_KEY = "devtron.ai/node-selector" - - preCdStage = "preCD" - postCdStage = "postCD" ) func (impl *WorkflowServiceImpl) SubmitWorkflow(workflowRequest *types.WorkflowRequest) (*unstructured.UnstructuredList, string, error) { @@ -120,13 +118,16 @@ func (impl *WorkflowServiceImpl) SubmitWorkflow(workflowRequest *types.WorkflowR if err != nil { return nil, "", err } - workflowExecutor := impl.getWorkflowExecutor(workflowRequest.WorkflowExecutor) - if workflowExecutor == nil { - return nil, "", errors.New("workflow executor not found") + var createdWf *unstructured.UnstructuredList + canExecuteWorkflow, jobHelmChartPath, err := impl.checkIfCanExecuteWorkflowAndHandleVirtualExec(workflowRequest, workflowTemplate) + if canExecuteWorkflow { + workflowExecutor := impl.getWorkflowExecutor(workflowRequest.WorkflowExecutor) + if workflowExecutor == nil { + return nil, "", errors.New("workflow executor not found") + } + createdWf, err = workflowExecutor.ExecuteWorkflow(workflowTemplate) } - createdWf, err := workflowExecutor.ExecuteWorkflow(workflowTemplate) - jobHelmPackagePath := "" // due to ENT - return createdWf, jobHelmPackagePath, err + return createdWf, jobHelmChartPath, err } func (impl *WorkflowServiceImpl) createWorkflowTemplate(workflowRequest *types.WorkflowRequest) (bean3.WorkflowTemplate, error) { @@ -142,6 +143,11 @@ func (impl *WorkflowServiceImpl) createWorkflowTemplate(workflowRequest *types.W return bean3.WorkflowTemplate{}, err } + workflowTemplate, err = impl.updateWorkflowTemplateWithLabels(workflowRequest, workflowTemplate) + if err != nil { + impl.Logger.Errorw("error occurred while updating workflow template with labels", "err", err) + return bean3.WorkflowTemplate{}, err + } workflowRequest.AddNodeConstraintsFromConfig(&workflowTemplate, impl.ciCdConfig) infraConfiguration := &v1.InfraConfig{} shouldAddExistingCmCsInWorkflow := impl.shouldAddExistingCmCsInWorkflow(workflowRequest) From 3120992fb41866bf72590b9fd0199f55d8f85d84 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Mon, 7 Apr 2025 10:07:36 +0530 Subject: [PATCH 10/17] workflow service ent change --- pkg/executor/WorkflowService_ent.go | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 pkg/executor/WorkflowService_ent.go diff --git a/pkg/executor/WorkflowService_ent.go b/pkg/executor/WorkflowService_ent.go new file mode 100644 index 0000000000..f01a801a90 --- /dev/null +++ b/pkg/executor/WorkflowService_ent.go @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2020-2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package executor + +import ( + bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/types" +) + +func (impl *WorkflowServiceImpl) checkIfCanExecuteWorkflowAndHandleVirtualExec(workflowRequest *types.WorkflowRequest, workflowTemplate bean3.WorkflowTemplate) (canExecuteWorkflow bool, jobHelmChartPath string, err error) { + return true, "", nil +} + +func (impl *WorkflowServiceImpl) updateWorkflowTemplateWithLabels(workflowRequest *types.WorkflowRequest, workflowTemplate bean3.WorkflowTemplate) (bean3.WorkflowTemplate, error) { + return workflowTemplate, nil +} From 7ae5b3d8eabefb0c7cbfc2929b0722d4110748d1 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Mon, 7 Apr 2025 10:53:37 +0530 Subject: [PATCH 11/17] Redundant import removed --- .../trigger/devtronApps/TriggerService.go | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/TriggerService.go index 942761fc47..bcef986aa7 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService.go @@ -31,7 +31,6 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer" bean7 "github.com/devtron-labs/devtron/client/argocdServer/bean" client "github.com/devtron-labs/devtron/client/events" - gitSensorClient "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/middleware" "github.com/devtron-labs/devtron/internal/sql/models" repository3 "github.com/devtron-labs/devtron/internal/sql/repository" @@ -146,7 +145,6 @@ type TriggerServiceImpl struct { scopedVariableManager variables.ScopedVariableCMCSManager imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService userService user.UserService - gitSensorClient gitSensorClient.Client config *types.CdConfig helmAppService client2.HelmAppService imageScanService security2.ImageScanService @@ -175,7 +173,6 @@ type TriggerServiceImpl struct { K8sUtil *util5.K8sServiceImpl transactionUtilImpl *sql.TransactionUtilImpl deploymentConfigService common.DeploymentConfigService - deploymentServiceTypeConfig *globalUtil.DeploymentServiceTypeConfig ciCdPipelineOrchestrator pipeline.CiCdPipelineOrchestrator gitOperationService git.GitOperationService attributeService attributes.AttributesService @@ -207,7 +204,6 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, scopedVariableManager variables.ScopedVariableCMCSManager, imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService, userService user.UserService, - gitSensorClient gitSensorClient.Client, helmAppService client2.HelmAppService, enforcerUtil rbac.EnforcerUtil, userDeploymentRequestService service.UserDeploymentRequestService, @@ -269,7 +265,6 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, scopedVariableManager: scopedVariableManager, imageDigestPolicyService: imageDigestPolicyService, userService: userService, - gitSensorClient: gitSensorClient, helmAppService: helmAppService, enforcerUtil: enforcerUtil, eventFactory: eventFactory, @@ -303,12 +298,11 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, transactionUtilImpl: transactionUtilImpl, - deploymentConfigService: deploymentConfigService, - deploymentServiceTypeConfig: envVariables.DeploymentServiceTypeConfig, - ciCdPipelineOrchestrator: ciCdPipelineOrchestrator, - gitOperationService: gitOperationService, - attributeService: attributeService, - cdWorkflowRunnerService: cdWorkflowRunnerService, + deploymentConfigService: deploymentConfigService, + ciCdPipelineOrchestrator: ciCdPipelineOrchestrator, + gitOperationService: gitOperationService, + attributeService: attributeService, + cdWorkflowRunnerService: cdWorkflowRunnerService, clusterRepository: clusterRepository, clusterService: clusterService, From 5ab4dfb183e15f6ad3ceefd11819056ea6f16883 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Mon, 7 Apr 2025 12:02:07 +0530 Subject: [PATCH 12/17] rename pkg/ --- api/argoApplication/ArgoApplicationRestHandler.go | 4 ++-- api/argoApplication/wire_argoApplication.go | 6 +++--- api/helm-app/HelmAppRestHandler.go | 4 ++-- api/k8s/application/k8sApplicationRestHandler.go | 4 ++-- cmd/external-app/wire_gen.go | 6 +++--- pkg/app/AppService.go | 2 +- .../InstalledAppDeploymentTypeChangeService.go | 4 ++-- .../service/FullMode/resource/ResourceTreeService.go | 4 ++-- .../argoApplication/ArgoApplicationService.go | 6 +++--- .../argoApplication/ArgoApplicationServiceExtended.go | 4 ++-- pkg/{nucleus => coreEntities}/argoApplication/bean/bean.go | 0 .../argoApplication/helper/deploymentStatusHelper.go | 0 .../argoApplication/helper/helper.go | 2 +- .../argoApplication/read/ArgoApplicationReadService.go | 4 ++-- .../read/config/ArgoApplicationConfigService.go | 4 ++-- .../deployedApp/status/resourceTree/ResourceTreeService.go | 4 ++-- pkg/deployment/trigger/devtronApps/TriggerService_ent1.go | 2 +- pkg/k8s/K8sCommonService.go | 2 +- pkg/k8s/application/k8sApplicationService.go | 2 +- pkg/terminal/terminalSesion.go | 2 +- wire_gen.go | 6 +++--- 21 files changed, 36 insertions(+), 36 deletions(-) rename pkg/{nucleus => coreEntities}/argoApplication/ArgoApplicationService.go (97%) rename pkg/{nucleus => coreEntities}/argoApplication/ArgoApplicationServiceExtended.go (99%) rename pkg/{nucleus => coreEntities}/argoApplication/bean/bean.go (100%) rename pkg/{nucleus => coreEntities}/argoApplication/helper/deploymentStatusHelper.go (100%) rename pkg/{nucleus => coreEntities}/argoApplication/helper/helper.go (98%) rename pkg/{nucleus => coreEntities}/argoApplication/read/ArgoApplicationReadService.go (98%) rename pkg/{nucleus => coreEntities}/argoApplication/read/config/ArgoApplicationConfigService.go (97%) diff --git a/api/argoApplication/ArgoApplicationRestHandler.go b/api/argoApplication/ArgoApplicationRestHandler.go index 27d9a8632e..2ac9e84e33 100644 --- a/api/argoApplication/ArgoApplicationRestHandler.go +++ b/api/argoApplication/ArgoApplicationRestHandler.go @@ -21,8 +21,8 @@ import ( "errors" "github.com/devtron-labs/devtron/api/restHandler/common" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read" "go.uber.org/zap" "net/http" "strconv" diff --git a/api/argoApplication/wire_argoApplication.go b/api/argoApplication/wire_argoApplication.go index 6373e274f8..95af6b89d1 100644 --- a/api/argoApplication/wire_argoApplication.go +++ b/api/argoApplication/wire_argoApplication.go @@ -17,9 +17,9 @@ package argoApplication import ( - argoApplication2 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read/config" + argoApplication2 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read/config" "github.com/google/wire" ) diff --git a/api/helm-app/HelmAppRestHandler.go b/api/helm-app/HelmAppRestHandler.go index feab53cc95..a8010b3503 100644 --- a/api/helm-app/HelmAppRestHandler.go +++ b/api/helm-app/HelmAppRestHandler.go @@ -25,11 +25,11 @@ import ( "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/EAMode" clusterBean "github.com/devtron-labs/devtron/pkg/cluster/bean" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" clientErrors "github.com/devtron-labs/devtron/pkg/errors" "github.com/devtron-labs/devtron/pkg/fluxApplication" bean2 "github.com/devtron-labs/devtron/pkg/k8s/application/bean" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/pipeline" "net/http" "strconv" diff --git a/api/k8s/application/k8sApplicationRestHandler.go b/api/k8s/application/k8sApplicationRestHandler.go index 55d42ed15c..190c79c05b 100644 --- a/api/k8s/application/k8sApplicationRestHandler.go +++ b/api/k8s/application/k8sApplicationRestHandler.go @@ -36,14 +36,14 @@ import ( "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/auth/user" bean4 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read" clientErrors "github.com/devtron-labs/devtron/pkg/errors" "github.com/devtron-labs/devtron/pkg/fluxApplication" "github.com/devtron-labs/devtron/pkg/k8s" application2 "github.com/devtron-labs/devtron/pkg/k8s/application" bean2 "github.com/devtron-labs/devtron/pkg/k8s/application/bean" bean3 "github.com/devtron-labs/devtron/pkg/k8s/bean" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read" "github.com/devtron-labs/devtron/pkg/terminal" "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/rbac" diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index 7e1c13fc04..01f77271cd 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -115,9 +115,9 @@ import ( read7 "github.com/devtron-labs/devtron/pkg/module/read" "github.com/devtron-labs/devtron/pkg/module/repo" "github.com/devtron-labs/devtron/pkg/module/store" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" - read9 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read" - config3 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read/config" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" + read9 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read" + config3 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool" repository11 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 5d44b6f4a4..217f6d0097 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -29,6 +29,7 @@ import ( internalUtil "github.com/devtron-labs/devtron/internal/util" bean3 "github.com/devtron-labs/devtron/pkg/app/bean" installedAppReader "github.com/devtron-labs/devtron/pkg/appStore/installedApp/read" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" bean2 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" @@ -38,7 +39,6 @@ import ( bean6 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/bean" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/read" bean4 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/workflow/cd" "net/url" "strconv" diff --git a/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go b/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go index 1895a3975b..a3c530dba2 100644 --- a/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go +++ b/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go @@ -42,13 +42,13 @@ import ( "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/cluster/read" repository5 "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" + bean4 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" "github.com/devtron-labs/devtron/pkg/deployment/common" bean3 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" bean2 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/k8s" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" - bean4 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" util3 "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" "go.uber.org/zap" diff --git a/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go b/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go index 082cb6856e..49c965f78b 100644 --- a/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go +++ b/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go @@ -37,12 +37,12 @@ import ( "github.com/devtron-labs/devtron/pkg/appStore/bean" appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/repository" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" + bean3 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" "github.com/devtron-labs/devtron/pkg/deployment/common" bean2 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/k8s" application3 "github.com/devtron-labs/devtron/pkg/k8s/application" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" - bean3 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" util3 "github.com/devtron-labs/devtron/pkg/util" util2 "github.com/devtron-labs/devtron/util" "github.com/tidwall/gjson" diff --git a/pkg/nucleus/argoApplication/ArgoApplicationService.go b/pkg/coreEntities/argoApplication/ArgoApplicationService.go similarity index 97% rename from pkg/nucleus/argoApplication/ArgoApplicationService.go rename to pkg/coreEntities/argoApplication/ArgoApplicationService.go index 4bd4f893bc..db8ae43dd3 100644 --- a/pkg/nucleus/argoApplication/ArgoApplicationService.go +++ b/pkg/coreEntities/argoApplication/ArgoApplicationService.go @@ -27,13 +27,13 @@ import ( util2 "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/cluster/adapter" clusterRepository "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/deployment/common" commonBean "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/k8s/application" k8s2 "github.com/devtron-labs/devtron/pkg/k8s/bean" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read/config" "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/sliceUtil" "go.uber.org/zap" diff --git a/pkg/nucleus/argoApplication/ArgoApplicationServiceExtended.go b/pkg/coreEntities/argoApplication/ArgoApplicationServiceExtended.go similarity index 99% rename from pkg/nucleus/argoApplication/ArgoApplicationServiceExtended.go rename to pkg/coreEntities/argoApplication/ArgoApplicationServiceExtended.go index 2643bbe432..40c239ea16 100644 --- a/pkg/nucleus/argoApplication/ArgoApplicationServiceExtended.go +++ b/pkg/coreEntities/argoApplication/ArgoApplicationServiceExtended.go @@ -29,8 +29,8 @@ import ( util2 "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app/appDetails/adapter" "github.com/devtron-labs/devtron/pkg/cluster" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read" util3 "github.com/devtron-labs/devtron/pkg/util" "github.com/devtron-labs/devtron/util" "google.golang.org/grpc" diff --git a/pkg/nucleus/argoApplication/bean/bean.go b/pkg/coreEntities/argoApplication/bean/bean.go similarity index 100% rename from pkg/nucleus/argoApplication/bean/bean.go rename to pkg/coreEntities/argoApplication/bean/bean.go diff --git a/pkg/nucleus/argoApplication/helper/deploymentStatusHelper.go b/pkg/coreEntities/argoApplication/helper/deploymentStatusHelper.go similarity index 100% rename from pkg/nucleus/argoApplication/helper/deploymentStatusHelper.go rename to pkg/coreEntities/argoApplication/helper/deploymentStatusHelper.go diff --git a/pkg/nucleus/argoApplication/helper/helper.go b/pkg/coreEntities/argoApplication/helper/helper.go similarity index 98% rename from pkg/nucleus/argoApplication/helper/helper.go rename to pkg/coreEntities/argoApplication/helper/helper.go index f68840f416..1d377555b2 100644 --- a/pkg/nucleus/argoApplication/helper/helper.go +++ b/pkg/coreEntities/argoApplication/helper/helper.go @@ -5,7 +5,7 @@ import ( "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/api/helm-app/gRPC" "github.com/devtron-labs/devtron/pkg/cluster/repository" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" "strconv" "strings" ) diff --git a/pkg/nucleus/argoApplication/read/ArgoApplicationReadService.go b/pkg/coreEntities/argoApplication/read/ArgoApplicationReadService.go similarity index 98% rename from pkg/nucleus/argoApplication/read/ArgoApplicationReadService.go rename to pkg/coreEntities/argoApplication/read/ArgoApplicationReadService.go index d95521ba17..10742756a9 100644 --- a/pkg/nucleus/argoApplication/read/ArgoApplicationReadService.go +++ b/pkg/coreEntities/argoApplication/read/ArgoApplicationReadService.go @@ -26,9 +26,9 @@ import ( "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/pkg/cluster/adapter" clusterRepository "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" clientErrors "github.com/devtron-labs/devtron/pkg/errors" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" "go.uber.org/zap" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/pkg/nucleus/argoApplication/read/config/ArgoApplicationConfigService.go b/pkg/coreEntities/argoApplication/read/config/ArgoApplicationConfigService.go similarity index 97% rename from pkg/nucleus/argoApplication/read/config/ArgoApplicationConfigService.go rename to pkg/coreEntities/argoApplication/read/config/ArgoApplicationConfigService.go index 95aee5d25c..dfcda06dbc 100644 --- a/pkg/nucleus/argoApplication/read/config/ArgoApplicationConfigService.go +++ b/pkg/coreEntities/argoApplication/read/config/ArgoApplicationConfigService.go @@ -8,8 +8,8 @@ import ( k8sCommonBean "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/pkg/cluster/adapter" clusterRepository "github.com/devtron-labs/devtron/pkg/cluster/repository" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" "go.uber.org/zap" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/pkg/deployment/deployedApp/status/resourceTree/ResourceTreeService.go b/pkg/deployment/deployedApp/status/resourceTree/ResourceTreeService.go index 4f5ea83ca5..1462c172a5 100644 --- a/pkg/deployment/deployedApp/status/resourceTree/ResourceTreeService.go +++ b/pkg/deployment/deployedApp/status/resourceTree/ResourceTreeService.go @@ -35,11 +35,11 @@ import ( "github.com/devtron-labs/devtron/pkg/app" "github.com/devtron-labs/devtron/pkg/appStatus" read2 "github.com/devtron-labs/devtron/pkg/cluster/environment/read" + argoApplication2 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" + bean2 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" commonBean "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/k8s" application2 "github.com/devtron-labs/devtron/pkg/k8s/application" - argoApplication2 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" - bean2 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/bean" util2 "github.com/devtron-labs/devtron/util" "go.uber.org/zap" "strconv" diff --git a/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go b/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go index 6c54cf9c57..fb45e1ac27 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go @@ -10,9 +10,9 @@ import ( bean4 "github.com/devtron-labs/devtron/pkg/app/bean" bean2 "github.com/devtron-labs/devtron/pkg/bean" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/deployment/manifest/publish" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/pipeline/repository" "helm.sh/helm/v3/pkg/chart" ) diff --git a/pkg/k8s/K8sCommonService.go b/pkg/k8s/K8sCommonService.go index d448dd2311..6d58635abc 100644 --- a/pkg/k8s/K8sCommonService.go +++ b/pkg/k8s/K8sCommonService.go @@ -28,9 +28,9 @@ import ( bean2 "github.com/devtron-labs/devtron/pkg/cluster/bean" bean4 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" "github.com/devtron-labs/devtron/pkg/cluster/read" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read/config" bean3 "github.com/devtron-labs/devtron/pkg/k8s/application/bean" bean5 "github.com/devtron-labs/devtron/pkg/k8s/bean" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read/config" "github.com/devtron-labs/devtron/util" "go.opentelemetry.io/otel" "go.uber.org/zap" diff --git a/pkg/k8s/application/k8sApplicationService.go b/pkg/k8s/application/k8sApplicationService.go index 7ab5a6b6b7..d555018363 100644 --- a/pkg/k8s/application/k8sApplicationService.go +++ b/pkg/k8s/application/k8sApplicationService.go @@ -28,11 +28,11 @@ import ( "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" bean5 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" "github.com/devtron-labs/devtron/pkg/cluster/read" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" clientErrors "github.com/devtron-labs/devtron/pkg/errors" "github.com/devtron-labs/devtron/pkg/fluxApplication" bean2 "github.com/devtron-labs/devtron/pkg/fluxApplication/bean" bean4 "github.com/devtron-labs/devtron/pkg/k8s/bean" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/helper" "io" v1 "k8s.io/client-go/kubernetes/typed/core/v1" "net/http" diff --git a/pkg/terminal/terminalSesion.go b/pkg/terminal/terminalSesion.go index 07549991f7..7f720b770d 100644 --- a/pkg/terminal/terminalSesion.go +++ b/pkg/terminal/terminalSesion.go @@ -32,7 +32,7 @@ import ( bean2 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" "github.com/devtron-labs/devtron/pkg/cluster/read" "github.com/devtron-labs/devtron/pkg/cluster/repository" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read/config" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read/config" errors1 "github.com/juju/errors" "go.uber.org/zap" "io" diff --git a/wire_gen.go b/wire_gen.go index 4f51c1ac3c..f0051f606c 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -227,9 +227,9 @@ import ( "github.com/devtron-labs/devtron/pkg/module/repo" "github.com/devtron-labs/devtron/pkg/module/store" "github.com/devtron-labs/devtron/pkg/notifier" - "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication" - read22 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read" - config2 "github.com/devtron-labs/devtron/pkg/nucleus/argoApplication/read/config" + "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" + read22 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read" + config2 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/history" From 6bd6815c1cf22ed35f2b19522ab67204cc440b12 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Mon, 7 Apr 2025 12:06:53 +0530 Subject: [PATCH 13/17] reverted pkg categorisation --- .../ArgoApplicationRestHandler.go | 4 ++-- api/argoApplication/wire_argoApplication.go | 16 ++++++++-------- api/helm-app/HelmAppRestHandler.go | 4 ++-- api/k8s/application/k8sApplicationRestHandler.go | 4 ++-- cmd/external-app/wire_gen.go | 6 +++--- pkg/app/AppService.go | 2 +- .../InstalledAppDeploymentTypeChangeService.go | 4 ++-- .../FullMode/resource/ResourceTreeService.go | 4 ++-- .../argoApplication/ArgoApplicationService.go | 6 +++--- .../ArgoApplicationServiceExtended.go | 4 ++-- .../argoApplication/bean/bean.go | 0 .../helper/deploymentStatusHelper.go | 0 .../argoApplication/helper/helper.go | 2 +- .../read/ArgoApplicationReadService.go | 4 ++-- .../read/config/ArgoApplicationConfigService.go | 4 ++-- .../status/resourceTree/ResourceTreeService.go | 4 ++-- .../trigger/devtronApps/TriggerService_ent1.go | 2 +- pkg/k8s/K8sCommonService.go | 2 +- pkg/k8s/application/k8sApplicationService.go | 2 +- pkg/terminal/terminalSesion.go | 2 +- wire_gen.go | 10 +++++----- 21 files changed, 43 insertions(+), 43 deletions(-) rename pkg/{coreEntities => }/argoApplication/ArgoApplicationService.go (97%) rename pkg/{coreEntities => }/argoApplication/ArgoApplicationServiceExtended.go (99%) rename pkg/{coreEntities => }/argoApplication/bean/bean.go (100%) rename pkg/{coreEntities => }/argoApplication/helper/deploymentStatusHelper.go (100%) rename pkg/{coreEntities => }/argoApplication/helper/helper.go (98%) rename pkg/{coreEntities => }/argoApplication/read/ArgoApplicationReadService.go (98%) rename pkg/{coreEntities => }/argoApplication/read/config/ArgoApplicationConfigService.go (97%) diff --git a/api/argoApplication/ArgoApplicationRestHandler.go b/api/argoApplication/ArgoApplicationRestHandler.go index 2ac9e84e33..edcbabdca9 100644 --- a/api/argoApplication/ArgoApplicationRestHandler.go +++ b/api/argoApplication/ArgoApplicationRestHandler.go @@ -20,9 +20,9 @@ import ( "context" "errors" "github.com/devtron-labs/devtron/api/restHandler/common" + "github.com/devtron-labs/devtron/pkg/argoApplication" + "github.com/devtron-labs/devtron/pkg/argoApplication/read" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read" "go.uber.org/zap" "net/http" "strconv" diff --git a/api/argoApplication/wire_argoApplication.go b/api/argoApplication/wire_argoApplication.go index 95af6b89d1..45ef8139db 100644 --- a/api/argoApplication/wire_argoApplication.go +++ b/api/argoApplication/wire_argoApplication.go @@ -17,9 +17,9 @@ package argoApplication import ( - argoApplication2 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read/config" + argoApplication3 "github.com/devtron-labs/devtron/pkg/argoApplication" + "github.com/devtron-labs/devtron/pkg/argoApplication/read" + "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" "github.com/google/wire" ) @@ -30,9 +30,9 @@ var ArgoApplicationWireSetFull = wire.NewSet( config.NewArgoApplicationConfigServiceImpl, wire.Bind(new(config.ArgoApplicationConfigService), new(*config.ArgoApplicationConfigServiceImpl)), - argoApplication2.NewArgoApplicationServiceImpl, - argoApplication2.NewArgoApplicationServiceExtendedServiceImpl, - wire.Bind(new(argoApplication2.ArgoApplicationService), new(*argoApplication2.ArgoApplicationServiceExtendedImpl)), + argoApplication3.NewArgoApplicationServiceImpl, + argoApplication3.NewArgoApplicationServiceExtendedServiceImpl, + wire.Bind(new(argoApplication3.ArgoApplicationService), new(*argoApplication3.ArgoApplicationServiceExtendedImpl)), NewArgoApplicationRestHandlerImpl, wire.Bind(new(ArgoApplicationRestHandler), new(*ArgoApplicationRestHandlerImpl)), @@ -48,8 +48,8 @@ var ArgoApplicationWireSetEA = wire.NewSet( config.NewArgoApplicationConfigServiceImpl, wire.Bind(new(config.ArgoApplicationConfigService), new(*config.ArgoApplicationConfigServiceImpl)), - argoApplication2.NewArgoApplicationServiceImpl, - wire.Bind(new(argoApplication2.ArgoApplicationService), new(*argoApplication2.ArgoApplicationServiceImpl)), + argoApplication3.NewArgoApplicationServiceImpl, + wire.Bind(new(argoApplication3.ArgoApplicationService), new(*argoApplication3.ArgoApplicationServiceImpl)), NewArgoApplicationRestHandlerImpl, wire.Bind(new(ArgoApplicationRestHandler), new(*ArgoApplicationRestHandlerImpl)), diff --git a/api/helm-app/HelmAppRestHandler.go b/api/helm-app/HelmAppRestHandler.go index a8010b3503..a21cf730ff 100644 --- a/api/helm-app/HelmAppRestHandler.go +++ b/api/helm-app/HelmAppRestHandler.go @@ -24,9 +24,9 @@ import ( service2 "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/EAMode" + "github.com/devtron-labs/devtron/pkg/argoApplication" + "github.com/devtron-labs/devtron/pkg/argoApplication/helper" clusterBean "github.com/devtron-labs/devtron/pkg/cluster/bean" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" clientErrors "github.com/devtron-labs/devtron/pkg/errors" "github.com/devtron-labs/devtron/pkg/fluxApplication" bean2 "github.com/devtron-labs/devtron/pkg/k8s/application/bean" diff --git a/api/k8s/application/k8sApplicationRestHandler.go b/api/k8s/application/k8sApplicationRestHandler.go index 190c79c05b..dd68e18e15 100644 --- a/api/k8s/application/k8sApplicationRestHandler.go +++ b/api/k8s/application/k8sApplicationRestHandler.go @@ -33,11 +33,11 @@ import ( client "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/api/restHandler/common" util2 "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/argoApplication/helper" + "github.com/devtron-labs/devtron/pkg/argoApplication/read" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/auth/user" bean4 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read" clientErrors "github.com/devtron-labs/devtron/pkg/errors" "github.com/devtron-labs/devtron/pkg/fluxApplication" "github.com/devtron-labs/devtron/pkg/k8s" diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index 01f77271cd..e827637c0a 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -75,6 +75,9 @@ import ( "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/common" "github.com/devtron-labs/devtron/pkg/appStore/values/repository" service4 "github.com/devtron-labs/devtron/pkg/appStore/values/service" + "github.com/devtron-labs/devtron/pkg/argoApplication" + read9 "github.com/devtron-labs/devtron/pkg/argoApplication/read" + config3 "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/attributes" "github.com/devtron-labs/devtron/pkg/auth/authentication" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" @@ -115,9 +118,6 @@ import ( read7 "github.com/devtron-labs/devtron/pkg/module/read" "github.com/devtron-labs/devtron/pkg/module/repo" "github.com/devtron-labs/devtron/pkg/module/store" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" - read9 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read" - config3 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool" repository11 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 217f6d0097..5bb33db0c8 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -29,7 +29,7 @@ import ( internalUtil "github.com/devtron-labs/devtron/internal/util" bean3 "github.com/devtron-labs/devtron/pkg/app/bean" installedAppReader "github.com/devtron-labs/devtron/pkg/appStore/installedApp/read" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" + "github.com/devtron-labs/devtron/pkg/argoApplication/helper" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" bean2 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" diff --git a/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go b/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go index a3c530dba2..2d713500d6 100644 --- a/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go +++ b/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go @@ -37,13 +37,13 @@ import ( deployment2 "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/EAMode/deployment" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/FullMode/deployment" util2 "github.com/devtron-labs/devtron/pkg/appStore/util" + "github.com/devtron-labs/devtron/pkg/argoApplication" + bean4 "github.com/devtron-labs/devtron/pkg/argoApplication/bean" "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/cluster" "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/cluster/read" repository5 "github.com/devtron-labs/devtron/pkg/cluster/repository" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" - bean4 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" "github.com/devtron-labs/devtron/pkg/deployment/common" bean3 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" diff --git a/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go b/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go index 49c965f78b..5a4cd57313 100644 --- a/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go +++ b/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go @@ -37,8 +37,8 @@ import ( "github.com/devtron-labs/devtron/pkg/appStore/bean" appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/repository" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" - bean3 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/argoApplication" + bean3 "github.com/devtron-labs/devtron/pkg/argoApplication/bean" "github.com/devtron-labs/devtron/pkg/deployment/common" bean2 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/k8s" diff --git a/pkg/coreEntities/argoApplication/ArgoApplicationService.go b/pkg/argoApplication/ArgoApplicationService.go similarity index 97% rename from pkg/coreEntities/argoApplication/ArgoApplicationService.go rename to pkg/argoApplication/ArgoApplicationService.go index db8ae43dd3..48cd31c56d 100644 --- a/pkg/coreEntities/argoApplication/ArgoApplicationService.go +++ b/pkg/argoApplication/ArgoApplicationService.go @@ -25,11 +25,11 @@ import ( "github.com/devtron-labs/devtron/api/helm-app/service" argoApplication "github.com/devtron-labs/devtron/client/argocdServer/bean" util2 "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/argoApplication/helper" + "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/cluster/adapter" clusterRepository "github.com/devtron-labs/devtron/pkg/cluster/repository" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/deployment/common" commonBean "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/k8s/application" diff --git a/pkg/coreEntities/argoApplication/ArgoApplicationServiceExtended.go b/pkg/argoApplication/ArgoApplicationServiceExtended.go similarity index 99% rename from pkg/coreEntities/argoApplication/ArgoApplicationServiceExtended.go rename to pkg/argoApplication/ArgoApplicationServiceExtended.go index 40c239ea16..0cfb81b2d3 100644 --- a/pkg/coreEntities/argoApplication/ArgoApplicationServiceExtended.go +++ b/pkg/argoApplication/ArgoApplicationServiceExtended.go @@ -28,9 +28,9 @@ import ( argoApplication "github.com/devtron-labs/devtron/client/argocdServer/bean" util2 "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app/appDetails/adapter" + "github.com/devtron-labs/devtron/pkg/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/argoApplication/read" "github.com/devtron-labs/devtron/pkg/cluster" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read" util3 "github.com/devtron-labs/devtron/pkg/util" "github.com/devtron-labs/devtron/util" "google.golang.org/grpc" diff --git a/pkg/coreEntities/argoApplication/bean/bean.go b/pkg/argoApplication/bean/bean.go similarity index 100% rename from pkg/coreEntities/argoApplication/bean/bean.go rename to pkg/argoApplication/bean/bean.go diff --git a/pkg/coreEntities/argoApplication/helper/deploymentStatusHelper.go b/pkg/argoApplication/helper/deploymentStatusHelper.go similarity index 100% rename from pkg/coreEntities/argoApplication/helper/deploymentStatusHelper.go rename to pkg/argoApplication/helper/deploymentStatusHelper.go diff --git a/pkg/coreEntities/argoApplication/helper/helper.go b/pkg/argoApplication/helper/helper.go similarity index 98% rename from pkg/coreEntities/argoApplication/helper/helper.go rename to pkg/argoApplication/helper/helper.go index 1d377555b2..6197e150b4 100644 --- a/pkg/coreEntities/argoApplication/helper/helper.go +++ b/pkg/argoApplication/helper/helper.go @@ -4,8 +4,8 @@ import ( "fmt" "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/api/helm-app/gRPC" + "github.com/devtron-labs/devtron/pkg/argoApplication/bean" "github.com/devtron-labs/devtron/pkg/cluster/repository" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" "strconv" "strings" ) diff --git a/pkg/coreEntities/argoApplication/read/ArgoApplicationReadService.go b/pkg/argoApplication/read/ArgoApplicationReadService.go similarity index 98% rename from pkg/coreEntities/argoApplication/read/ArgoApplicationReadService.go rename to pkg/argoApplication/read/ArgoApplicationReadService.go index 10742756a9..b0f30ccb8f 100644 --- a/pkg/coreEntities/argoApplication/read/ArgoApplicationReadService.go +++ b/pkg/argoApplication/read/ArgoApplicationReadService.go @@ -24,10 +24,10 @@ import ( k8sCommonBean "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/api/helm-app/gRPC" "github.com/devtron-labs/devtron/api/helm-app/service" + "github.com/devtron-labs/devtron/pkg/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/cluster/adapter" clusterRepository "github.com/devtron-labs/devtron/pkg/cluster/repository" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" clientErrors "github.com/devtron-labs/devtron/pkg/errors" "go.uber.org/zap" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/coreEntities/argoApplication/read/config/ArgoApplicationConfigService.go b/pkg/argoApplication/read/config/ArgoApplicationConfigService.go similarity index 97% rename from pkg/coreEntities/argoApplication/read/config/ArgoApplicationConfigService.go rename to pkg/argoApplication/read/config/ArgoApplicationConfigService.go index dfcda06dbc..0c5a8bf819 100644 --- a/pkg/coreEntities/argoApplication/read/config/ArgoApplicationConfigService.go +++ b/pkg/argoApplication/read/config/ArgoApplicationConfigService.go @@ -6,10 +6,10 @@ import ( "fmt" "github.com/devtron-labs/common-lib/utils/k8s" k8sCommonBean "github.com/devtron-labs/common-lib/utils/k8s/commonBean" + "github.com/devtron-labs/devtron/pkg/argoApplication/bean" + "github.com/devtron-labs/devtron/pkg/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/cluster/adapter" clusterRepository "github.com/devtron-labs/devtron/pkg/cluster/repository" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" "go.uber.org/zap" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/pkg/deployment/deployedApp/status/resourceTree/ResourceTreeService.go b/pkg/deployment/deployedApp/status/resourceTree/ResourceTreeService.go index 1462c172a5..b6fd6b5f69 100644 --- a/pkg/deployment/deployedApp/status/resourceTree/ResourceTreeService.go +++ b/pkg/deployment/deployedApp/status/resourceTree/ResourceTreeService.go @@ -34,9 +34,9 @@ import ( "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" "github.com/devtron-labs/devtron/pkg/appStatus" + argoApplication2 "github.com/devtron-labs/devtron/pkg/argoApplication" + bean2 "github.com/devtron-labs/devtron/pkg/argoApplication/bean" read2 "github.com/devtron-labs/devtron/pkg/cluster/environment/read" - argoApplication2 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" - bean2 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/bean" commonBean "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/k8s" application2 "github.com/devtron-labs/devtron/pkg/k8s/application" diff --git a/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go b/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go index fb45e1ac27..f008f72adc 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go @@ -8,9 +8,9 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/app" bean4 "github.com/devtron-labs/devtron/pkg/app/bean" + "github.com/devtron-labs/devtron/pkg/argoApplication/helper" bean2 "github.com/devtron-labs/devtron/pkg/bean" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/deployment/manifest/publish" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/pipeline/repository" diff --git a/pkg/k8s/K8sCommonService.go b/pkg/k8s/K8sCommonService.go index 6d58635abc..0c3a2a9401 100644 --- a/pkg/k8s/K8sCommonService.go +++ b/pkg/k8s/K8sCommonService.go @@ -25,10 +25,10 @@ import ( "github.com/devtron-labs/devtron/api/bean/AppView" helmBean "github.com/devtron-labs/devtron/api/helm-app/service/bean" internalUtil "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" bean2 "github.com/devtron-labs/devtron/pkg/cluster/bean" bean4 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" "github.com/devtron-labs/devtron/pkg/cluster/read" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read/config" bean3 "github.com/devtron-labs/devtron/pkg/k8s/application/bean" bean5 "github.com/devtron-labs/devtron/pkg/k8s/bean" "github.com/devtron-labs/devtron/util" diff --git a/pkg/k8s/application/k8sApplicationService.go b/pkg/k8s/application/k8sApplicationService.go index d555018363..39588ef15e 100644 --- a/pkg/k8s/application/k8sApplicationService.go +++ b/pkg/k8s/application/k8sApplicationService.go @@ -25,10 +25,10 @@ import ( "github.com/devtron-labs/devtron/api/helm-app/gRPC" client "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/api/helm-app/service/bean" + "github.com/devtron-labs/devtron/pkg/argoApplication/helper" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" bean5 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" "github.com/devtron-labs/devtron/pkg/cluster/read" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/helper" clientErrors "github.com/devtron-labs/devtron/pkg/errors" "github.com/devtron-labs/devtron/pkg/fluxApplication" bean2 "github.com/devtron-labs/devtron/pkg/fluxApplication/bean" diff --git a/pkg/terminal/terminalSesion.go b/pkg/terminal/terminalSesion.go index 7f720b770d..b2b5f2841a 100644 --- a/pkg/terminal/terminalSesion.go +++ b/pkg/terminal/terminalSesion.go @@ -26,13 +26,13 @@ import ( "github.com/caarlos0/env" "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/internal/middleware" + "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/cluster" "github.com/devtron-labs/devtron/pkg/cluster/bean" "github.com/devtron-labs/devtron/pkg/cluster/environment" bean2 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" "github.com/devtron-labs/devtron/pkg/cluster/read" "github.com/devtron-labs/devtron/pkg/cluster/repository" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read/config" errors1 "github.com/juju/errors" "go.uber.org/zap" "io" diff --git a/wire_gen.go b/wire_gen.go index f0051f606c..e397a3a3a3 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -130,6 +130,9 @@ import ( "github.com/devtron-labs/devtron/pkg/appStore/values/repository" service4 "github.com/devtron-labs/devtron/pkg/appStore/values/service" appWorkflow2 "github.com/devtron-labs/devtron/pkg/appWorkflow" + argoApplication3 "github.com/devtron-labs/devtron/pkg/argoApplication" + read22 "github.com/devtron-labs/devtron/pkg/argoApplication/read" + config2 "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/asyncProvider" "github.com/devtron-labs/devtron/pkg/attributes" "github.com/devtron-labs/devtron/pkg/auth/authentication" @@ -227,9 +230,6 @@ import ( "github.com/devtron-labs/devtron/pkg/module/repo" "github.com/devtron-labs/devtron/pkg/module/store" "github.com/devtron-labs/devtron/pkg/notifier" - "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication" - read22 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read" - config2 "github.com/devtron-labs/devtron/pkg/coreEntities/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/history" @@ -824,9 +824,9 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - argoApplicationServiceImpl := argoApplication.NewArgoApplicationServiceImpl(sugaredLogger, clusterRepositoryImpl, k8sServiceImpl, helmAppClientImpl, helmAppServiceImpl, k8sApplicationServiceImpl, argoApplicationConfigServiceImpl, deploymentConfigServiceImpl) + argoApplicationServiceImpl := argoApplication3.NewArgoApplicationServiceImpl(sugaredLogger, clusterRepositoryImpl, k8sServiceImpl, helmAppClientImpl, helmAppServiceImpl, k8sApplicationServiceImpl, argoApplicationConfigServiceImpl, deploymentConfigServiceImpl) argoApplicationReadServiceImpl := read22.NewArgoApplicationReadServiceImpl(sugaredLogger, clusterRepositoryImpl, k8sServiceImpl, helmAppClientImpl, helmAppServiceImpl) - argoApplicationServiceExtendedImpl := argoApplication.NewArgoApplicationServiceExtendedServiceImpl(acdAuthConfig, argoApplicationServiceImpl, argoClientWrapperServiceImpl, argoApplicationReadServiceImpl, clusterServiceImplExtended) + argoApplicationServiceExtendedImpl := argoApplication3.NewArgoApplicationServiceExtendedServiceImpl(acdAuthConfig, argoApplicationServiceImpl, argoClientWrapperServiceImpl, argoApplicationReadServiceImpl, clusterServiceImplExtended) installedAppResourceServiceImpl := resource.NewInstalledAppResourceServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, argoClientWrapperServiceImpl, acdAuthConfig, installedAppVersionHistoryRepositoryImpl, helmAppServiceImpl, helmAppReadServiceImpl, appStatusServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, k8sServiceImpl, deploymentConfigServiceImpl, ociRegistryConfigRepositoryImpl, argoApplicationServiceExtendedImpl) chartGroupEntriesRepositoryImpl := repository28.NewChartGroupEntriesRepositoryImpl(db, sugaredLogger) chartGroupReposotoryImpl := repository28.NewChartGroupReposotoryImpl(db, sugaredLogger) From a5bf2612c4fbdea916a2c4f73d2174ce6999a313 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 8 Apr 2025 12:10:19 +0530 Subject: [PATCH 14/17] wip --- .../configure/BuildPipelineRestHandler.go | 10 +- .../DeploymentPipelineRestHandler.go | 6 +- .../configure/PipelineConfigRestHandler.go | 12 +- .../trigger/PipelineTriggerRestHandler.go | 8 +- client/cron/CiTriggerCron.go | 8 +- pkg/build/git/gitWebhook/GitWebhookService.go | 8 +- .../trigger/{Service.go => HandlerService.go} | 91 ++++++++------- .../{Service_ent.go => HandlerService_ent.go} | 30 ++--- pkg/build/trigger/wire_trigger.go | 4 +- pkg/bulkAction/service/BulkUpdateService.go | 8 +- .../deployedApp/DeployedAppService.go | 8 +- .../{TriggerService.go => HandlerService.go} | 104 +++++++++--------- ...Service_ent1.go => HandlerService_ent1.go} | 22 ++-- .../devtronApps/PostStageTriggerService.go | 4 +- .../PostStageTriggerService_ent.go | 4 +- .../devtronApps/PrePostCommonService.go | 14 +-- .../devtronApps/PreStageTriggerService.go | 30 ++--- .../devtronApps/PreStageTriggerService_ent.go | 12 +- .../trigger/devtronApps/feasibilityChecker.go | 2 +- .../wire_devtronAppsDeployTrigger.go | 4 +- .../in/CDPipelineEventProcessorService.go | 8 +- .../in/WorkflowEventProcessorService.go | 16 +-- pkg/workflow/dag/WorkflowDagExecutor.go | 28 ++--- wire_gen.go | 34 +++--- 24 files changed, 237 insertions(+), 238 deletions(-) rename pkg/build/trigger/{Service.go => HandlerService.go} (92%) rename pkg/build/trigger/{Service_ent.go => HandlerService_ent.go} (60%) rename pkg/deployment/trigger/devtronApps/{TriggerService.go => HandlerService.go} (96%) rename pkg/deployment/trigger/devtronApps/{TriggerService_ent1.go => HandlerService_ent1.go} (79%) diff --git a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go index 78ddc57454..95cc31f80b 100644 --- a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go @@ -740,7 +740,7 @@ func (handler *PipelineConfigRestHandlerImpl) TriggerCiPipeline(w http.ResponseW ciTriggerRequest.TriggeredBy = userId handler.Logger.Infow("request payload, TriggerCiPipeline", "payload", ciTriggerRequest) response := make(map[string]string) - resp, err := handler.ciTriggerService.HandleCIManual(ciTriggerRequest) + resp, err := handler.ciHandlerService.HandleCIManual(ciTriggerRequest) if errors.Is(err, bean1.ErrImagePathInUse) { handler.Logger.Errorw("service err duplicate image tag, TriggerCiPipeline", "err", err, "payload", ciTriggerRequest) common.WriteJsonResp(w, err, err, http.StatusConflict) @@ -974,7 +974,7 @@ func (handler *PipelineConfigRestHandlerImpl) DownloadCiWorkflowArtifacts(w http } //RBAC - file, err := handler.ciTriggerService.DownloadCiWorkflowArtifacts(pipelineId, buildId) + file, err := handler.ciHandlerService.DownloadCiWorkflowArtifacts(pipelineId, buildId) defer file.Close() if err != nil { handler.Logger.Errorw("service err, DownloadCiWorkflowArtifacts", "err", err, "pipelineId", pipelineId, "buildId", buildId) @@ -1029,7 +1029,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetHistoricBuildLogs(w http.Respon return } //RBAC - resp, err := handler.ciTriggerService.GetHistoricBuildLogs(workflowId, nil) + resp, err := handler.ciHandlerService.GetHistoricBuildLogs(workflowId, nil) if err != nil { handler.Logger.Errorw("service err, GetHistoricBuildLogs", "err", err, "pipelineId", pipelineId, "workflowId", workflowId) common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) @@ -1168,7 +1168,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetBuildLogs(w http.ResponseWriter return } } - logsReader, cleanUp, err := handler.ciTriggerService.GetRunningWorkflowLogs(workflowId) + logsReader, cleanUp, err := handler.ciHandlerService.GetRunningWorkflowLogs(workflowId) if err != nil { handler.Logger.Errorw("service err, GetBuildLogs", "err", err, "pipelineId", pipelineId, "workflowId", workflowId, "lastEventId", lastEventId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) @@ -1611,7 +1611,7 @@ func (handler *PipelineConfigRestHandlerImpl) CancelWorkflow(w http.ResponseWrit //RBAC - resp, err := handler.ciTriggerService.CancelBuild(workflowId, forceAbort) + resp, err := handler.ciHandlerService.CancelBuild(workflowId, forceAbort) if err != nil { handler.Logger.Errorw("service err, CancelWorkflow", "err", err, "workflowId", workflowId, "pipelineId", pipelineId) if util.IsErrNoRows(err) { diff --git a/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go index a856e40c56..79b8a1b980 100644 --- a/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go @@ -1643,7 +1643,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetPrePostDeploymentLogs(w http.Re } //RBAC CHECK - logsReader, cleanUp, err := handler.cdTriggerService.GetRunningWorkflowLogs(environmentId, pipelineId, workflowId) + logsReader, cleanUp, err := handler.cdHandlerService.GetRunningWorkflowLogs(environmentId, pipelineId, workflowId) if err != nil { handler.Logger.Errorw("service err, GetPrePostDeploymentLogs", "err", err, "appId", appId, "environmentId", environmentId, "pipelineId", pipelineId, "workflowId", workflowId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) @@ -1768,7 +1768,7 @@ func (handler *PipelineConfigRestHandlerImpl) DownloadArtifacts(w http.ResponseW } //RBAC CHECK - file, err := handler.cdTriggerService.DownloadCdWorkflowArtifacts(buildId) + file, err := handler.cdHandlerService.DownloadCdWorkflowArtifacts(buildId) defer file.Close() if err != nil { @@ -1952,7 +1952,7 @@ func (handler *PipelineConfigRestHandlerImpl) CancelStage(w http.ResponseWriter, } //RBAC - resp, err := handler.cdTriggerService.CancelStage(workflowRunnerId, forceAbort, userId) + resp, err := handler.cdHandlerService.CancelStage(workflowRunnerId, forceAbort, userId) if err != nil { handler.Logger.Errorw("service err, CancelStage", "err", err, "pipelineId", pipelineId, "workflowRunnerId", workflowRunnerId) if util.IsErrNoRows(err) { diff --git a/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go b/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go index 11de4523aa..7781bbc508 100644 --- a/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go +++ b/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go @@ -140,8 +140,8 @@ type PipelineConfigRestHandlerImpl struct { teamReadService read3.TeamReadService environmentRepository repository2.EnvironmentRepository chartReadService read5.ChartReadService - ciTriggerService trigger.Service - cdTriggerService devtronApps.TriggerService + ciHandlerService trigger.HandlerService + cdHandlerService devtronApps.HandlerService } func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger *zap.SugaredLogger, @@ -176,8 +176,8 @@ func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger teamReadService read3.TeamReadService, EnvironmentRepository repository2.EnvironmentRepository, chartReadService read5.ChartReadService, - ciTriggerService trigger.Service, - cdTriggerService devtronApps.TriggerService, + ciHandlerService trigger.HandlerService, + cdHandlerService devtronApps.HandlerService, ) *PipelineConfigRestHandlerImpl { envConfig := &PipelineRestHandlerEnvConfig{} err := env.Parse(envConfig) @@ -220,8 +220,8 @@ func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger teamReadService: teamReadService, environmentRepository: EnvironmentRepository, chartReadService: chartReadService, - ciTriggerService: ciTriggerService, - cdTriggerService: cdTriggerService, + ciHandlerService: ciHandlerService, + cdHandlerService: cdHandlerService, } } diff --git a/api/restHandler/app/pipeline/trigger/PipelineTriggerRestHandler.go b/api/restHandler/app/pipeline/trigger/PipelineTriggerRestHandler.go index 08b43625b1..8fab862852 100644 --- a/api/restHandler/app/pipeline/trigger/PipelineTriggerRestHandler.go +++ b/api/restHandler/app/pipeline/trigger/PipelineTriggerRestHandler.go @@ -66,7 +66,7 @@ type PipelineTriggerRestHandlerImpl struct { deploymentGroupService deploymentGroup.DeploymentGroupService deploymentConfigService pipeline.PipelineDeploymentConfigService deployedAppService deployedApp.DeployedAppService - cdTriggerService devtronApps.TriggerService + cdHandlerService devtronApps.HandlerService workflowEventPublishService out.WorkflowEventPublishService } @@ -75,7 +75,7 @@ func NewPipelineRestHandler(appService app.AppService, userAuthService user.User deploymentGroupService deploymentGroup.DeploymentGroupService, deploymentConfigService pipeline.PipelineDeploymentConfigService, deployedAppService deployedApp.DeployedAppService, - cdTriggerService devtronApps.TriggerService, + cdHandlerService devtronApps.HandlerService, workflowEventPublishService out.WorkflowEventPublishService) *PipelineTriggerRestHandlerImpl { pipelineHandler := &PipelineTriggerRestHandlerImpl{ appService: appService, @@ -88,7 +88,7 @@ func NewPipelineRestHandler(appService app.AppService, userAuthService user.User deploymentGroupService: deploymentGroupService, deploymentConfigService: deploymentConfigService, deployedAppService: deployedAppService, - cdTriggerService: cdTriggerService, + cdHandlerService: cdHandlerService, workflowEventPublishService: workflowEventPublishService, } return pipelineHandler @@ -140,7 +140,7 @@ func (handler PipelineTriggerRestHandlerImpl) OverrideConfig(w http.ResponseWrit triggerContext := bean3.TriggerContext{ Context: ctx, } - mergeResp, helmPackageName, _, err := handler.cdTriggerService.ManualCdTrigger(triggerContext, &overrideRequest) + mergeResp, helmPackageName, _, err := handler.cdHandlerService.ManualCdTrigger(triggerContext, &overrideRequest) span.End() if err != nil { handler.logger.Errorw("request err, OverrideConfig", "err", err, "payload", overrideRequest) diff --git a/client/cron/CiTriggerCron.go b/client/cron/CiTriggerCron.go index b5b9305afa..5eb7020d36 100644 --- a/client/cron/CiTriggerCron.go +++ b/client/cron/CiTriggerCron.go @@ -42,12 +42,12 @@ type CiTriggerCronImpl struct { pipelineStageRepository repository.PipelineStageRepository ciArtifactRepository repository2.CiArtifactRepository globalPluginRepository repository3.GlobalPluginRepository - ciTriggerService trigger.Service + ciHandlerService trigger.HandlerService } func NewCiTriggerCronImpl(logger *zap.SugaredLogger, cfg *CiTriggerCronConfig, pipelineStageRepository repository.PipelineStageRepository, ciArtifactRepository repository2.CiArtifactRepository, globalPluginRepository repository3.GlobalPluginRepository, cronLogger *cron2.CronLoggerImpl, - ciTriggerService trigger.Service) *CiTriggerCronImpl { + ciHandlerService trigger.HandlerService) *CiTriggerCronImpl { cron := cron.New( cron.WithChain(cron.Recover(cronLogger))) cron.Start() @@ -58,7 +58,7 @@ func NewCiTriggerCronImpl(logger *zap.SugaredLogger, cfg *CiTriggerCronConfig, p cfg: cfg, ciArtifactRepository: ciArtifactRepository, globalPluginRepository: globalPluginRepository, - ciTriggerService: ciTriggerService, + ciHandlerService: ciHandlerService, } _, err := cron.AddFunc(fmt.Sprintf("@every %dm", cfg.SourceControllerCronTime), impl.TriggerCiCron) @@ -104,7 +104,7 @@ func (impl *CiTriggerCronImpl) TriggerCiCron() { InvalidateCache: false, PipelineType: string(common.CI_JOB), } - _, err = impl.ciTriggerService.HandleCIManual(ciTriggerRequest) + _, err = impl.ciHandlerService.HandleCIManual(ciTriggerRequest) if err != nil { return } diff --git a/pkg/build/git/gitWebhook/GitWebhookService.go b/pkg/build/git/gitWebhook/GitWebhookService.go index 8b4405a68b..7ef9e0edc6 100644 --- a/pkg/build/git/gitWebhook/GitWebhookService.go +++ b/pkg/build/git/gitWebhook/GitWebhookService.go @@ -34,15 +34,15 @@ type GitWebhookService interface { type GitWebhookServiceImpl struct { logger *zap.SugaredLogger gitWebhookRepository repository.GitWebhookRepository - ciTriggerService trigger.Service + ciHandlerService trigger.HandlerService } func NewGitWebhookServiceImpl(Logger *zap.SugaredLogger, gitWebhookRepository repository.GitWebhookRepository, - ciTriggerService trigger.Service) *GitWebhookServiceImpl { + ciHandlerService trigger.HandlerService) *GitWebhookServiceImpl { return &GitWebhookServiceImpl{ logger: Logger, gitWebhookRepository: gitWebhookRepository, - ciTriggerService: ciTriggerService, + ciHandlerService: ciHandlerService, } } @@ -71,7 +71,7 @@ func (impl *GitWebhookServiceImpl) HandleGitWebhook(gitWebhookRequest gitSensor. } } - resp, err := impl.ciTriggerService.HandleCIWebhook(bean.GitCiTriggerRequest{ + resp, err := impl.ciHandlerService.HandleCIWebhook(bean.GitCiTriggerRequest{ CiPipelineMaterial: ciPipelineMaterial, TriggeredBy: bean2.SYSTEM_USER_ID, // Automatic trigger, system user ExtraEnvironmentVariables: gitWebhookRequest.ExtraEnvironmentVariables, diff --git a/pkg/build/trigger/Service.go b/pkg/build/trigger/HandlerService.go similarity index 92% rename from pkg/build/trigger/Service.go rename to pkg/build/trigger/HandlerService.go index 0812c485ba..8a4b521666 100644 --- a/pkg/build/trigger/Service.go +++ b/pkg/build/trigger/HandlerService.go @@ -68,12 +68,11 @@ import ( "time" ) -type Service interface { +type HandlerService interface { HandlePodDeleted(ciWorkflow *pipelineConfig.CiWorkflow) CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) - //TriggerCiPipeline(trigger types.Trigger) (int, error) StartCiWorkflowAndPrepareWfRequest(trigger types.Trigger) (*pipelineConfig.CiPipeline, map[string]string, *pipelineConfig.CiWorkflow, *types.WorkflowRequest, error) @@ -88,7 +87,7 @@ type BuildxCacheFlags struct { AsyncBuildxCacheExport bool `env:"ASYNC_BUILDX_CACHE_EXPORT" envDefault:"false"` } -type ServiceImpl struct { +type HandlerServiceImpl struct { Logger *zap.SugaredLogger workflowService executor.WorkflowService ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository @@ -118,7 +117,7 @@ type ServiceImpl struct { K8sUtil *k8s.K8sServiceImpl } -func NewServiceImpl(Logger *zap.SugaredLogger, workflowService executor.WorkflowService, +func NewHandlerServiceImpl(Logger *zap.SugaredLogger, workflowService executor.WorkflowService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, ciPipelineRepository pipelineConfig.CiPipelineRepository, ciArtifactRepository repository5.CiArtifactRepository, @@ -141,13 +140,13 @@ func NewServiceImpl(Logger *zap.SugaredLogger, workflowService executor.Workflow clusterService cluster.ClusterService, envService environment.EnvironmentService, K8sUtil *k8s.K8sServiceImpl, -) *ServiceImpl { +) *HandlerServiceImpl { buildxCacheFlags := &BuildxCacheFlags{} err := env.Parse(buildxCacheFlags) if err != nil { Logger.Infow("error occurred while parsing BuildxCacheFlags env,so setting BuildxCacheModeMin and AsyncBuildxCacheExport to default value", "err", err) } - cis := &ServiceImpl{ + cis := &HandlerServiceImpl{ Logger: Logger, workflowService: workflowService, ciPipelineMaterialRepository: ciPipelineMaterialRepository, @@ -183,7 +182,7 @@ func NewServiceImpl(Logger *zap.SugaredLogger, workflowService executor.Workflow return cis } -func (impl *ServiceImpl) HandlePodDeleted(ciWorkflow *pipelineConfig.CiWorkflow) { +func (impl *HandlerServiceImpl) HandlePodDeleted(ciWorkflow *pipelineConfig.CiWorkflow) { if !impl.config.WorkflowRetriesEnabled() { impl.Logger.Debug("ci workflow retry feature disabled") return @@ -199,7 +198,7 @@ func (impl *ServiceImpl) HandlePodDeleted(ciWorkflow *pipelineConfig.CiWorkflow) } } -func (impl *ServiceImpl) CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error { +func (impl *HandlerServiceImpl) CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error { // return if re-trigger feature is disabled if !impl.config.WorkflowRetriesEnabled() { @@ -233,7 +232,7 @@ func (impl *ServiceImpl) CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowSta return err } -func (impl *ServiceImpl) reTriggerCi(retryCount int, refCiWorkflow *pipelineConfig.CiWorkflow) error { +func (impl *HandlerServiceImpl) reTriggerCi(retryCount int, refCiWorkflow *pipelineConfig.CiWorkflow) error { if retryCount >= impl.config.MaxCiWorkflowRetries { impl.Logger.Infow("maximum retries exhausted for this ciWorkflow", "ciWorkflowId", refCiWorkflow.Id, "retries", retryCount, "configuredRetries", impl.config.MaxCiWorkflowRetries) return nil @@ -268,7 +267,7 @@ func (impl *ServiceImpl) reTriggerCi(retryCount int, refCiWorkflow *pipelineConf return nil } -func (impl *ServiceImpl) HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) { +func (impl *HandlerServiceImpl) HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) { impl.Logger.Debugw("HandleCIManual for pipeline ", "PipelineId", ciTriggerRequest.PipelineId) commitHashes, runtimeParams, err := impl.buildManualTriggerCommitHashes(ciTriggerRequest) if err != nil { @@ -305,7 +304,7 @@ func (impl *ServiceImpl) HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) return id, nil } -func (impl *ServiceImpl) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) { +func (impl *HandlerServiceImpl) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) { impl.Logger.Debugw("HandleCIWebhook for material ", "material", gitCiTriggerRequest.CiPipelineMaterial) ciPipeline, err := impl.GetCiPipeline(gitCiTriggerRequest.CiPipelineMaterial.Id) if err != nil { @@ -357,7 +356,7 @@ func (impl *ServiceImpl) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRe return id, nil } -func (impl *ServiceImpl) extractPodStatusAndWorkflow(workflowStatus v1alpha1.WorkflowStatus) (string, string, *pipelineConfig.CiWorkflow, error) { +func (impl *HandlerServiceImpl) extractPodStatusAndWorkflow(workflowStatus v1alpha1.WorkflowStatus) (string, string, *pipelineConfig.CiWorkflow, error) { workflowName, status, _, message, _, _ := pipeline2.ExtractWorkflowStatus(workflowStatus) if workflowName == "" { impl.Logger.Errorw("extract workflow status, invalid wf name", "workflowName", workflowName, "status", status, "message", message) @@ -379,7 +378,7 @@ func (impl *ServiceImpl) extractPodStatusAndWorkflow(workflowStatus v1alpha1.Wor } -func (impl *ServiceImpl) getRefWorkflowAndCiRetryCount(savedWorkflow *pipelineConfig.CiWorkflow) (int, *pipelineConfig.CiWorkflow, error) { +func (impl *HandlerServiceImpl) getRefWorkflowAndCiRetryCount(savedWorkflow *pipelineConfig.CiWorkflow) (int, *pipelineConfig.CiWorkflow, error) { var err error if savedWorkflow.ReferenceCiWorkflowId != 0 { @@ -393,7 +392,7 @@ func (impl *ServiceImpl) getRefWorkflowAndCiRetryCount(savedWorkflow *pipelineCo return retryCount, savedWorkflow, err } -func (impl *ServiceImpl) validateBuildSequence(gitCiTriggerRequest bean.GitCiTriggerRequest, pipelineId int) (bool, error) { +func (impl *HandlerServiceImpl) validateBuildSequence(gitCiTriggerRequest bean.GitCiTriggerRequest, pipelineId int) (bool, error) { isValid := true lastTriggeredBuild, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflow(pipelineId) if !(lastTriggeredBuild.Status == string(v1alpha1.NodePending) || lastTriggeredBuild.Status == string(v1alpha1.NodeRunning)) { @@ -416,7 +415,7 @@ func (impl *ServiceImpl) validateBuildSequence(gitCiTriggerRequest bean.GitCiTri return isValid, nil } -func (impl *ServiceImpl) buildAutomaticTriggerCommitHashes(ciMaterials []*pipelineConfig.CiPipelineMaterial, request bean.GitCiTriggerRequest) (map[int]pipelineConfig.GitCommit, error) { +func (impl *HandlerServiceImpl) buildAutomaticTriggerCommitHashes(ciMaterials []*pipelineConfig.CiPipelineMaterial, request bean.GitCiTriggerRequest) (map[int]pipelineConfig.GitCommit, error) { commitHashes := map[int]pipelineConfig.GitCommit{} for _, ciMaterial := range ciMaterials { if ciMaterial.Id == request.CiPipelineMaterial.Id || len(ciMaterials) == 1 { @@ -435,7 +434,7 @@ func (impl *ServiceImpl) buildAutomaticTriggerCommitHashes(ciMaterials []*pipeli return commitHashes, nil } -func (impl *ServiceImpl) GetCiPipeline(ciMaterialId int) (*pipelineConfig.CiPipeline, error) { +func (impl *HandlerServiceImpl) GetCiPipeline(ciMaterialId int) (*pipelineConfig.CiPipeline, error) { ciMaterial, err := impl.ciPipelineMaterialRepository.GetById(ciMaterialId) if err != nil { return nil, err @@ -444,7 +443,7 @@ func (impl *ServiceImpl) GetCiPipeline(ciMaterialId int) (*pipelineConfig.CiPipe return ciPipeline, nil } -func (impl *ServiceImpl) buildManualTriggerCommitHashes(ciTriggerRequest bean.CiTriggerRequest) (map[int]pipelineConfig.GitCommit, *common.RuntimeParameters, error) { +func (impl *HandlerServiceImpl) buildManualTriggerCommitHashes(ciTriggerRequest bean.CiTriggerRequest) (map[int]pipelineConfig.GitCommit, *common.RuntimeParameters, error) { commitHashes := map[int]pipelineConfig.GitCommit{} runtimeParams := impl.getRuntimeParamsForBuildingManualTriggerHashes(ciTriggerRequest) for _, ciPipelineMaterial := range ciTriggerRequest.CiPipelineMaterial { @@ -479,7 +478,7 @@ func (impl *ServiceImpl) buildManualTriggerCommitHashes(ciTriggerRequest bean.Ci return commitHashes, runtimeParams, nil } -func (impl *ServiceImpl) BuildManualTriggerCommitHashesForSourceTypeBranchFix(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (pipelineConfig.GitCommit, error) { +func (impl *HandlerServiceImpl) BuildManualTriggerCommitHashesForSourceTypeBranchFix(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (pipelineConfig.GitCommit, error) { commitMetadataRequest := &gitSensor.CommitMetadataRequest{ PipelineMaterialId: ciPipelineMaterial.Id, GitHash: ciPipelineMaterial.GitCommit.Commit, @@ -509,7 +508,7 @@ func (impl *ServiceImpl) BuildManualTriggerCommitHashesForSourceTypeBranchFix(ci return gitCommit, nil } -func (impl *ServiceImpl) BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (pipelineConfig.GitCommit, map[string]string, error) { +func (impl *HandlerServiceImpl) BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (pipelineConfig.GitCommit, map[string]string, error) { webhookDataInput := ciPipelineMaterial.GitCommit.WebhookData // fetch webhook data on the basis of Id @@ -569,7 +568,7 @@ func (impl *ServiceImpl) BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPi return gitCommit, webhookAndCiData.ExtraEnvironmentVariables, nil } -func (impl *ServiceImpl) getLastSeenCommit(ciMaterialId int) (pipelineConfig.GitCommit, error) { +func (impl *HandlerServiceImpl) getLastSeenCommit(ciMaterialId int) (pipelineConfig.GitCommit, error) { var materialIds []int materialIds = append(materialIds, ciMaterialId) headReq := &gitSensor.HeadRequest{ @@ -601,7 +600,7 @@ func SetGitCommitValuesForBuildingCommitHash(ciMaterial *pipelineConfig.CiPipeli return newGitCommit } -func (impl *ServiceImpl) triggerCiPipeline(trigger types.Trigger) (int, error) { +func (impl *HandlerServiceImpl) triggerCiPipeline(trigger types.Trigger) (int, error) { pipeline, variableSnapshot, savedCiWf, workflowRequest, err := impl.StartCiWorkflowAndPrepareWfRequest(trigger) if err != nil { return 0, err @@ -632,7 +631,7 @@ func (impl *ServiceImpl) triggerCiPipeline(trigger types.Trigger) (int, error) { return savedCiWf.Id, err } -func (impl *ServiceImpl) GetCiMaterials(pipelineId int, ciMaterials []*pipelineConfig.CiPipelineMaterial) ([]*pipelineConfig.CiPipelineMaterial, error) { +func (impl *HandlerServiceImpl) GetCiMaterials(pipelineId int, ciMaterials []*pipelineConfig.CiPipelineMaterial) ([]*pipelineConfig.CiPipelineMaterial, error) { if !(len(ciMaterials) == 0) { return ciMaterials, nil } else { @@ -646,7 +645,7 @@ func (impl *ServiceImpl) GetCiMaterials(pipelineId int, ciMaterials []*pipelineC } } -func (impl *ServiceImpl) StartCiWorkflowAndPrepareWfRequest(trigger types.Trigger) (*pipelineConfig.CiPipeline, map[string]string, *pipelineConfig.CiWorkflow, *types.WorkflowRequest, error) { +func (impl *HandlerServiceImpl) StartCiWorkflowAndPrepareWfRequest(trigger types.Trigger) (*pipelineConfig.CiPipeline, map[string]string, *pipelineConfig.CiWorkflow, *types.WorkflowRequest, error) { impl.Logger.Debugw("ci pipeline manual trigger", "request", trigger) ciMaterials, err := impl.GetCiMaterials(trigger.PipelineId, trigger.CiMaterials) if err != nil { @@ -783,7 +782,7 @@ func (impl *ServiceImpl) StartCiWorkflowAndPrepareWfRequest(trigger types.Trigge return pipeline, variableSnapshot, savedCiWf, workflowRequest, nil } -func (impl *ServiceImpl) setBuildxK8sDriverData(workflowRequest *types.WorkflowRequest) error { +func (impl *HandlerServiceImpl) setBuildxK8sDriverData(workflowRequest *types.WorkflowRequest) error { dockerBuildConfig := workflowRequest.CiBuildConfig.DockerBuildConfig k8sDriverOptions, err := impl.getK8sDriverOptions(workflowRequest, dockerBuildConfig.TargetPlatform) if err != nil { @@ -793,7 +792,7 @@ func (impl *ServiceImpl) setBuildxK8sDriverData(workflowRequest *types.WorkflowR return nil } -func (impl *ServiceImpl) getEnvironmentForJob(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger) (*repository6.Environment, bool, error) { +func (impl *HandlerServiceImpl) getEnvironmentForJob(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger) (*repository6.Environment, bool, error) { app, err := impl.appRepository.FindById(pipeline.AppId) if err != nil { impl.Logger.Errorw("could not find app", "err", err) @@ -817,14 +816,14 @@ func (impl *ServiceImpl) getEnvironmentForJob(pipeline *pipelineConfig.CiPipelin } // TODO: Send all trigger data -func (impl *ServiceImpl) BuildPayload(trigger types.Trigger, pipeline *pipelineConfig.CiPipeline) *client.Payload { +func (impl *HandlerServiceImpl) BuildPayload(trigger types.Trigger, pipeline *pipelineConfig.CiPipeline) *client.Payload { payload := &client.Payload{} payload.AppName = pipeline.App.AppName payload.PipelineName = pipeline.Name return payload } -func (impl *ServiceImpl) saveNewWorkflowForCITrigger(pipeline *pipelineConfig.CiPipeline, ciWorkflowConfigNamespace string, +func (impl *HandlerServiceImpl) saveNewWorkflowForCITrigger(pipeline *pipelineConfig.CiPipeline, ciWorkflowConfigNamespace string, commitHashes map[int]pipelineConfig.GitCommit, userId int32, ciMaterials []*pipelineConfig.CiPipelineMaterial, EnvironmentId int, isJob bool, refCiWorkflowId int) (*pipelineConfig.CiWorkflow, error) { isCiTriggerBlocked, err := impl.checkIfCITriggerIsBlocked(pipeline, ciMaterials, isJob) @@ -862,7 +861,7 @@ func (impl *ServiceImpl) saveNewWorkflowForCITrigger(pipeline *pipelineConfig.Ci return ciWorkflow, nil } -func (impl *ServiceImpl) executeCiPipeline(workflowRequest *types.WorkflowRequest) error { +func (impl *HandlerServiceImpl) executeCiPipeline(workflowRequest *types.WorkflowRequest) error { _, _, err := impl.workflowService.SubmitWorkflow(workflowRequest) if err != nil { impl.Logger.Errorw("workflow error", "err", err) @@ -871,20 +870,20 @@ func (impl *ServiceImpl) executeCiPipeline(workflowRequest *types.WorkflowReques return nil } -func (impl *ServiceImpl) buildS3ArtifactLocation(ciWorkflowConfigLogsBucket string, savedWf *pipelineConfig.CiWorkflow) (string, string, string) { +func (impl *HandlerServiceImpl) buildS3ArtifactLocation(ciWorkflowConfigLogsBucket string, savedWf *pipelineConfig.CiWorkflow) (string, string, string) { ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() ArtifactLocation := fmt.Sprintf("s3://"+path.Join(ciWorkflowConfigLogsBucket, ciArtifactLocationFormat), savedWf.Id, savedWf.Id) artifactFileName := fmt.Sprintf(ciArtifactLocationFormat, savedWf.Id, savedWf.Id) return ArtifactLocation, ciWorkflowConfigLogsBucket, artifactFileName } -func (impl *ServiceImpl) buildDefaultArtifactLocation(savedWf *pipelineConfig.CiWorkflow) string { +func (impl *HandlerServiceImpl) buildDefaultArtifactLocation(savedWf *pipelineConfig.CiWorkflow) string { ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() ArtifactLocation := fmt.Sprintf(ciArtifactLocationFormat, savedWf.Id, savedWf.Id) return ArtifactLocation } -func (impl *ServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, savedWf *pipelineConfig.CiWorkflow, ciWorkflowConfigNamespace string, ciPipelineScripts []*pipelineConfig.CiPipelineScript, preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, refPluginsData []*pipelineConfigBean.RefPluginObject, isJob bool) (*types.WorkflowRequest, error) { +func (impl *HandlerServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, savedWf *pipelineConfig.CiWorkflow, ciWorkflowConfigNamespace string, ciPipelineScripts []*pipelineConfig.CiPipelineScript, preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, refPluginsData []*pipelineConfigBean.RefPluginObject, isJob bool) (*types.WorkflowRequest, error) { var ciProjectDetails []pipelineConfigBean.CiProjectDetails commitHashes := trigger.CommitHashes for _, ciMaterial := range ciMaterials { @@ -1263,7 +1262,7 @@ func (impl *ServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.Ci return workflowRequest, nil } -func (impl *ServiceImpl) GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, customTag string, customTagId int, buildImagePath string, buildImagedockerRegistryId string) (map[string][]string, map[string]bean2.RegistryCredentials, string, []int, error) { +func (impl *HandlerServiceImpl) GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, customTag string, customTagId int, buildImagePath string, buildImagedockerRegistryId string) (map[string][]string, map[string]bean2.RegistryCredentials, string, []int, error) { copyContainerImagePluginDetail, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(buildCommonBean.COPY_CONTAINER_IMAGE) if err != nil && err != pg.ErrNoRows { @@ -1334,7 +1333,7 @@ func (impl *ServiceImpl) GetWorkflowRequestVariablesForCopyContainerImagePlugin( return registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imagePathReservationIds, nil } -func (impl *ServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, destinationImages []string) ([]int, error) { +func (impl *HandlerServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, destinationImages []string) ([]int, error) { var imagePathReservationIds []int for _, image := range destinationImages { imagePathReservationData, err := impl.customTagService.ReserveImagePath(image, customTagId) @@ -1392,7 +1391,7 @@ func buildCiStepsDataFromDockerBuildScripts(dockerBuildScripts []*bean.CiScript) return ciSteps } -func (impl *ServiceImpl) buildImageTag(commitHashes map[int]pipelineConfig.GitCommit, id int, wfId int) string { +func (impl *HandlerServiceImpl) buildImageTag(commitHashes map[int]pipelineConfig.GitCommit, id int, wfId int) string { dockerImageTag := "" toAppendDevtronParamInTag := true for _, v := range commitHashes { @@ -1441,14 +1440,14 @@ func getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, commitOrCh return dockerImageTag } -func (impl *ServiceImpl) updateCiWorkflow(request *types.WorkflowRequest, savedWf *pipelineConfig.CiWorkflow) error { +func (impl *HandlerServiceImpl) updateCiWorkflow(request *types.WorkflowRequest, savedWf *pipelineConfig.CiWorkflow) error { ciBuildConfig := request.CiBuildConfig ciBuildType := string(ciBuildConfig.CiBuildType) savedWf.CiBuildType = ciBuildType return impl.ciService.UpdateCiWorkflowWithStage(savedWf) } -func (impl *ServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, workflowRequest *types.WorkflowRequest) error { +func (impl *HandlerServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, workflowRequest *types.WorkflowRequest) error { // externalCi artifact is meant only for CI_JOB if trigger.PipelineType != string(buildCommonBean.CI_JOB) { return nil @@ -1534,7 +1533,7 @@ func _getTruncatedImageTag(imageTag string) string { } } -func (impl *ServiceImpl) markCurrentCiWorkflowFailed(savedCiWf *pipelineConfig.CiWorkflow, validationErr error) error { +func (impl *HandlerServiceImpl) markCurrentCiWorkflowFailed(savedCiWf *pipelineConfig.CiWorkflow, validationErr error) error { // currently such requirement is not there if savedCiWf == nil { return nil @@ -1562,7 +1561,7 @@ func (impl *ServiceImpl) markCurrentCiWorkflowFailed(savedCiWf *pipelineConfig.C return nil } -func (impl *ServiceImpl) CancelBuild(workflowId int, forceAbort bool) (int, error) { +func (impl *HandlerServiceImpl) CancelBuild(workflowId int, forceAbort bool) (int, error) { workflow, err := impl.ciWorkflowRepository.FindById(workflowId) if err != nil { impl.Logger.Errorw("error in finding ci-workflow by workflow id", "ciWorkflowId", workflowId, "err", err) @@ -1639,7 +1638,7 @@ func (impl *ServiceImpl) CancelBuild(workflowId int, forceAbort bool) (int, erro return workflow.Id, nil } -func (impl *ServiceImpl) handleForceAbortCaseForCi(workflow *pipelineConfig.CiWorkflow, forceAbort bool) error { +func (impl *HandlerServiceImpl) handleForceAbortCaseForCi(workflow *pipelineConfig.CiWorkflow, forceAbort bool) error { isWorkflowInNonTerminalStage := workflow.Status == string(v1alpha1.NodePending) || workflow.Status == string(v1alpha1.NodeRunning) if !isWorkflowInNonTerminalStage { if forceAbort { @@ -1655,7 +1654,7 @@ func (impl *ServiceImpl) handleForceAbortCaseForCi(workflow *pipelineConfig.CiWo return nil } -func (impl *ServiceImpl) updateWorkflowForForceAbort(workflow *pipelineConfig.CiWorkflow) error { +func (impl *HandlerServiceImpl) updateWorkflowForForceAbort(workflow *pipelineConfig.CiWorkflow) error { workflow.Status = cdWorkflow.WorkflowCancel workflow.PodStatus = string(bean.Failed) workflow.Message = constants2.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE @@ -1667,7 +1666,7 @@ func (impl *ServiceImpl) updateWorkflowForForceAbort(workflow *pipelineConfig.Ci return nil } -func (impl *ServiceImpl) getRestConfig(workflow *pipelineConfig.CiWorkflow) (*rest.Config, error) { +func (impl *HandlerServiceImpl) getRestConfig(workflow *pipelineConfig.CiWorkflow) (*rest.Config, error) { env, err := impl.envRepository.FindById(workflow.EnvironmentId) if err != nil { impl.Logger.Errorw("could not fetch stage env", "err", err) @@ -1685,7 +1684,7 @@ func (impl *ServiceImpl) getRestConfig(workflow *pipelineConfig.CiWorkflow) (*re return restConfig, nil } -func (impl *ServiceImpl) GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, func() error, error) { +func (impl *HandlerServiceImpl) GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, func() error, error) { ciWorkflow, err := impl.ciWorkflowRepository.FindById(workflowId) if err != nil { impl.Logger.Errorw("err", "err", err) @@ -1694,7 +1693,7 @@ func (impl *ServiceImpl) GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, return impl.getWorkflowLogs(ciWorkflow) } -func (impl *ServiceImpl) getWorkflowLogs(ciWorkflow *pipelineConfig.CiWorkflow) (*bufio.Reader, func() error, error) { +func (impl *HandlerServiceImpl) getWorkflowLogs(ciWorkflow *pipelineConfig.CiWorkflow) (*bufio.Reader, func() error, error) { if string(v1alpha1.NodePending) == ciWorkflow.PodStatus { return bufio.NewReader(strings.NewReader("")), func() error { return nil }, nil } @@ -1736,7 +1735,7 @@ func (impl *ServiceImpl) getWorkflowLogs(ciWorkflow *pipelineConfig.CiWorkflow) return logReader, cleanUp, err } -func (impl *ServiceImpl) getLogsFromRepository(ciWorkflow *pipelineConfig.CiWorkflow, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { +func (impl *HandlerServiceImpl) getLogsFromRepository(ciWorkflow *pipelineConfig.CiWorkflow, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { impl.Logger.Debug("getting historic logs", "ciWorkflowId", ciWorkflow.Id) ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion @@ -1791,7 +1790,7 @@ func (impl *ServiceImpl) getLogsFromRepository(ciWorkflow *pipelineConfig.CiWork return logReader, cleanUp, err } -func (impl *ServiceImpl) GetHistoricBuildLogs(workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) { +func (impl *HandlerServiceImpl) GetHistoricBuildLogs(workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) { var err error if ciWorkflow == nil { ciWorkflow, err = impl.ciWorkflowRepository.FindById(workflowId) @@ -1863,7 +1862,7 @@ func (impl *ServiceImpl) GetHistoricBuildLogs(workflowId int, ciWorkflow *pipeli return resp, err } -func (impl *ServiceImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) { +func (impl *HandlerServiceImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) { ciWorkflow, err := impl.ciWorkflowRepository.FindById(buildId) if err != nil { impl.Logger.Errorw("unable to fetch ciWorkflow", "err", err) diff --git a/pkg/build/trigger/Service_ent.go b/pkg/build/trigger/HandlerService_ent.go similarity index 60% rename from pkg/build/trigger/Service_ent.go rename to pkg/build/trigger/HandlerService_ent.go index 0844a5986b..3938288091 100644 --- a/pkg/build/trigger/Service_ent.go +++ b/pkg/build/trigger/HandlerService_ent.go @@ -17,67 +17,67 @@ import ( "net/http" ) -func (impl *ServiceImpl) updateRuntimeParamsForAutoCI(ciPipelineId int, runtimeParameters *common.RuntimeParameters) (*common.RuntimeParameters, error) { +func (impl *HandlerServiceImpl) updateRuntimeParamsForAutoCI(ciPipelineId int, runtimeParameters *common.RuntimeParameters) (*common.RuntimeParameters, error) { return runtimeParameters, nil } -func (impl *ServiceImpl) getRuntimeParamsForBuildingManualTriggerHashes(ciTriggerRequest bean3.CiTriggerRequest) *common.RuntimeParameters { +func (impl *HandlerServiceImpl) getRuntimeParamsForBuildingManualTriggerHashes(ciTriggerRequest bean3.CiTriggerRequest) *common.RuntimeParameters { return common.NewRuntimeParameters() } -func (impl *ServiceImpl) fetchImageScanExecutionMedium() (*repository.ScanToolMetadata, bean.ScanExecutionMedium, error) { +func (impl *HandlerServiceImpl) fetchImageScanExecutionMedium() (*repository.ScanToolMetadata, bean.ScanExecutionMedium, error) { return &repository.ScanToolMetadata{}, "", nil } -func (impl *ServiceImpl) fetchImageScanExecutionStepsForWfRequest(scanToolMetadata *repository.ScanToolMetadata) ([]*types.ImageScanningSteps, []*pipelineConfigBean.RefPluginObject, error) { +func (impl *HandlerServiceImpl) fetchImageScanExecutionStepsForWfRequest(scanToolMetadata *repository.ScanToolMetadata) ([]*types.ImageScanningSteps, []*pipelineConfigBean.RefPluginObject, error) { return nil, nil, nil } -func (impl *ServiceImpl) checkIfCITriggerIsBlocked(pipeline *pipelineConfig.CiPipeline, +func (impl *HandlerServiceImpl) checkIfCITriggerIsBlocked(pipeline *pipelineConfig.CiPipeline, ciMaterials []*pipelineConfig.CiPipelineMaterial, isJob bool) (bool, error) { return false, nil } -func (impl *ServiceImpl) handleWFIfCITriggerIsBlocked(ciWorkflow *pipelineConfig.CiWorkflow) (*pipelineConfig.CiWorkflow, error) { +func (impl *HandlerServiceImpl) handleWFIfCITriggerIsBlocked(ciWorkflow *pipelineConfig.CiWorkflow) (*pipelineConfig.CiWorkflow, error) { impl.Logger.Errorw("cannot trigger pipeline, blocked by mandatory plugin policy", "ciPipelineId", ciWorkflow.CiPipelineId) return &pipelineConfig.CiWorkflow{}, util.GetApiErrorAdapter(http.StatusInternalServerError, "500", "Invalid flow access, corrupt data possibility", "Invalid flow access, corrupt data possibility") } -func (impl *ServiceImpl) checkArgoSetupRequirement(envModal *repository2.Environment) error { +func (impl *HandlerServiceImpl) checkArgoSetupRequirement(envModal *repository2.Environment) error { return nil } -func (impl *ServiceImpl) updateWorkflowRequestForDigestPull(pipelineId int, workflowRequest *types.WorkflowRequest) (*types.WorkflowRequest, error) { +func (impl *HandlerServiceImpl) updateWorkflowRequestForDigestPull(pipelineId int, workflowRequest *types.WorkflowRequest) (*types.WorkflowRequest, error) { return workflowRequest, nil } -func (impl *ServiceImpl) updateCIProjectDetailWithCloningMode(appId int, ciMaterial *pipelineConfig.CiPipelineMaterial, +func (impl *HandlerServiceImpl) updateCIProjectDetailWithCloningMode(appId int, ciMaterial *pipelineConfig.CiPipelineMaterial, ciProjectDetail pipelineConfigBean.CiProjectDetails) (pipelineConfigBean.CiProjectDetails, error) { return ciProjectDetail, nil } -func (impl *ServiceImpl) updateWorkflowRequestWithRemoteConnConf(dockerRegistry *repository3.DockerArtifactStore, +func (impl *HandlerServiceImpl) updateWorkflowRequestWithRemoteConnConf(dockerRegistry *repository3.DockerArtifactStore, workflowRequest *types.WorkflowRequest) (*types.WorkflowRequest, error) { return workflowRequest, nil } -func (impl *ServiceImpl) updateWorkflowRequestWithEntSupportData(workflowRequest *types.WorkflowRequest) *types.WorkflowRequest { +func (impl *HandlerServiceImpl) updateWorkflowRequestWithEntSupportData(workflowRequest *types.WorkflowRequest) *types.WorkflowRequest { return workflowRequest } -func (impl *ServiceImpl) updateWorkflowRequestWithBuildCacheData(workflowRequest *types.WorkflowRequest, +func (impl *HandlerServiceImpl) updateWorkflowRequestWithBuildCacheData(workflowRequest *types.WorkflowRequest, scope resourceQualifiers.Scope) (*types.WorkflowRequest, error) { workflowRequest.BuildxCacheModeMin = impl.buildxCacheFlags.BuildxCacheModeMin workflowRequest.AsyncBuildxCacheExport = impl.buildxCacheFlags.AsyncBuildxCacheExport return workflowRequest, nil } -func (impl *ServiceImpl) canSetK8sDriverData(workflowRequest *types.WorkflowRequest) bool { +func (impl *HandlerServiceImpl) canSetK8sDriverData(workflowRequest *types.WorkflowRequest) bool { return impl.config != nil && impl.config.BuildxK8sDriverOptions != "" && workflowRequest.CiBuildConfig != nil && workflowRequest.CiBuildConfig.DockerBuildConfig != nil } -func (impl *ServiceImpl) getK8sDriverOptions(workflowRequest *types.WorkflowRequest, targetPlatforms string) ([]map[string]string, error) { +func (impl *HandlerServiceImpl) getK8sDriverOptions(workflowRequest *types.WorkflowRequest, targetPlatforms string) ([]map[string]string, error) { buildxK8sDriverOptions := make([]map[string]string, 0) err := json.Unmarshal([]byte(impl.config.BuildxK8sDriverOptions), &buildxK8sDriverOptions) if err != nil { @@ -86,7 +86,7 @@ func (impl *ServiceImpl) getK8sDriverOptions(workflowRequest *types.WorkflowRequ return buildxK8sDriverOptions, nil } -func (impl *ServiceImpl) updateCIBuildConfig(ciBuildConfigBean *bean2.CiBuildConfigBean) *bean2.CiBuildConfigBean { +func (impl *HandlerServiceImpl) updateCIBuildConfig(ciBuildConfigBean *bean2.CiBuildConfigBean) *bean2.CiBuildConfigBean { defaultTargetPlatform := impl.config.DefaultTargetPlatform useBuildx := impl.config.UseBuildx if ciBuildConfigBean.DockerBuildConfig != nil { diff --git a/pkg/build/trigger/wire_trigger.go b/pkg/build/trigger/wire_trigger.go index dc462bcd88..aca718e3cd 100644 --- a/pkg/build/trigger/wire_trigger.go +++ b/pkg/build/trigger/wire_trigger.go @@ -5,6 +5,6 @@ import ( ) var WireSet = wire.NewSet( - NewServiceImpl, - wire.Bind(new(Service), new(*ServiceImpl)), + NewHandlerServiceImpl, + wire.Bind(new(HandlerService), new(*HandlerServiceImpl)), ) diff --git a/pkg/bulkAction/service/BulkUpdateService.go b/pkg/bulkAction/service/BulkUpdateService.go index afd8efeb41..3e6c0f7f78 100644 --- a/pkg/bulkAction/service/BulkUpdateService.go +++ b/pkg/bulkAction/service/BulkUpdateService.go @@ -103,7 +103,7 @@ type BulkUpdateServiceImpl struct { chartRefService chartRef.ChartRefService deployedAppService deployedApp.DeployedAppService cdPipelineEventPublishService out.CDPipelineEventPublishService - ciTriggerService trigger.Service + ciHandlerService trigger.HandlerService } func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateRepository, @@ -124,7 +124,7 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateReposito chartRefService chartRef.ChartRefService, deployedAppService deployedApp.DeployedAppService, cdPipelineEventPublishService out.CDPipelineEventPublishService, - ciTriggerService trigger.Service) *BulkUpdateServiceImpl { + ciHandlerService trigger.HandlerService) *BulkUpdateServiceImpl { return &BulkUpdateServiceImpl{ bulkUpdateRepository: bulkUpdateRepository, logger: logger, @@ -144,7 +144,7 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateReposito chartRefService: chartRefService, deployedAppService: deployedAppService, cdPipelineEventPublishService: cdPipelineEventPublishService, - ciTriggerService: ciTriggerService, + ciHandlerService: ciHandlerService, } } @@ -1471,7 +1471,7 @@ func (impl BulkUpdateServiceImpl) BulkBuildTrigger(request *bean4.BulkApplicatio } ciTriggerRequest := latestCommitsMap[pipeline.CiPipelineId] - _, err = impl.ciTriggerService.HandleCIManual(ciTriggerRequest) + _, err = impl.ciHandlerService.HandleCIManual(ciTriggerRequest) if err != nil { impl.logger.Errorw("service err, HandleCIManual", "err", err, "ciTriggerRequest", ciTriggerRequest) //return nil, err diff --git a/pkg/deployment/deployedApp/DeployedAppService.go b/pkg/deployment/deployedApp/DeployedAppService.go index f8c65d2373..83107b22fa 100644 --- a/pkg/deployment/deployedApp/DeployedAppService.go +++ b/pkg/deployment/deployedApp/DeployedAppService.go @@ -46,7 +46,7 @@ type DeployedAppService interface { type DeployedAppServiceImpl struct { logger *zap.SugaredLogger k8sCommonService k8s.K8sCommonService - cdTriggerService devtronApps.TriggerService + cdHandlerService devtronApps.HandlerService envRepository repository.EnvironmentRepository pipelineRepository pipelineConfig.PipelineRepository cdWorkflowRepository pipelineConfig.CdWorkflowRepository @@ -54,14 +54,14 @@ type DeployedAppServiceImpl struct { func NewDeployedAppServiceImpl(logger *zap.SugaredLogger, k8sCommonService k8s.K8sCommonService, - cdTriggerService devtronApps.TriggerService, + cdHandlerService devtronApps.HandlerService, envRepository repository.EnvironmentRepository, pipelineRepository pipelineConfig.PipelineRepository, cdWorkflowRepository pipelineConfig.CdWorkflowRepository) *DeployedAppServiceImpl { return &DeployedAppServiceImpl{ logger: logger, k8sCommonService: k8sCommonService, - cdTriggerService: cdTriggerService, + cdHandlerService: cdHandlerService, envRepository: envRepository, pipelineRepository: pipelineRepository, cdWorkflowRepository: cdWorkflowRepository, @@ -127,7 +127,7 @@ func (impl *DeployedAppServiceImpl) stopStartApp(ctx context.Context, stopReques Context: ctx, ReferenceId: stopRequest.ReferenceId, } - id, _, _, err := impl.cdTriggerService.ManualCdTrigger(triggerContext, overrideRequest) + id, _, _, err := impl.cdHandlerService.ManualCdTrigger(triggerContext, overrideRequest) if err != nil { impl.logger.Errorw("error in stopping app", "err", err, "appId", stopRequest.AppId, "envId", stopRequest.EnvironmentId) return 0, err diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/HandlerService.go similarity index 96% rename from pkg/deployment/trigger/devtronApps/TriggerService.go rename to pkg/deployment/trigger/devtronApps/HandlerService.go index bcef986aa7..7c915a3195 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService.go +++ b/pkg/deployment/trigger/devtronApps/HandlerService.go @@ -103,7 +103,7 @@ import ( "time" ) -type TriggerService interface { +type HandlerService interface { TriggerPostStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) TriggerPreStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) @@ -121,7 +121,7 @@ type TriggerService interface { GetRunningWorkflowLogs(environmentId int, pipelineId int, workflowId int) (*bufio.Reader, func() error, error) } -type TriggerServiceImpl struct { +type HandlerServiceImpl struct { logger *zap.SugaredLogger cdWorkflowCommonService cd.CdWorkflowCommonService gitOpsManifestPushService publish.GitOpsPushService @@ -184,7 +184,7 @@ type TriggerServiceImpl struct { blobConfigStorageService pipeline.BlobStorageConfigService } -func NewTriggerServiceImpl(logger *zap.SugaredLogger, +func NewHandlerServiceImpl(logger *zap.SugaredLogger, cdWorkflowCommonService cd.CdWorkflowCommonService, gitOpsManifestPushService publish.GitOpsPushService, gitOpsConfigReadService config.GitOpsConfigReadService, @@ -243,8 +243,8 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, ciLogService pipeline.CiLogService, workflowService executor.WorkflowService, blobConfigStorageService pipeline.BlobStorageConfigService, -) (*TriggerServiceImpl, error) { - impl := &TriggerServiceImpl{ +) (*HandlerServiceImpl, error) { + impl := &HandlerServiceImpl{ logger: logger, cdWorkflowCommonService: cdWorkflowCommonService, gitOpsManifestPushService: gitOpsManifestPushService, @@ -318,7 +318,7 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, return impl, nil } -func (impl *TriggerServiceImpl) TriggerStageForBulk(triggerRequest bean.TriggerRequest) error { +func (impl *HandlerServiceImpl) TriggerStageForBulk(triggerRequest bean.TriggerRequest) error { preStage, err := impl.pipelineStageService.GetCdStageByCdPipelineIdAndStageType(triggerRequest.Pipeline.Id, repository.PIPELINE_STAGE_TYPE_PRE_CD, false) if err != nil && err != pg.ErrNoRows { @@ -352,8 +352,8 @@ func (impl *TriggerServiceImpl) TriggerStageForBulk(triggerRequest bean.TriggerR } } -func (impl *TriggerServiceImpl) getCdPipelineForManualCdTrigger(ctx context.Context, pipelineId int) (*pipelineConfig.Pipeline, error) { - _, span := otel.Tracer("TriggerService").Start(ctx, "getCdPipelineForManualCdTrigger") +func (impl *HandlerServiceImpl) getCdPipelineForManualCdTrigger(ctx context.Context, pipelineId int) (*pipelineConfig.Pipeline, error) { + _, span := otel.Tracer("HandlerService").Start(ctx, "getCdPipelineForManualCdTrigger") defer span.End() cdPipeline, err := impl.pipelineRepository.FindById(pipelineId) if err != nil { @@ -373,8 +373,8 @@ func (impl *TriggerServiceImpl) getCdPipelineForManualCdTrigger(ctx context.Cont return cdPipeline, nil } -func (impl *TriggerServiceImpl) validateDeploymentTriggerRequest(ctx context.Context, validateDeploymentTriggerObj *bean.ValidateDeploymentTriggerObj) error { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.validateDeploymentTriggerRequest") +func (impl *HandlerServiceImpl) validateDeploymentTriggerRequest(ctx context.Context, validateDeploymentTriggerObj *bean.ValidateDeploymentTriggerObj) error { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.validateDeploymentTriggerRequest") defer span.End() // custom GitOps repo url validation --> Start err := impl.handleCustomGitOpsRepoValidation(validateDeploymentTriggerObj.Runner, validateDeploymentTriggerObj.CdPipeline, validateDeploymentTriggerObj.DeploymentConfig, validateDeploymentTriggerObj.TriggeredBy) @@ -406,7 +406,7 @@ func (impl *TriggerServiceImpl) validateDeploymentTriggerRequest(ctx context.Con } // TODO: write a wrapper to handle auto and manual trigger -func (impl *TriggerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerContext, overrideRequest *bean3.ValuesOverrideRequest) (int, string, *bean4.ManifestPushTemplate, error) { +func (impl *HandlerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerContext, overrideRequest *bean3.ValuesOverrideRequest) (int, string, *bean4.ManifestPushTemplate, error) { triggerContext.TriggerType = bean.Manual // setting triggeredAt variable to have consistent data for various audit log places in db for deployment time @@ -628,7 +628,7 @@ func isNotHibernateRequest(deploymentType models.DeploymentType) bool { } // TODO: write a wrapper to handle auto and manual trigger -func (impl *TriggerServiceImpl) TriggerAutomaticDeployment(request bean.TriggerRequest) error { +func (impl *HandlerServiceImpl) TriggerAutomaticDeployment(request bean.TriggerRequest) error { // in case of manual trigger auth is already applied and for auto triggers there is no need for auth check here triggeredBy := request.TriggeredBy pipeline := request.Pipeline @@ -705,7 +705,7 @@ func (impl *TriggerServiceImpl) TriggerAutomaticDeployment(request bean.TriggerR return nil } -func (impl *TriggerServiceImpl) TriggerCD(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) error { +func (impl *HandlerServiceImpl) TriggerCD(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) error { impl.logger.Debugw("automatic pipeline trigger attempt async", "artifactId", artifact.Id) err := impl.triggerReleaseAsync(ctx, artifact, cdWorkflowId, wfrId, pipeline, envDeploymentConfig, triggeredAt, triggeredBy) if err != nil { @@ -715,7 +715,7 @@ func (impl *TriggerServiceImpl) TriggerCD(ctx context.Context, artifact *reposit return err } -func (impl *TriggerServiceImpl) triggerReleaseAsync(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) error { +func (impl *HandlerServiceImpl) triggerReleaseAsync(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) error { err := impl.validateAndTrigger(ctx, pipeline, envDeploymentConfig, artifact, cdWorkflowId, wfrId, triggeredAt, triggeredBy) if err != nil { impl.logger.Errorw("error in trigger for pipeline", "pipelineId", strconv.Itoa(pipeline.Id)) @@ -724,7 +724,7 @@ func (impl *TriggerServiceImpl) triggerReleaseAsync(ctx context.Context, artifac return err } -func (impl *TriggerServiceImpl) validateAndTrigger(ctx context.Context, p *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time, triggeredBy int32) error { +func (impl *HandlerServiceImpl) validateAndTrigger(ctx context.Context, p *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time, triggeredBy int32) error { //TODO: verify this logic object := impl.enforcerUtil.GetAppRBACNameByAppId(p.AppId) envApp := strings.Split(object, "/") @@ -736,7 +736,7 @@ func (impl *TriggerServiceImpl) validateAndTrigger(ctx context.Context, p *pipel return err } -func (impl *TriggerServiceImpl) releasePipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time, triggeredBy int32) error { +func (impl *HandlerServiceImpl) releasePipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time, triggeredBy int32) error { startTime := time.Now() defer func() { impl.logger.Debugw("auto trigger release process completed", "timeTaken", time.Since(startTime), "cdPipelineId", pipeline.Id, "artifactId", artifact.Id, "wfrId", wfrId) @@ -770,9 +770,9 @@ func (impl *TriggerServiceImpl) releasePipeline(ctx context.Context, pipeline *p return err } -func (impl *TriggerServiceImpl) triggerAsyncRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, +func (impl *HandlerServiceImpl) triggerAsyncRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, userDeploymentRequestId int, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.triggerAsyncRelease") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.triggerAsyncRelease") defer span.End() // build merged values and save PCO history for the release valuesOverrideResponse, err := impl.manifestCreationService.GetValuesOverrideForTrigger(newCtx, overrideRequest, envDeploymentConfig, triggeredAt) @@ -791,8 +791,8 @@ func (impl *TriggerServiceImpl) triggerAsyncRelease(ctx context.Context, overrid return impl.workflowEventPublishService.TriggerAsyncRelease(userDeploymentRequestId, overrideRequest, valuesOverrideResponse, newCtx, deployedBy) } -func (impl *TriggerServiceImpl) handleCDTriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.handleCDTriggerRelease") +func (impl *HandlerServiceImpl) handleCDTriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.handleCDTriggerRelease") defer span.End() // Handling for auto trigger if overrideRequest.UserId == 0 { @@ -840,7 +840,7 @@ func (impl *TriggerServiceImpl) handleCDTriggerRelease(ctx context.Context, over return impl.TriggerRelease(newCtx, overrideRequest, envDeploymentConfig, triggeredAt, deployedBy) } -func (impl *TriggerServiceImpl) auditDeploymentTriggerHistory(cdWfrId int, valuesOverrideResponse *app.ValuesOverrideResponse, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (err error) { +func (impl *HandlerServiceImpl) auditDeploymentTriggerHistory(cdWfrId int, valuesOverrideResponse *app.ValuesOverrideResponse, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (err error) { if valuesOverrideResponse.Pipeline == nil || valuesOverrideResponse.EnvOverride == nil { impl.logger.Warnw("unable to save histories for deployment trigger, invalid valuesOverrideResponse received", "cdWfrId", cdWfrId) return nil @@ -854,9 +854,9 @@ func (impl *TriggerServiceImpl) auditDeploymentTriggerHistory(cdWfrId int, value } // TriggerRelease will trigger Install/Upgrade request for Devtron App releases synchronously -func (impl *TriggerServiceImpl) TriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, +func (impl *HandlerServiceImpl) TriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.TriggerRelease") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.TriggerRelease") defer span.End() triggerEvent, skipRequest, err := impl.buildTriggerEventForOverrideRequest(overrideRequest, triggeredAt) if err != nil { @@ -908,10 +908,10 @@ func (impl *TriggerServiceImpl) TriggerRelease(ctx context.Context, overrideRequ return releaseNo, valuesOverrideResponse.ManifestPushTemplate, nil } -func (impl *TriggerServiceImpl) performGitOps(ctx context.Context, +func (impl *HandlerServiceImpl) performGitOps(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, triggerEvent bean.TriggerEvent) error { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.performGitOps") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.performGitOps") defer span.End() // update workflow runner status, used in app workflow view err := impl.cdWorkflowCommonService.UpdateNonTerminalStatusInRunner(newCtx, overrideRequest.WfrId, overrideRequest.UserId, cdWorkflow.WorkflowInProgress) @@ -938,7 +938,7 @@ func (impl *TriggerServiceImpl) performGitOps(ctx context.Context, return nil } -func (impl *TriggerServiceImpl) buildTriggerEventForOverrideRequest(overrideRequest *bean3.ValuesOverrideRequest, triggeredAt time.Time) (triggerEvent bean.TriggerEvent, skipRequest bool, err error) { +func (impl *HandlerServiceImpl) buildTriggerEventForOverrideRequest(overrideRequest *bean3.ValuesOverrideRequest, triggeredAt time.Time) (triggerEvent bean.TriggerEvent, skipRequest bool, err error) { triggerEvent = helper.NewTriggerEvent(overrideRequest.DeploymentAppType, triggeredAt, overrideRequest.UserId) request := statusBean.NewTimelineGetRequest(). WithCdWfrId(overrideRequest.WfrId). @@ -973,8 +973,8 @@ func (impl *TriggerServiceImpl) buildTriggerEventForOverrideRequest(overrideRequ return triggerEvent, skipRequest, nil } -func (impl *TriggerServiceImpl) triggerPipeline(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, triggerEvent bean.TriggerEvent, ctx context.Context) (releaseNo int, err error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.triggerPipeline") +func (impl *HandlerServiceImpl) triggerPipeline(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, triggerEvent bean.TriggerEvent, ctx context.Context) (releaseNo int, err error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.triggerPipeline") defer span.End() if triggerEvent.PerformChartPush { impl.logger.Debugw("performing chart push operation in triggerPipeline", "cdWfrId", overrideRequest.WfrId) @@ -1010,7 +1010,7 @@ func (impl *TriggerServiceImpl) triggerPipeline(overrideRequest *bean3.ValuesOve return valuesOverrideResponse.PipelineOverride.PipelineReleaseCounter, nil } -func (impl *TriggerServiceImpl) buildManifestPushTemplate(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string) (*bean4.ManifestPushTemplate, error) { +func (impl *HandlerServiceImpl) buildManifestPushTemplate(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string) (*bean4.ManifestPushTemplate, error) { manifestPushTemplate := &bean4.ManifestPushTemplate{ WorkflowRunnerId: overrideRequest.WfrId, @@ -1057,8 +1057,8 @@ func (impl *TriggerServiceImpl) buildManifestPushTemplate(overrideRequest *bean3 return manifestPushTemplate, nil } -func (impl *TriggerServiceImpl) deployApp(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, triggerEvent bean.TriggerEvent) error { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.deployApp") +func (impl *HandlerServiceImpl) deployApp(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, triggerEvent bean.TriggerEvent) error { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.deployApp") defer span.End() var referenceChartByte []byte var err error @@ -1080,8 +1080,8 @@ func (impl *TriggerServiceImpl) deployApp(ctx context.Context, overrideRequest * return nil } -func (impl *TriggerServiceImpl) createHelmAppForCdPipeline(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse) (bool, []byte, error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.createHelmAppForCdPipeline") +func (impl *HandlerServiceImpl) createHelmAppForCdPipeline(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse) (bool, []byte, error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.createHelmAppForCdPipeline") defer span.End() pipelineModel := valuesOverrideResponse.Pipeline envOverride := valuesOverrideResponse.EnvOverride @@ -1186,7 +1186,7 @@ func (impl *TriggerServiceImpl) createHelmAppForCdPipeline(ctx context.Context, return true, referenceChartByte, nil } -func (impl *TriggerServiceImpl) getHelmHistoryLimitAndChartMetadataForHelmAppCreation(ctx context.Context, +func (impl *HandlerServiceImpl) getHelmHistoryLimitAndChartMetadataForHelmAppCreation(ctx context.Context, valuesOverrideResponse *app.ValuesOverrideResponse) (*chart.Metadata, int32, *gRPC.ReleaseIdentifier, error) { pipelineModel := valuesOverrideResponse.Pipeline envOverride := valuesOverrideResponse.EnvOverride @@ -1235,9 +1235,9 @@ func (impl *TriggerServiceImpl) getHelmHistoryLimitAndChartMetadataForHelmAppCre return chartMetaData, helmRevisionHistory, releaseIdentifier, nil } -func (impl *TriggerServiceImpl) deployArgoCdApp(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, +func (impl *HandlerServiceImpl) deployArgoCdApp(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse) error { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.deployArgoCdApp") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.deployArgoCdApp") defer span.End() name, err := impl.createArgoApplicationIfRequired(newCtx, valuesOverrideResponse.EnvOverride, valuesOverrideResponse.Pipeline, valuesOverrideResponse.DeploymentConfig, overrideRequest.UserId) if err != nil { @@ -1281,7 +1281,7 @@ func (impl *TriggerServiceImpl) deployArgoCdApp(ctx context.Context, overrideReq } // update repoUrl, revision and argo app sync mode (auto/manual) if needed -func (impl *TriggerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envOverride *bean10.EnvConfigOverride, deploymentConfig *bean9.DeploymentConfig) (bool, error) { +func (impl *HandlerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envOverride *bean10.EnvConfigOverride, deploymentConfig *bean9.DeploymentConfig) (bool, error) { if !deploymentConfig.IsArgoAppPatchSupported() { impl.logger.Infow("argo app patch not supported", "pipelineId", pipeline.Id, "pipelineName", pipeline.Name) return false, nil @@ -1290,7 +1290,7 @@ func (impl *TriggerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline impl.logger.Errorw("err in syncing ACD, ctx is NULL", "pipelineId", pipeline.Id, "pipelineName", pipeline.Name) return false, nil } - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.updateArgoPipeline") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.updateArgoPipeline") defer span.End() argoAppName := pipeline.DeploymentAppName impl.logger.Infow("received payload, updateArgoPipeline", "appId", pipeline.AppId, "pipelineName", pipeline.Name, "envId", envOverride.TargetEnvironment, "argoAppName", argoAppName) @@ -1346,9 +1346,9 @@ func (impl *TriggerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline } } -func (impl *TriggerServiceImpl) createArgoApplicationIfRequired(ctx context.Context, envConfigOverride *bean10.EnvConfigOverride, +func (impl *HandlerServiceImpl) createArgoApplicationIfRequired(ctx context.Context, envConfigOverride *bean10.EnvConfigOverride, pipeline *pipelineConfig.Pipeline, deploymentConfig *bean9.DeploymentConfig, userId int32) (string, error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.createArgoApplicationIfRequired") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.createArgoApplicationIfRequired") defer span.End() envModel, err := impl.envRepository.FindById(envConfigOverride.TargetEnvironment) if err != nil { @@ -1395,7 +1395,7 @@ func (impl *TriggerServiceImpl) createArgoApplicationIfRequired(ctx context.Cont } } -func (impl *TriggerServiceImpl) updatePipeline(pipeline *pipelineConfig.Pipeline, userId int32) (bool, error) { +func (impl *HandlerServiceImpl) updatePipeline(pipeline *pipelineConfig.Pipeline, userId int32) (bool, error) { err := impl.pipelineRepository.SetDeploymentAppCreatedInPipeline(true, pipeline.Id, userId) if err != nil { impl.logger.Errorw("error on updating cd pipeline for setting deployment app created", "err", err) @@ -1405,9 +1405,9 @@ func (impl *TriggerServiceImpl) updatePipeline(pipeline *pipelineConfig.Pipeline } // helmInstallReleaseWithCustomChart performs helm install with custom chart -func (impl *TriggerServiceImpl) helmInstallReleaseWithCustomChart(ctx context.Context, releaseIdentifier *gRPC.ReleaseIdentifier, +func (impl *HandlerServiceImpl) helmInstallReleaseWithCustomChart(ctx context.Context, releaseIdentifier *gRPC.ReleaseIdentifier, referenceChartByte []byte, valuesYaml, k8sServerVersion string, forceSync bool) (*gRPC.HelmInstallCustomResponse, error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.helmInstallReleaseWithCustomChart") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.helmInstallReleaseWithCustomChart") defer span.End() helmInstallRequest := gRPC.HelmInstallCustomRequest{ ValuesYaml: valuesYaml, @@ -1424,7 +1424,7 @@ func (impl *TriggerServiceImpl) helmInstallReleaseWithCustomChart(ctx context.Co return impl.helmAppClient.InstallReleaseWithCustomChart(newCtx, &helmInstallRequest) } -func (impl *TriggerServiceImpl) writeCDTriggerEvent(overrideRequest *bean3.ValuesOverrideRequest, artifact *repository3.CiArtifact, releaseId, pipelineOverrideId, wfrId int) { +func (impl *HandlerServiceImpl) writeCDTriggerEvent(overrideRequest *bean3.ValuesOverrideRequest, artifact *repository3.CiArtifact, releaseId, pipelineOverrideId, wfrId int) { event, err := impl.eventFactory.Build(util2.Trigger, &overrideRequest.PipelineId, overrideRequest.AppId, &overrideRequest.EnvId, util2.CD) if err != nil { @@ -1467,9 +1467,9 @@ func (impl *TriggerServiceImpl) writeCDTriggerEvent(overrideRequest *bean3.Value } } -func (impl *TriggerServiceImpl) markImageScanDeployed(ctx context.Context, appId, envId, clusterId int, +func (impl *HandlerServiceImpl) markImageScanDeployed(ctx context.Context, appId, envId, clusterId int, imageDigest string, isScanEnabled bool, image string) error { - _, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.markImageScanDeployed") + _, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.markImageScanDeployed") defer span.End() // TODO KB: send NATS event for self consumption impl.logger.Debugw("mark image scan deployed for devtron app, from cd auto or manual trigger", "imageDigest", imageDigest) @@ -1537,11 +1537,11 @@ func (impl *TriggerServiceImpl) markImageScanDeployed(ctx context.Context, appId return err } -func (impl *TriggerServiceImpl) isDevtronAsyncHelmInstallModeEnabled(forceSync bool) bool { +func (impl *HandlerServiceImpl) isDevtronAsyncHelmInstallModeEnabled(forceSync bool) bool { return impl.globalEnvVariables.EnableAsyncHelmInstallDevtronChart && !forceSync } -func (impl *TriggerServiceImpl) isDevtronAsyncInstallModeEnabled(overrideRequest *bean3.ValuesOverrideRequest) (bool, error) { +func (impl *HandlerServiceImpl) isDevtronAsyncInstallModeEnabled(overrideRequest *bean3.ValuesOverrideRequest) (bool, error) { if util.IsHelmApp(overrideRequest.DeploymentAppType) { return impl.isDevtronAsyncHelmInstallModeEnabled(overrideRequest.ForceSyncDeployment), nil } else if util.IsAcdApp(overrideRequest.DeploymentAppType) { @@ -1551,7 +1551,7 @@ func (impl *TriggerServiceImpl) isDevtronAsyncInstallModeEnabled(overrideRequest return false, nil } -func (impl *TriggerServiceImpl) deleteCorruptedPipelineStage(pipelineStage *repository.PipelineStage, triggeredBy int32) (error, bool) { +func (impl *HandlerServiceImpl) deleteCorruptedPipelineStage(pipelineStage *repository.PipelineStage, triggeredBy int32) (error, bool) { if pipelineStage != nil { stageReq := &bean8.PipelineStageDto{ Id: pipelineStage.Id, @@ -1567,7 +1567,7 @@ func (impl *TriggerServiceImpl) deleteCorruptedPipelineStage(pipelineStage *repo return nil, false } -func (impl *TriggerServiceImpl) handleCustomGitOpsRepoValidation(runner *pipelineConfig.CdWorkflowRunner, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredBy int32) error { +func (impl *HandlerServiceImpl) handleCustomGitOpsRepoValidation(runner *pipelineConfig.CdWorkflowRunner, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredBy int32) error { if !util.IsAcdApp(pipeline.DeploymentAppName) { return nil } @@ -1600,7 +1600,7 @@ func (impl *TriggerServiceImpl) handleCustomGitOpsRepoValidation(runner *pipelin return nil } -func (impl *TriggerServiceImpl) getSanitizedK8sVersion(referenceTemplate string) (string, error) { +func (impl *HandlerServiceImpl) getSanitizedK8sVersion(referenceTemplate string) (string, error) { var sanitizedK8sVersion string //handle specific case for all cronjob charts from cronjob-chart_1-2-0 to cronjob-chart_1-5-0 where semverCompare //comparison func has wrong api version mentioned, so for already installed charts via these charts that comparison @@ -1617,7 +1617,7 @@ func (impl *TriggerServiceImpl) getSanitizedK8sVersion(referenceTemplate string) return sanitizedK8sVersion, nil } -func (impl *TriggerServiceImpl) getReferenceChartByteForHelmTypeApp(envOverride *bean10.EnvConfigOverride, +func (impl *HandlerServiceImpl) getReferenceChartByteForHelmTypeApp(envOverride *bean10.EnvConfigOverride, chartMetaData *chart.Metadata, referenceTemplatePath string, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse) ([]byte, error) { referenceChartByte := envOverride.Chart.ReferenceChart diff --git a/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go b/pkg/deployment/trigger/devtronApps/HandlerService_ent1.go similarity index 79% rename from pkg/deployment/trigger/devtronApps/TriggerService_ent1.go rename to pkg/deployment/trigger/devtronApps/HandlerService_ent1.go index f008f72adc..24b7dab264 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go +++ b/pkg/deployment/trigger/devtronApps/HandlerService_ent1.go @@ -17,29 +17,29 @@ import ( "helm.sh/helm/v3/pkg/chart" ) -func (impl *TriggerServiceImpl) getEnrichedWorkflowRunner(overrideRequest *bean3.ValuesOverrideRequest, artifact *repository3.CiArtifact, wfrId int) *pipelineConfig.CdWorkflowRunner { +func (impl *HandlerServiceImpl) getEnrichedWorkflowRunner(overrideRequest *bean3.ValuesOverrideRequest, artifact *repository3.CiArtifact, wfrId int) *pipelineConfig.CdWorkflowRunner { return nil } -func (impl *TriggerServiceImpl) postDeployHook(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, referenceChartByte []byte, err error) { +func (impl *HandlerServiceImpl) postDeployHook(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, referenceChartByte []byte, err error) { impl.logger.Debugw("no post deploy hook registered") } -func (impl *TriggerServiceImpl) isDevtronAsyncArgoCdInstallModeEnabledForApp(appId, envId int, forceSync bool) (bool, error) { +func (impl *HandlerServiceImpl) isDevtronAsyncArgoCdInstallModeEnabledForApp(appId, envId int, forceSync bool) (bool, error) { return impl.globalEnvVariables.EnableAsyncArgoCdInstallDevtronChart && !forceSync, nil } -func (impl *TriggerServiceImpl) getClusterGRPCConfig(cluster repository2.Cluster) *gRPC.ClusterConfig { +func (impl *HandlerServiceImpl) getClusterGRPCConfig(cluster repository2.Cluster) *gRPC.ClusterConfig { clusterConfig := helper.ConvertClusterBeanToGrpcConfig(cluster) return clusterConfig } -func (impl *TriggerServiceImpl) overrideReferenceChartByteForHelmTypeApp(valuesOverrideResponse *app.ValuesOverrideResponse, +func (impl *HandlerServiceImpl) overrideReferenceChartByteForHelmTypeApp(valuesOverrideResponse *app.ValuesOverrideResponse, chartMetaData *chart.Metadata, referenceTemplatePath string, referenceChartByte []byte) ([]byte, error) { return referenceChartByte, nil } -func (impl *TriggerServiceImpl) getManifestPushService(storageType string) publish.ManifestPushService { +func (impl *HandlerServiceImpl) getManifestPushService(storageType string) publish.ManifestPushService { var manifestPushService publish.ManifestPushService if storageType == bean2.ManifestStorageGit { manifestPushService = impl.gitOpsManifestPushService @@ -47,25 +47,25 @@ func (impl *TriggerServiceImpl) getManifestPushService(storageType string) publi return manifestPushService } -func (impl *TriggerServiceImpl) preStageHandlingForTriggerStageInBulk(triggerRequest *bean.TriggerRequest) error { +func (impl *HandlerServiceImpl) preStageHandlingForTriggerStageInBulk(triggerRequest *bean.TriggerRequest) error { return nil } -func (impl *TriggerServiceImpl) manifestGenerationFailedTimelineHandling(triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, err error) { +func (impl *HandlerServiceImpl) manifestGenerationFailedTimelineHandling(triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, err error) { } -func (impl *TriggerServiceImpl) getHelmManifestForTriggerRelease(ctx context.Context, triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, +func (impl *HandlerServiceImpl) getHelmManifestForTriggerRelease(ctx context.Context, triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string) ([]byte, error) { return nil, nil } -func (impl *TriggerServiceImpl) buildManifestPushTemplateForNonGitStorageType(overrideRequest *bean3.ValuesOverrideRequest, +func (impl *HandlerServiceImpl) buildManifestPushTemplateForNonGitStorageType(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, err error, manifestPushConfig *repository.ManifestPushConfig, manifestPushTemplate *bean4.ManifestPushTemplate) error { return nil } -func (impl *TriggerServiceImpl) triggerReleaseSuccessHandling(triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, +func (impl *HandlerServiceImpl) triggerReleaseSuccessHandling(triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, helmManifest []byte) error { return nil } diff --git a/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go index defe63b75d..1bf2399ffc 100644 --- a/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go @@ -30,7 +30,7 @@ import ( "time" ) -func (impl *TriggerServiceImpl) TriggerPostStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) { +func (impl *HandlerServiceImpl) TriggerPostStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) { request.WorkflowType = bean2.CD_WORKFLOW_TYPE_POST // setting triggeredAt variable to have consistent data for various audit log places in db for deployment time triggeredAt := time.Now() @@ -164,7 +164,7 @@ func (impl *TriggerServiceImpl) TriggerPostStage(request bean.TriggerRequest) (* return manifestPushTempate, nil } -func (impl *TriggerServiceImpl) buildWfRequestErrorHandler(runner *pipelineConfig.CdWorkflowRunner, err error, triggeredBy int32) (*bean4.ManifestPushTemplate, error) { +func (impl *HandlerServiceImpl) buildWfRequestErrorHandler(runner *pipelineConfig.CdWorkflowRunner, err error, triggeredBy int32) (*bean4.ManifestPushTemplate, error) { dbErr := impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, err, triggeredBy) if dbErr != nil { impl.logger.Errorw("error while updating current runner status to failed, buildWfRequestErrorHandler", "runner", runner.Id, "err", dbErr, "releaseErr", err) diff --git a/pkg/deployment/trigger/devtronApps/PostStageTriggerService_ent.go b/pkg/deployment/trigger/devtronApps/PostStageTriggerService_ent.go index d99b813b1d..197aecd22d 100644 --- a/pkg/deployment/trigger/devtronApps/PostStageTriggerService_ent.go +++ b/pkg/deployment/trigger/devtronApps/PostStageTriggerService_ent.go @@ -26,13 +26,13 @@ import ( "time" ) -func (impl *TriggerServiceImpl) checkFeasibilityForPostStage(pipeline *pipelineConfig.Pipeline, request *bean.TriggerRequest, +func (impl *HandlerServiceImpl) checkFeasibilityForPostStage(pipeline *pipelineConfig.Pipeline, request *bean.TriggerRequest, env *repository.Environment, cdWf *pipelineConfig.CdWorkflow, triggeredBy int32) (interface{}, error) { //here return type is interface as ResourceFilterEvaluationAudit is not present in this version return nil, nil } -func (impl *TriggerServiceImpl) getManifestPushTemplateForPostStage(request bean.TriggerRequest, envDevploymentConfig *bean5.DeploymentConfig, +func (impl *HandlerServiceImpl) getManifestPushTemplateForPostStage(request bean.TriggerRequest, envDevploymentConfig *bean5.DeploymentConfig, jobHelmPackagePath string, cdStageWorkflowRequest *types.WorkflowRequest, cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner, pipeline *pipelineConfig.Pipeline, triggeredBy int32, triggeredAt time.Time) (*bean4.ManifestPushTemplate, error) { return nil, nil diff --git a/pkg/deployment/trigger/devtronApps/PrePostCommonService.go b/pkg/deployment/trigger/devtronApps/PrePostCommonService.go index d1cf588032..d3ff8b5e5f 100644 --- a/pkg/deployment/trigger/devtronApps/PrePostCommonService.go +++ b/pkg/deployment/trigger/devtronApps/PrePostCommonService.go @@ -26,7 +26,7 @@ import ( "time" ) -func (impl *TriggerServiceImpl) CancelStage(workflowRunnerId int, forceAbort bool, userId int32) (int, error) { +func (impl *HandlerServiceImpl) CancelStage(workflowRunnerId int, forceAbort bool, userId int32) (int, error) { workflowRunner, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(workflowRunnerId) if err != nil { impl.logger.Errorw("err", "err", err) @@ -113,7 +113,7 @@ func (impl *TriggerServiceImpl) CancelStage(workflowRunnerId int, forceAbort boo return workflowRunner.Id, nil } -func (impl *TriggerServiceImpl) updateWorkflowRunnerForForceAbort(workflowRunner *pipelineConfig.CdWorkflowRunner) error { +func (impl *HandlerServiceImpl) updateWorkflowRunnerForForceAbort(workflowRunner *pipelineConfig.CdWorkflowRunner) error { workflowRunner.Status = cdWorkflow2.WorkflowCancel workflowRunner.PodStatus = string(bean2.Failed) workflowRunner.Message = constants.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE @@ -125,7 +125,7 @@ func (impl *TriggerServiceImpl) updateWorkflowRunnerForForceAbort(workflowRunner return nil } -func (impl *TriggerServiceImpl) handleForceAbortCaseForCdStage(workflowRunner *pipelineConfig.CdWorkflowRunner, forceAbort bool) error { +func (impl *HandlerServiceImpl) handleForceAbortCaseForCdStage(workflowRunner *pipelineConfig.CdWorkflowRunner, forceAbort bool) error { isWorkflowInNonTerminalStage := workflowRunner.Status == string(v1alpha1.NodePending) || workflowRunner.Status == string(v1alpha1.NodeRunning) if !isWorkflowInNonTerminalStage { if forceAbort { @@ -141,7 +141,7 @@ func (impl *TriggerServiceImpl) handleForceAbortCaseForCdStage(workflowRunner *p return nil } -func (impl *TriggerServiceImpl) DownloadCdWorkflowArtifacts(buildId int) (*os.File, error) { +func (impl *HandlerServiceImpl) DownloadCdWorkflowArtifacts(buildId int) (*os.File, error) { wfr, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(buildId) if err != nil { impl.logger.Errorw("unable to fetch ciWorkflow", "err", err) @@ -224,7 +224,7 @@ func (impl *TriggerServiceImpl) DownloadCdWorkflowArtifacts(buildId int) (*os.Fi return file, nil } -func (impl *TriggerServiceImpl) GetRunningWorkflowLogs(environmentId int, pipelineId int, wfrId int) (*bufio.Reader, func() error, error) { +func (impl *HandlerServiceImpl) GetRunningWorkflowLogs(environmentId int, pipelineId int, wfrId int) (*bufio.Reader, func() error, error) { cdWorkflow, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(wfrId) if err != nil { impl.logger.Errorw("error on fetch wf runner", "err", err) @@ -256,7 +256,7 @@ func (impl *TriggerServiceImpl) GetRunningWorkflowLogs(environmentId int, pipeli return impl.getWorkflowLogs(pipelineId, cdWorkflow, clusterConfig, isExtCluster) } -func (impl *TriggerServiceImpl) getWorkflowLogs(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, runStageInEnv bool) (*bufio.Reader, func() error, error) { +func (impl *HandlerServiceImpl) getWorkflowLogs(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, runStageInEnv bool) (*bufio.Reader, func() error, error) { cdLogRequest := types.BuildLogRequest{ PodName: cdWorkflow.PodName, Namespace: cdWorkflow.Namespace, @@ -281,7 +281,7 @@ func (impl *TriggerServiceImpl) getWorkflowLogs(pipelineId int, cdWorkflow *pipe return logReader, cleanUp, err } -func (impl *TriggerServiceImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { +func (impl *HandlerServiceImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { impl.logger.Debug("getting historic logs", "pipelineId", pipelineId) cdConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() // TODO -fixme diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go index 64c373a235..ad0051e76d 100644 --- a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go @@ -61,7 +61,7 @@ import ( "time" ) -func (impl *TriggerServiceImpl) TriggerPreStage(request bean.TriggerRequest) (*bean6.ManifestPushTemplate, error) { +func (impl *HandlerServiceImpl) TriggerPreStage(request bean.TriggerRequest) (*bean6.ManifestPushTemplate, error) { request.WorkflowType = bean2.CD_WORKFLOW_TYPE_PRE // setting triggeredAt variable to have consistent data for various audit log places in db for deployment time triggeredAt := time.Now() @@ -177,7 +177,7 @@ func (impl *TriggerServiceImpl) TriggerPreStage(request bean.TriggerRequest) (*b return manifestPushTemplate, nil } -func (impl *TriggerServiceImpl) TriggerAutoCDOnPreStageSuccess(triggerContext bean.TriggerContext, cdPipelineId, ciArtifactId, workflowId int) error { +func (impl *HandlerServiceImpl) TriggerAutoCDOnPreStageSuccess(triggerContext bean.TriggerContext, cdPipelineId, ciArtifactId, workflowId int) error { pipeline, err := impl.pipelineRepository.FindById(cdPipelineId) if err != nil { return err @@ -216,7 +216,7 @@ func (impl *TriggerServiceImpl) TriggerAutoCDOnPreStageSuccess(triggerContext be } return nil } -func (impl *TriggerServiceImpl) checkDeploymentTriggeredAlready(wfId int) bool { +func (impl *HandlerServiceImpl) checkDeploymentTriggeredAlready(wfId int) bool { deploymentTriggeredAlready := false // TODO : need to check this logic for status check in case of multiple deployments requirement for same workflow workflowRunner, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(context.Background(), wfId, bean2.CD_WORKFLOW_TYPE_DEPLOY) @@ -228,7 +228,7 @@ func (impl *TriggerServiceImpl) checkDeploymentTriggeredAlready(wfId int) bool { return deploymentTriggeredAlready } -func (impl *TriggerServiceImpl) createStartingWfAndRunner(request bean.TriggerRequest, triggeredAt time.Time) (*pipelineConfig.CdWorkflow, *pipelineConfig.CdWorkflowRunner, error) { +func (impl *HandlerServiceImpl) createStartingWfAndRunner(request bean.TriggerRequest, triggeredAt time.Time) (*pipelineConfig.CdWorkflow, *pipelineConfig.CdWorkflowRunner, error) { triggeredBy := request.TriggeredBy artifact := request.Artifact pipeline := request.Pipeline @@ -272,7 +272,7 @@ func (impl *TriggerServiceImpl) createStartingWfAndRunner(request bean.TriggerRe return cdWf, runner, nil } -func (impl *TriggerServiceImpl) getEnvAndNsIfRunStageInEnv(ctx context.Context, request bean.TriggerRequest) (*repository4.Environment, string, error) { +func (impl *HandlerServiceImpl) getEnvAndNsIfRunStageInEnv(ctx context.Context, request bean.TriggerRequest) (*repository4.Environment, string, error) { workflowStage := request.WorkflowType pipeline := request.Pipeline var env *repository4.Environment @@ -297,7 +297,7 @@ func (impl *TriggerServiceImpl) getEnvAndNsIfRunStageInEnv(ctx context.Context, return env, namespace, nil } -func (impl *TriggerServiceImpl) checkVulnerabilityStatusAndFailWfIfNeeded(ctx context.Context, artifact *repository.CiArtifact, +func (impl *HandlerServiceImpl) checkVulnerabilityStatusAndFailWfIfNeeded(ctx context.Context, artifact *repository.CiArtifact, cdPipeline *pipelineConfig.Pipeline, runner *pipelineConfig.CdWorkflowRunner, triggeredBy int32) error { //checking vulnerability for the selected image vulnerabilityCheckRequest := adapter2.GetVulnerabilityCheckRequest(cdPipeline, artifact.ImageDigest) @@ -324,7 +324,7 @@ func (impl *TriggerServiceImpl) checkVulnerabilityStatusAndFailWfIfNeeded(ctx co } // setCopyContainerImagePluginDataAndReserveImages sets required fields in cdStageWorkflowRequest and reserve images generated by plugin -func (impl *TriggerServiceImpl) setCopyContainerImagePluginDataAndReserveImages(cdStageWorkflowRequest *types.WorkflowRequest, pipelineId int, pipelineStage string, artifact *repository.CiArtifact) ([]int, error) { +func (impl *HandlerServiceImpl) setCopyContainerImagePluginDataAndReserveImages(cdStageWorkflowRequest *types.WorkflowRequest, pipelineId int, pipelineStage string, artifact *repository.CiArtifact) ([]int, error) { copyContainerImagePluginDetail, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(buildCommonBean.COPY_CONTAINER_IMAGE) if err != nil && err != pg.ErrNoRows { @@ -407,7 +407,7 @@ func (impl *TriggerServiceImpl) setCopyContainerImagePluginDataAndReserveImages( return imagePathReservationIds, nil } -func (impl *TriggerServiceImpl) getDockerTagAndCustomTagIdForPlugin(pipelineStage string, pipelineId int, artifact *repository.CiArtifact) (string, int, error) { +func (impl *HandlerServiceImpl) getDockerTagAndCustomTagIdForPlugin(pipelineStage string, pipelineId int, artifact *repository.CiArtifact) (string, int, error) { var pipelineStageEntityType int if pipelineStage == types.PRE { pipelineStageEntityType = pipelineConfigBean.EntityTypePreCD @@ -442,7 +442,7 @@ func (impl *TriggerServiceImpl) getDockerTagAndCustomTagIdForPlugin(pipelineStag return DockerImageTag, customTagId, nil } -func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflowRunner, cdWf *pipelineConfig.CdWorkflow, cdPipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean5.DeploymentConfig, triggeredBy int32) (*types.WorkflowRequest, error) { +func (impl *HandlerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflowRunner, cdWf *pipelineConfig.CdWorkflow, cdPipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean5.DeploymentConfig, triggeredBy int32) (*types.WorkflowRequest, error) { if cdPipeline.App.Id == 0 { appModel, err := impl.appRepository.FindById(cdPipeline.AppId) if err != nil { @@ -910,7 +910,7 @@ getBuildRegistryConfigForArtifact performs the following logic to get Pre/Post C If the ci_pipeline_type type is CI_JOB We will always fetch the registry credentials from the ci_template_override table */ -func (impl *TriggerServiceImpl) getBuildRegistryConfigForArtifact(sourceCiPipeline *pipelineConfig.CiPipeline, artifact *repository.CiArtifact, appId int) (*types.DockerArtifactStoreBean, error) { +func (impl *HandlerServiceImpl) getBuildRegistryConfigForArtifact(sourceCiPipeline *pipelineConfig.CiPipeline, artifact *repository.CiArtifact, appId int) (*types.DockerArtifactStoreBean, error) { var buildRegistryConfig *types.DockerArtifactStoreBean var err error @@ -970,7 +970,7 @@ func (impl *TriggerServiceImpl) getBuildRegistryConfigForArtifact(sourceCiPipeli return buildRegistryConfig, nil } -func (impl *TriggerServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, destinationImages []string) ([]int, error) { +func (impl *HandlerServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, destinationImages []string) ([]int, error) { var imagePathReservationIds []int for _, image := range destinationImages { @@ -1004,7 +1004,7 @@ func setExtraEnvVariableInDeployStep(deploySteps []*pipelineConfigBean.StepObjec } } -func (impl *TriggerServiceImpl) getDeployStageDetails(pipelineId int) (pipelineConfig.CdWorkflowRunner, string, int, error) { +func (impl *HandlerServiceImpl) getDeployStageDetails(pipelineId int) (pipelineConfig.CdWorkflowRunner, string, int, error) { deployStageWfr := pipelineConfig.CdWorkflowRunner{} //getting deployment pipeline latest wfr by pipelineId deployStageWfr, err := impl.cdWorkflowRepository.FindLatestByPipelineIdAndRunnerType(pipelineId, bean2.CD_WORKFLOW_TYPE_DEPLOY) @@ -1025,7 +1025,7 @@ func (impl *TriggerServiceImpl) getDeployStageDetails(pipelineId int) (pipelineC return deployStageWfr, deployStageTriggeredByUserEmail, pipelineReleaseCounter, nil } -func (impl *TriggerServiceImpl) buildArtifactLocationForS3(cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) (string, string, string) { +func (impl *HandlerServiceImpl) buildArtifactLocationForS3(cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) (string, string, string) { cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() cdWorkflowConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() ArtifactLocation := fmt.Sprintf("s3://"+path.Join(cdWorkflowConfigLogsBucket, cdArtifactLocationFormat), cdWf.Id, runner.Id) @@ -1033,7 +1033,7 @@ func (impl *TriggerServiceImpl) buildArtifactLocationForS3(cdWf *pipelineConfig. return ArtifactLocation, cdWorkflowConfigLogsBucket, artifactFileName } -func (impl *TriggerServiceImpl) buildDefaultArtifactLocation(savedWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) string { +func (impl *HandlerServiceImpl) buildDefaultArtifactLocation(savedWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) string { cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() ArtifactLocation := fmt.Sprintf(cdArtifactLocationFormat, savedWf.Id, runner.Id) return ArtifactLocation @@ -1045,7 +1045,7 @@ func ReplaceImageTagWithDigest(image, digest string) string { return imageWithDigest } -func (impl *TriggerServiceImpl) sendPreStageNotification(ctx context.Context, cdWf *pipelineConfig.CdWorkflow, pipeline *pipelineConfig.Pipeline) error { +func (impl *HandlerServiceImpl) sendPreStageNotification(ctx context.Context, cdWf *pipelineConfig.CdWorkflow, pipeline *pipelineConfig.Pipeline) error { wfr, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, cdWf.Id, bean2.CD_WORKFLOW_TYPE_PRE) if err != nil { return err diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService_ent.go b/pkg/deployment/trigger/devtronApps/PreStageTriggerService_ent.go index 8f5b3a5c39..d285ba8224 100644 --- a/pkg/deployment/trigger/devtronApps/PreStageTriggerService_ent.go +++ b/pkg/deployment/trigger/devtronApps/PreStageTriggerService_ent.go @@ -14,33 +14,33 @@ import ( "time" ) -func (impl *TriggerServiceImpl) checkFeasibilityForPreStage(pipeline *pipelineConfig.Pipeline, request *bean2.TriggerRequest, +func (impl *HandlerServiceImpl) checkFeasibilityForPreStage(pipeline *pipelineConfig.Pipeline, request *bean2.TriggerRequest, env *repository.Environment, artifact *repository2.CiArtifact, triggeredBy int32) (interface{}, error) { //here return type is interface as ResourceFilterEvaluationAudit is not present in this version return nil, nil } -func (impl *TriggerServiceImpl) createAuditDataForDeploymentWindowBypass(request bean2.TriggerRequest, wfrId int) error { +func (impl *HandlerServiceImpl) createAuditDataForDeploymentWindowBypass(request bean2.TriggerRequest, wfrId int) error { return nil } -func (impl *TriggerServiceImpl) getManifestPushTemplateForPreStage(ctx context.Context, envDeploymentConfig *bean3.DeploymentConfig, +func (impl *HandlerServiceImpl) getManifestPushTemplateForPreStage(ctx context.Context, envDeploymentConfig *bean3.DeploymentConfig, pipeline *pipelineConfig.Pipeline, artifact *repository2.CiArtifact, jobHelmPackagePath string, cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner, triggeredBy int32, triggeredAt time.Time, request bean2.TriggerRequest) (*bean6.ManifestPushTemplate, error) { return nil, nil } -func (impl *TriggerServiceImpl) setCloningModeInCIProjectDetail(ciProjectDetail *bean.CiProjectDetails, appId int, +func (impl *HandlerServiceImpl) setCloningModeInCIProjectDetail(ciProjectDetail *bean.CiProjectDetails, appId int, m *pipelineConfig.CiPipelineMaterial) error { return nil } -func (impl *TriggerServiceImpl) getPreStageBuildRegistryConfigIfSourcePipelineNotPresent(appId int) (*types.DockerArtifactStoreBean, error) { +func (impl *HandlerServiceImpl) getPreStageBuildRegistryConfigIfSourcePipelineNotPresent(appId int) (*types.DockerArtifactStoreBean, error) { return nil, fmt.Errorf("soucePipeline is mandatory, corrupt data") } -func (impl *TriggerServiceImpl) handlerFilterEvaluationAudit(filterEvaluationAudit interface{}, +func (impl *HandlerServiceImpl) handlerFilterEvaluationAudit(filterEvaluationAudit interface{}, runner *pipelineConfig.CdWorkflowRunner) error { //here ip type of filterEvaluationAudit is interface as ResourceFilterEvaluationAudit is not present in this version return nil diff --git a/pkg/deployment/trigger/devtronApps/feasibilityChecker.go b/pkg/deployment/trigger/devtronApps/feasibilityChecker.go index 43926bb86a..c03968d5de 100644 --- a/pkg/deployment/trigger/devtronApps/feasibilityChecker.go +++ b/pkg/deployment/trigger/devtronApps/feasibilityChecker.go @@ -24,7 +24,7 @@ type FeasibilityManager interface { CheckFeasibility(triggerRequirementRequest *bean.TriggerRequirementRequestDto) error } -func (impl *TriggerServiceImpl) CheckFeasibility(triggerRequirementRequest *bean.TriggerRequirementRequestDto) error { +func (impl *HandlerServiceImpl) CheckFeasibility(triggerRequirementRequest *bean.TriggerRequirementRequestDto) error { // have not implemented right now, will be implemented in future for security vulnerability return nil } diff --git a/pkg/deployment/trigger/devtronApps/wire_devtronAppsDeployTrigger.go b/pkg/deployment/trigger/devtronApps/wire_devtronAppsDeployTrigger.go index 4bf4174cfe..953b46422f 100644 --- a/pkg/deployment/trigger/devtronApps/wire_devtronAppsDeployTrigger.go +++ b/pkg/deployment/trigger/devtronApps/wire_devtronAppsDeployTrigger.go @@ -23,6 +23,6 @@ import ( var DevtronAppsDeployTriggerWireSet = wire.NewSet( userDeploymentRequest.WireSet, - NewTriggerServiceImpl, - wire.Bind(new(TriggerService), new(*TriggerServiceImpl)), + NewHandlerServiceImpl, + wire.Bind(new(HandlerService), new(*HandlerServiceImpl)), ) diff --git a/pkg/eventProcessor/in/CDPipelineEventProcessorService.go b/pkg/eventProcessor/in/CDPipelineEventProcessorService.go index 5a54759107..cb18e00066 100644 --- a/pkg/eventProcessor/in/CDPipelineEventProcessorService.go +++ b/pkg/eventProcessor/in/CDPipelineEventProcessorService.go @@ -38,7 +38,7 @@ type CDPipelineEventProcessorImpl struct { pubSubClient *pubsub.PubSubClientServiceImpl cdWorkflowCommonService cd.CdWorkflowCommonService workflowStatusService status.WorkflowStatusService - cdTriggerService devtronApps.TriggerService + cdHandlerService devtronApps.HandlerService pipelineRepository pipelineConfig.PipelineRepository installedAppReadService installedAppReader.InstalledAppReadService } @@ -47,7 +47,7 @@ func NewCDPipelineEventProcessorImpl(logger *zap.SugaredLogger, pubSubClient *pubsub.PubSubClientServiceImpl, cdWorkflowCommonService cd.CdWorkflowCommonService, workflowStatusService status.WorkflowStatusService, - cdTriggerService devtronApps.TriggerService, + cdHandlerService devtronApps.HandlerService, pipelineRepository pipelineConfig.PipelineRepository, installedAppReadService installedAppReader.InstalledAppReadService) *CDPipelineEventProcessorImpl { cdPipelineEventProcessorImpl := &CDPipelineEventProcessorImpl{ @@ -55,7 +55,7 @@ func NewCDPipelineEventProcessorImpl(logger *zap.SugaredLogger, pubSubClient: pubSubClient, cdWorkflowCommonService: cdWorkflowCommonService, workflowStatusService: workflowStatusService, - cdTriggerService: cdTriggerService, + cdHandlerService: cdHandlerService, pipelineRepository: pipelineRepository, installedAppReadService: installedAppReadService, } @@ -78,7 +78,7 @@ func (impl *CDPipelineEventProcessorImpl) SubscribeCDBulkTriggerTopic() error { ReferenceId: pointer.String(msg.MsgId), Context: context2.Background(), } - _, _, _, err = impl.cdTriggerService.ManualCdTrigger(triggerContext, event.ValuesOverrideRequest) + _, _, _, err = impl.cdHandlerService.ManualCdTrigger(triggerContext, event.ValuesOverrideRequest) if err != nil { impl.logger.Errorw("Error triggering CD", "topic", pubsub.CD_BULK_DEPLOY_TRIGGER_TOPIC, "msg", msg.Data, "err", err) } diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index 5e97d5d99a..d7c1f8a9bb 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -79,7 +79,7 @@ type WorkflowEventProcessorImpl struct { cdHandler pipeline.CdHandler eventFactory client.EventFactory eventClient client.EventClient - cdTriggerService devtronApps.TriggerService + cdHandlerService devtronApps.HandlerService deployedAppService deployedApp.DeployedAppService webhookService pipeline.WebhookService validator *validator.Validate @@ -93,7 +93,7 @@ type WorkflowEventProcessorImpl struct { appServiceConfig *app.AppServiceConfig //ent only - ciTriggerService trigger.Service + ciHandlerService trigger.HandlerService // repositories import to be removed pipelineRepository pipelineConfig.PipelineRepository @@ -111,7 +111,7 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, workflowDagExecutor dag.WorkflowDagExecutor, ciHandler pipeline.CiHandler, cdHandler pipeline.CdHandler, eventFactory client.EventFactory, eventClient client.EventClient, - cdTriggerService devtronApps.TriggerService, + cdHandlerService devtronApps.HandlerService, deployedAppService deployedApp.DeployedAppService, webhookService pipeline.WebhookService, validator *validator.Validate, @@ -123,7 +123,7 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, ciArtifactRepository repository.CiArtifactRepository, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, deploymentConfigService common.DeploymentConfigService, - ciTriggerService trigger.Service) (*WorkflowEventProcessorImpl, error) { + ciHandlerService trigger.HandlerService) (*WorkflowEventProcessorImpl, error) { impl := &WorkflowEventProcessorImpl{ logger: logger, pubSubClient: pubSubClient, @@ -136,7 +136,7 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, eventFactory: eventFactory, eventClient: eventClient, workflowDagExecutor: workflowDagExecutor, - cdTriggerService: cdTriggerService, + cdHandlerService: cdHandlerService, deployedAppService: deployedAppService, webhookService: webhookService, validator: validator, @@ -150,7 +150,7 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, ciArtifactRepository: ciArtifactRepository, cdWorkflowRepository: cdWorkflowRepository, deploymentConfigService: deploymentConfigService, - ciTriggerService: ciTriggerService, + ciHandlerService: ciHandlerService, } appServiceConfig, err := app.GetAppServiceConfig() if err != nil { @@ -326,7 +326,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeTriggerBulkAction() error { ApplyAuth: false, TriggerContext: triggerContext, } - err = impl.cdTriggerService.TriggerStageForBulk(triggerRequest) + err = impl.cdHandlerService.TriggerStageForBulk(triggerRequest) if err != nil { impl.logger.Errorw("error in cd trigger ", "err", err) wf.WorkflowStatus = cdWorkflowModelBean.TRIGGER_ERROR @@ -400,7 +400,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCIWorkflowStatusUpdate() error return } - err = impl.ciTriggerService.CheckAndReTriggerCI(wfStatus) + err = impl.ciHandlerService.CheckAndReTriggerCI(wfStatus) if err != nil { impl.logger.Errorw("error in checking and re triggering ci", "err", err) //don't return as we have to update the workflow status diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index affa658f1d..a53e90dbd6 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -136,7 +136,7 @@ type WorkflowDagExecutorImpl struct { helmAppService client2.HelmAppService cdWorkflowCommonService cd.CdWorkflowCommonService - cdTriggerService devtronApps.TriggerService + cdHandlerService devtronApps.HandlerService userDeploymentRequestService service.UserDeploymentRequestService manifestCreationService manifest.ManifestCreationService @@ -150,7 +150,7 @@ type WorkflowDagExecutorImpl struct { envRepository repository5.EnvironmentRepository k8sCommonService k8sPkg.K8sCommonService workflowService executor.WorkflowService - ciTriggerService trigger.Service + ciHandlerService trigger.HandlerService } func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, @@ -171,7 +171,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi ciService pipeline.CiService, helmAppService client2.HelmAppService, cdWorkflowCommonService cd.CdWorkflowCommonService, - cdTriggerService devtronApps.TriggerService, + cdHandlerService devtronApps.HandlerService, userDeploymentRequestService service.UserDeploymentRequestService, manifestCreationService manifest.ManifestCreationService, commonArtifactService artifacts.CommonArtifactService, @@ -183,7 +183,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi envRepository repository5.EnvironmentRepository, k8sCommonService k8sPkg.K8sCommonService, workflowService executor.WorkflowService, - ciTriggerService trigger.Service, + ciHandlerService trigger.HandlerService, ) *WorkflowDagExecutorImpl { wde := &WorkflowDagExecutorImpl{logger: Logger, pipelineRepository: pipelineRepository, @@ -202,7 +202,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi pipelineStatusTimelineService: pipelineStatusTimelineService, helmAppService: helmAppService, cdWorkflowCommonService: cdWorkflowCommonService, - cdTriggerService: cdTriggerService, + cdHandlerService: cdHandlerService, userDeploymentRequestService: userDeploymentRequestService, manifestCreationService: manifestCreationService, commonArtifactService: commonArtifactService, @@ -216,7 +216,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi envRepository: envRepository, k8sCommonService: k8sCommonService, workflowService: workflowService, - ciTriggerService: ciTriggerService, + ciHandlerService: ciHandlerService, } config, err := types.GetCdConfig() if err != nil { @@ -324,7 +324,7 @@ func (impl *WorkflowDagExecutorImpl) UpdateCiWorkflowStatusFailure(timeoutForFai if isPodDeleted { ciWorkflow.Message = cdWorkflow2.POD_DELETED_MESSAGE // error logging handled inside handlePodDeleted - impl.ciTriggerService.HandlePodDeleted(ciWorkflow) + impl.ciHandlerService.HandlePodDeleted(ciWorkflow) } else { ciWorkflow.Message = "marked failed by job" } @@ -401,13 +401,13 @@ func (impl *WorkflowDagExecutorImpl) HandleCdStageReTrigger(runner *pipelineConf } if runner.WorkflowType == bean.CD_WORKFLOW_TYPE_PRE { - _, err = impl.cdTriggerService.TriggerPreStage(triggerRequest) + _, err = impl.cdHandlerService.TriggerPreStage(triggerRequest) if err != nil { impl.logger.Errorw("error in TriggerPreStage ", "err", err, "cdWorkflowRunnerId", runner.Id) return err } } else if runner.WorkflowType == bean.CD_WORKFLOW_TYPE_POST { - _, err = impl.cdTriggerService.TriggerPostStage(triggerRequest) + _, err = impl.cdHandlerService.TriggerPostStage(triggerRequest) if err != nil { impl.logger.Errorw("error in TriggerPostStage ", "err", err, "cdWorkflowRunnerId", runner.Id) return err @@ -560,7 +560,7 @@ func (impl *WorkflowDagExecutorImpl) ProcessDevtronAsyncInstallRequest(cdAsyncIn impl.logger.Errorw("error in getting deployment config by appId and envId", "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId, "err", err) return err } - releaseId, _, releaseErr := impl.cdTriggerService.TriggerRelease(newCtx, overrideRequest, envDeploymentConfig, cdAsyncInstallReq.TriggeredAt, cdAsyncInstallReq.TriggeredBy) + releaseId, _, releaseErr := impl.cdHandlerService.TriggerRelease(newCtx, overrideRequest, envDeploymentConfig, cdAsyncInstallReq.TriggeredAt, cdAsyncInstallReq.TriggeredBy) if releaseErr != nil { impl.logger.Errorw("error encountered in ProcessDevtronAsyncInstallRequest", "err", releaseErr, "cdWfrId", cdWfr.Id) impl.handleAsyncTriggerReleaseError(newCtx, releaseErr, cdWfr, overrideRequest) @@ -690,13 +690,13 @@ func (impl *WorkflowDagExecutorImpl) triggerIfAutoStageCdPipeline(request trigge // pre stage exists if request.Pipeline.PreTriggerType == pipelineConfig.TRIGGER_TYPE_AUTOMATIC { impl.logger.Debugw("trigger pre stage for pipeline", "artifactId", request.Artifact.Id, "pipelineId", request.Pipeline.Id) - _, err = impl.cdTriggerService.TriggerPreStage(request) // TODO handle error here + _, err = impl.cdHandlerService.TriggerPreStage(request) // TODO handle error here return err } } else if request.Pipeline.TriggerType == pipelineConfig.TRIGGER_TYPE_AUTOMATIC { // trigger deployment impl.logger.Debugw("trigger cd for pipeline", "artifactId", request.Artifact.Id, "pipelineId", request.Pipeline.Id) - err = impl.cdTriggerService.TriggerAutomaticDeployment(request) + err = impl.cdHandlerService.TriggerAutomaticDeployment(request) return err } return nil @@ -766,7 +766,7 @@ func (impl *WorkflowDagExecutorImpl) HandlePreStageSuccessEvent(triggerContext t } else { ciArtifactId = cdStageCompleteEvent.CiArtifactDTO.Id } - err = impl.cdTriggerService.TriggerAutoCDOnPreStageSuccess(triggerContext, cdStageCompleteEvent.CdPipelineId, ciArtifactId, cdStageCompleteEvent.WorkflowId) + err = impl.cdHandlerService.TriggerAutoCDOnPreStageSuccess(triggerContext, cdStageCompleteEvent.CdPipelineId, ciArtifactId, cdStageCompleteEvent.WorkflowId) if err != nil { impl.logger.Errorw("error in triggering cd on pre cd succcess", "err", err) return err @@ -810,7 +810,7 @@ func (impl *WorkflowDagExecutorImpl) HandleDeploymentSuccessEvent(triggerContext RefCdWorkflowRunnerId: 0, } triggerRequest.TriggerContext.Context = context.Background() - _, err = impl.cdTriggerService.TriggerPostStage(triggerRequest) + _, err = impl.cdHandlerService.TriggerPostStage(triggerRequest) if err != nil { impl.logger.Errorw("error in triggering post stage after successful deployment event", "err", err, "cdWorkflow", cdWorkflow) return err diff --git a/wire_gen.go b/wire_gen.go index e397a3a3a3..4d81fc08b8 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -130,7 +130,7 @@ import ( "github.com/devtron-labs/devtron/pkg/appStore/values/repository" service4 "github.com/devtron-labs/devtron/pkg/appStore/values/service" appWorkflow2 "github.com/devtron-labs/devtron/pkg/appWorkflow" - argoApplication3 "github.com/devtron-labs/devtron/pkg/argoApplication" + "github.com/devtron-labs/devtron/pkg/argoApplication" read22 "github.com/devtron-labs/devtron/pkg/argoApplication/read" config2 "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" "github.com/devtron-labs/devtron/pkg/asyncProvider" @@ -626,7 +626,7 @@ func InitializeApp() (*App, error) { } ciInfraGetter := ci.NewCiInfraGetter(sugaredLogger, infraConfigServiceImpl, infraConfigAuditServiceImpl) infraProviderImpl := infraProviders.NewInfraProviderImpl(sugaredLogger, infraGetter, ciInfraGetter) - workflowServiceImpl, err := executor.NewWorkflowServiceImpl(sugaredLogger, environmentRepositoryImpl, ciCdConfig, configReadServiceImpl, globalCMCSServiceImpl, argoWorkflowExecutorImpl, k8sServiceImpl, systemWorkflowExecutorImpl, k8sCommonServiceImpl, infraProviderImpl) + workflowServiceImpl, err := executor.NewWorkflowServiceImpl(sugaredLogger, environmentRepositoryImpl, ciCdConfig, configReadServiceImpl, globalCMCSServiceImpl, argoWorkflowExecutorImpl, systemWorkflowExecutorImpl, k8sCommonServiceImpl, infraProviderImpl, k8sServiceImpl) if err != nil { return nil, err } @@ -679,8 +679,8 @@ func InitializeApp() (*App, error) { return nil, err } blobStorageConfigServiceImpl := pipeline.NewBlobStorageConfigServiceImpl(sugaredLogger, k8sServiceImpl, ciCdConfig) - serviceImpl := trigger.NewServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciPipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateReadServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, ciCdPipelineOrchestratorImpl, attributesServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl, ciServiceImpl, ciWorkflowRepositoryImpl, clientImpl, ciLogServiceImpl, blobStorageConfigServiceImpl, clusterServiceImplExtended, environmentServiceImpl, k8sServiceImpl) - gitWebhookServiceImpl := gitWebhook.NewGitWebhookServiceImpl(sugaredLogger, gitWebhookRepositoryImpl, serviceImpl) + handlerServiceImpl := trigger.NewHandlerServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciPipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateReadServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, ciCdPipelineOrchestratorImpl, attributesServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl, ciServiceImpl, ciWorkflowRepositoryImpl, clientImpl, ciLogServiceImpl, blobStorageConfigServiceImpl, clusterServiceImplExtended, environmentServiceImpl, k8sServiceImpl) + gitWebhookServiceImpl := gitWebhook.NewGitWebhookServiceImpl(sugaredLogger, gitWebhookRepositoryImpl, handlerServiceImpl) gitWebhookRestHandlerImpl := restHandler.NewGitWebhookRestHandlerImpl(sugaredLogger, gitWebhookServiceImpl) ecrConfig, err := pipeline.GetEcrConfig() if err != nil { @@ -763,13 +763,13 @@ func InitializeApp() (*App, error) { scanToolExecutionHistoryMappingRepositoryImpl := repository24.NewScanToolExecutionHistoryMappingRepositoryImpl(db, sugaredLogger) cdWorkflowReadServiceImpl := read20.NewCdWorkflowReadServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) imageScanServiceImpl := imageScanning.NewImageScanServiceImpl(sugaredLogger, imageScanHistoryRepositoryImpl, imageScanResultRepositoryImpl, imageScanObjectMetaRepositoryImpl, cveStoreRepositoryImpl, imageScanDeployInfoRepositoryImpl, userServiceImpl, appRepositoryImpl, environmentServiceImpl, ciArtifactRepositoryImpl, policyServiceImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, scanToolMetadataRepositoryImpl, scanToolExecutionHistoryMappingRepositoryImpl, cvePolicyRepositoryImpl, cdWorkflowReadServiceImpl) - triggerServiceImpl, err := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, imageDigestPolicyServiceImpl, userServiceImpl, clientImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryReadServiceImpl, imageScanDeployInfoReadServiceImpl, imageScanDeployInfoServiceImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateReadServiceImpl, gitMaterialReadServiceImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, ciCdPipelineOrchestratorImpl, gitOperationServiceImpl, attributesServiceImpl, clusterRepositoryImpl, cdWorkflowRunnerServiceImpl, clusterServiceImplExtended, ciLogServiceImpl, workflowServiceImpl, blobStorageConfigServiceImpl) + devtronAppsHandlerServiceImpl, err := devtronApps.NewHandlerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, imageDigestPolicyServiceImpl, userServiceImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryReadServiceImpl, imageScanDeployInfoReadServiceImpl, imageScanDeployInfoServiceImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateReadServiceImpl, gitMaterialReadServiceImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, ciCdPipelineOrchestratorImpl, gitOperationServiceImpl, attributesServiceImpl, clusterRepositoryImpl, cdWorkflowRunnerServiceImpl, clusterServiceImplExtended, ciLogServiceImpl, workflowServiceImpl, blobStorageConfigServiceImpl) if err != nil { return nil, err } - pipelineConfigRestHandlerImpl := configure.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, deploymentTemplateValidationServiceImpl, chartServiceImpl, devtronAppGitOpConfigServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, generateManifestDeploymentTemplateServiceImpl, appWorkflowServiceImpl, gitMaterialReadServiceImpl, policyServiceImpl, imageScanResultReadServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingReadServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, ciCdPipelineOrchestratorImpl, gitProviderReadServiceImpl, teamReadServiceImpl, environmentRepositoryImpl, chartReadServiceImpl, serviceImpl, triggerServiceImpl) + pipelineConfigRestHandlerImpl := configure.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, deploymentTemplateValidationServiceImpl, chartServiceImpl, devtronAppGitOpConfigServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, generateManifestDeploymentTemplateServiceImpl, appWorkflowServiceImpl, gitMaterialReadServiceImpl, policyServiceImpl, imageScanResultReadServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingReadServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, ciCdPipelineOrchestratorImpl, gitProviderReadServiceImpl, teamReadServiceImpl, environmentRepositoryImpl, chartReadServiceImpl, handlerServiceImpl, devtronAppsHandlerServiceImpl) commonArtifactServiceImpl := artifacts.NewCommonArtifactServiceImpl(sugaredLogger, ciArtifactRepositoryImpl) - workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, cdWorkflowRunnerServiceImpl, ciServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl, runnable, imageScanHistoryRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, environmentRepositoryImpl, k8sCommonServiceImpl, workflowServiceImpl, serviceImpl) + workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, cdWorkflowRunnerServiceImpl, ciServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, devtronAppsHandlerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl, runnable, imageScanHistoryRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, environmentRepositoryImpl, k8sCommonServiceImpl, workflowServiceImpl, handlerServiceImpl) externalCiRestHandlerImpl := restHandler.NewExternalCiRestHandlerImpl(sugaredLogger, validate, userServiceImpl, enforcerImpl, workflowDagExecutorImpl) pubSubClientRestHandlerImpl := restHandler.NewPubSubClientRestHandlerImpl(pubSubClientServiceImpl, sugaredLogger, ciCdConfig) webhookRouterImpl := router.NewWebhookRouterImpl(gitWebhookRestHandlerImpl, pipelineConfigRestHandlerImpl, externalCiRestHandlerImpl, pubSubClientRestHandlerImpl) @@ -824,9 +824,9 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - argoApplicationServiceImpl := argoApplication3.NewArgoApplicationServiceImpl(sugaredLogger, clusterRepositoryImpl, k8sServiceImpl, helmAppClientImpl, helmAppServiceImpl, k8sApplicationServiceImpl, argoApplicationConfigServiceImpl, deploymentConfigServiceImpl) + argoApplicationServiceImpl := argoApplication.NewArgoApplicationServiceImpl(sugaredLogger, clusterRepositoryImpl, k8sServiceImpl, helmAppClientImpl, helmAppServiceImpl, k8sApplicationServiceImpl, argoApplicationConfigServiceImpl, deploymentConfigServiceImpl) argoApplicationReadServiceImpl := read22.NewArgoApplicationReadServiceImpl(sugaredLogger, clusterRepositoryImpl, k8sServiceImpl, helmAppClientImpl, helmAppServiceImpl) - argoApplicationServiceExtendedImpl := argoApplication3.NewArgoApplicationServiceExtendedServiceImpl(acdAuthConfig, argoApplicationServiceImpl, argoClientWrapperServiceImpl, argoApplicationReadServiceImpl, clusterServiceImplExtended) + argoApplicationServiceExtendedImpl := argoApplication.NewArgoApplicationServiceExtendedServiceImpl(acdAuthConfig, argoApplicationServiceImpl, argoClientWrapperServiceImpl, argoApplicationReadServiceImpl, clusterServiceImplExtended) installedAppResourceServiceImpl := resource.NewInstalledAppResourceServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, argoClientWrapperServiceImpl, acdAuthConfig, installedAppVersionHistoryRepositoryImpl, helmAppServiceImpl, helmAppReadServiceImpl, appStatusServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, k8sServiceImpl, deploymentConfigServiceImpl, ociRegistryConfigRepositoryImpl, argoApplicationServiceExtendedImpl) chartGroupEntriesRepositoryImpl := repository28.NewChartGroupEntriesRepositoryImpl(db, sugaredLogger) chartGroupReposotoryImpl := repository28.NewChartGroupReposotoryImpl(db, sugaredLogger) @@ -939,8 +939,8 @@ func InitializeApp() (*App, error) { telemetryRestHandlerImpl := restHandler.NewTelemetryRestHandlerImpl(sugaredLogger, telemetryEventClientImplExtended, enforcerImpl, userServiceImpl) telemetryRouterImpl := router.NewTelemetryRouterImpl(sugaredLogger, telemetryRestHandlerImpl) bulkUpdateRepositoryImpl := bulkUpdate.NewBulkUpdateRepository(db, sugaredLogger) - deployedAppServiceImpl := deployedApp.NewDeployedAppServiceImpl(sugaredLogger, k8sCommonServiceImpl, triggerServiceImpl, environmentRepositoryImpl, pipelineRepositoryImpl, cdWorkflowRepositoryImpl) - bulkUpdateServiceImpl := service7.NewBulkUpdateServiceImpl(bulkUpdateRepositoryImpl, sugaredLogger, environmentRepositoryImpl, pipelineRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineBuilderImpl, enforcerUtilImpl, ciHandlerImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, appWorkflowServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, deployedAppServiceImpl, cdPipelineEventPublishServiceImpl, serviceImpl) + deployedAppServiceImpl := deployedApp.NewDeployedAppServiceImpl(sugaredLogger, k8sCommonServiceImpl, devtronAppsHandlerServiceImpl, environmentRepositoryImpl, pipelineRepositoryImpl, cdWorkflowRepositoryImpl) + bulkUpdateServiceImpl := service7.NewBulkUpdateServiceImpl(bulkUpdateRepositoryImpl, sugaredLogger, environmentRepositoryImpl, pipelineRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineBuilderImpl, enforcerUtilImpl, ciHandlerImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, appWorkflowServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, deployedAppServiceImpl, cdPipelineEventPublishServiceImpl, handlerServiceImpl) bulkUpdateRestHandlerImpl := restHandler.NewBulkUpdateRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, bulkUpdateServiceImpl, chartServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, environmentServiceImpl, gitRegistryConfigImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl) bulkUpdateRouterImpl := router.NewBulkUpdateRouterImpl(bulkUpdateRestHandlerImpl) webhookSecretValidatorImpl := gitWebhook.NewWebhookSecretValidatorImpl(sugaredLogger) @@ -951,13 +951,13 @@ func InitializeApp() (*App, error) { webhookListenerRouterImpl := router.NewWebhookListenerRouterImpl(webhookEventHandlerImpl) appFilteringRestHandlerImpl := appList.NewAppFilteringRestHandlerImpl(sugaredLogger, teamServiceImpl, enforcerImpl, userServiceImpl, clusterServiceImplExtended, environmentServiceImpl, teamReadServiceImpl) appFilteringRouterImpl := appList2.NewAppFilteringRouterImpl(appFilteringRestHandlerImpl) - resourceTreeServiceImpl := resourceTree.NewServiceImpl(sugaredLogger, appListingServiceImpl, appStatusServiceImpl, argoApplicationServiceExtendedImpl, cdApplicationStatusUpdateHandlerImpl, helmAppReadServiceImpl, helmAppServiceImpl, k8sApplicationServiceImpl, k8sCommonServiceImpl, environmentReadServiceImpl) - appListingRestHandlerImpl := appList.NewAppListingRestHandlerImpl(appListingServiceImpl, enforcerImpl, pipelineBuilderImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, userServiceImpl, k8sCommonServiceImpl, installedAppDBExtendedServiceImpl, installedAppResourceServiceImpl, pipelineRepositoryImpl, k8sApplicationServiceImpl, deploymentConfigServiceImpl, resourceTreeServiceImpl) + serviceImpl := resourceTree.NewServiceImpl(sugaredLogger, appListingServiceImpl, appStatusServiceImpl, argoApplicationServiceExtendedImpl, cdApplicationStatusUpdateHandlerImpl, helmAppReadServiceImpl, helmAppServiceImpl, k8sApplicationServiceImpl, k8sCommonServiceImpl, environmentReadServiceImpl) + appListingRestHandlerImpl := appList.NewAppListingRestHandlerImpl(appListingServiceImpl, enforcerImpl, pipelineBuilderImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, userServiceImpl, k8sCommonServiceImpl, installedAppDBExtendedServiceImpl, installedAppResourceServiceImpl, pipelineRepositoryImpl, k8sApplicationServiceImpl, deploymentConfigServiceImpl, serviceImpl) appListingRouterImpl := appList2.NewAppListingRouterImpl(appListingRestHandlerImpl) appInfoRestHandlerImpl := appInfo.NewAppInfoRestHandlerImpl(sugaredLogger, appCrudOperationServiceImpl, userServiceImpl, validate, enforcerUtilImpl, enforcerImpl, helmAppServiceImpl, enforcerUtilHelmImpl, genericNoteServiceImpl, commonEnforcementUtilImpl) appInfoRouterImpl := appInfo2.NewAppInfoRouterImpl(sugaredLogger, appInfoRestHandlerImpl) pipelineDeploymentConfigServiceImpl := pipeline.NewPipelineDeploymentConfigServiceImpl(sugaredLogger, chartRepositoryImpl, pipelineRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, scopedVariableCMCSManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, configMapHistoryReadServiceImpl, envConfigOverrideReadServiceImpl) - pipelineTriggerRestHandlerImpl := trigger2.NewPipelineRestHandler(appServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, pipelineDeploymentConfigServiceImpl, deployedAppServiceImpl, triggerServiceImpl, workflowEventPublishServiceImpl) + pipelineTriggerRestHandlerImpl := trigger2.NewPipelineRestHandler(appServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, pipelineDeploymentConfigServiceImpl, deployedAppServiceImpl, devtronAppsHandlerServiceImpl, workflowEventPublishServiceImpl) sseSSE := sse.NewSSE() pipelineTriggerRouterImpl := trigger3.NewPipelineTriggerRouter(pipelineTriggerRestHandlerImpl, sseSSE) webhookDataRestHandlerImpl := webhook.NewWebhookDataRestHandlerImpl(sugaredLogger, userServiceImpl, ciPipelineMaterialRepositoryImpl, enforcerUtilImpl, enforcerImpl, clientImpl, webhookEventDataConfigImpl) @@ -1045,7 +1045,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - ciTriggerCronImpl := cron2.NewCiTriggerCronImpl(sugaredLogger, ciTriggerCronConfig, pipelineStageRepositoryImpl, ciArtifactRepositoryImpl, globalPluginRepositoryImpl, cronLoggerImpl, serviceImpl) + ciTriggerCronImpl := cron2.NewCiTriggerCronImpl(sugaredLogger, ciTriggerCronConfig, pipelineStageRepositoryImpl, ciArtifactRepositoryImpl, globalPluginRepositoryImpl, cronLoggerImpl, handlerServiceImpl) proxyConfig, err := proxy.GetProxyConfig() if err != nil { return nil, err @@ -1081,12 +1081,12 @@ func InitializeApp() (*App, error) { cdWorkflowServiceImpl := cd.NewCdWorkflowServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) cdWorkflowRunnerReadServiceImpl := read20.NewCdWorkflowRunnerReadServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) webhookServiceImpl := pipeline.NewWebhookServiceImpl(ciArtifactRepositoryImpl, sugaredLogger, ciPipelineRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowCommonServiceImpl, workFlowStageStatusServiceImpl, ciServiceImpl) - workflowEventProcessorImpl, err := in.NewWorkflowEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowServiceImpl, cdWorkflowReadServiceImpl, cdWorkflowRunnerServiceImpl, cdWorkflowRunnerReadServiceImpl, workflowDagExecutorImpl, ciHandlerImpl, cdHandlerImpl, eventSimpleFactoryImpl, eventRESTClientImpl, triggerServiceImpl, deployedAppServiceImpl, webhookServiceImpl, validate, environmentVariables, cdWorkflowCommonServiceImpl, cdPipelineConfigServiceImpl, userDeploymentRequestServiceImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, cdWorkflowRepositoryImpl, deploymentConfigServiceImpl, serviceImpl) + workflowEventProcessorImpl, err := in.NewWorkflowEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowServiceImpl, cdWorkflowReadServiceImpl, cdWorkflowRunnerServiceImpl, cdWorkflowRunnerReadServiceImpl, workflowDagExecutorImpl, ciHandlerImpl, cdHandlerImpl, eventSimpleFactoryImpl, eventRESTClientImpl, devtronAppsHandlerServiceImpl, deployedAppServiceImpl, webhookServiceImpl, validate, environmentVariables, cdWorkflowCommonServiceImpl, cdPipelineConfigServiceImpl, userDeploymentRequestServiceImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, cdWorkflowRepositoryImpl, deploymentConfigServiceImpl, handlerServiceImpl) if err != nil { return nil, err } ciPipelineEventProcessorImpl := in.NewCIPipelineEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, gitWebhookServiceImpl) - cdPipelineEventProcessorImpl := in.NewCDPipelineEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowCommonServiceImpl, workflowStatusServiceImpl, triggerServiceImpl, pipelineRepositoryImpl, installedAppReadServiceImpl) + cdPipelineEventProcessorImpl := in.NewCDPipelineEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowCommonServiceImpl, workflowStatusServiceImpl, devtronAppsHandlerServiceImpl, pipelineRepositoryImpl, installedAppReadServiceImpl) deployedApplicationEventProcessorImpl := in.NewDeployedApplicationEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, appServiceImpl, gitOpsConfigReadServiceImpl, installedAppDBExtendedServiceImpl, workflowDagExecutorImpl, cdWorkflowCommonServiceImpl, pipelineBuilderImpl, appStoreDeploymentServiceImpl, pipelineRepositoryImpl, installedAppReadServiceImpl, deploymentConfigServiceImpl) appStoreAppsEventProcessorImpl := in.NewAppStoreAppsEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, chartGroupServiceImpl, installedAppVersionHistoryRepositoryImpl) centralEventProcessor, err := eventProcessor.NewCentralEventProcessor(sugaredLogger, workflowEventProcessorImpl, ciPipelineEventProcessorImpl, cdPipelineEventProcessorImpl, deployedApplicationEventProcessorImpl, appStoreAppsEventProcessorImpl) From b21694cffac3226bdaada1beff32a1b46868f113 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 8 Apr 2025 12:29:47 +0530 Subject: [PATCH 15/17] file renaming --- .../trigger/devtronApps/HandlerService.go | 1330 +---------------- ..._ent1.go => deployStageHandlerCode_ent.go} | 16 + ...ggerService.go => postStageHandlerCode.go} | 0 ...ice_ent.go => postStageHandlerCode_ent.go} | 0 ...mmonService.go => prePostWfAndLogsCode.go} | 0 ...vice_ent.go => preStageHandlerCode_ent.go} | 0 ...erService.go => preStageHandlerService.go} | 0 7 files changed, 29 insertions(+), 1317 deletions(-) rename pkg/deployment/trigger/devtronApps/{HandlerService_ent1.go => deployStageHandlerCode_ent.go} (84%) rename pkg/deployment/trigger/devtronApps/{PostStageTriggerService.go => postStageHandlerCode.go} (100%) rename pkg/deployment/trigger/devtronApps/{PostStageTriggerService_ent.go => postStageHandlerCode_ent.go} (100%) rename pkg/deployment/trigger/devtronApps/{PrePostCommonService.go => prePostWfAndLogsCode.go} (100%) rename pkg/deployment/trigger/devtronApps/{PreStageTriggerService_ent.go => preStageHandlerCode_ent.go} (100%) rename pkg/deployment/trigger/devtronApps/{PreStageTriggerService.go => preStageHandlerService.go} (100%) diff --git a/pkg/deployment/trigger/devtronApps/HandlerService.go b/pkg/deployment/trigger/devtronApps/HandlerService.go index 7c915a3195..c92a009134 100644 --- a/pkg/deployment/trigger/devtronApps/HandlerService.go +++ b/pkg/deployment/trigger/devtronApps/HandlerService.go @@ -19,36 +19,25 @@ package devtronApps import ( "bufio" "context" - "errors" - "fmt" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" util5 "github.com/devtron-labs/common-lib/utils/k8s" bean3 "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/api/bean/gitOps" - bean6 "github.com/devtron-labs/devtron/api/helm-app/bean" "github.com/devtron-labs/devtron/api/helm-app/gRPC" client2 "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/client/argocdServer" - bean7 "github.com/devtron-labs/devtron/client/argocdServer/bean" client "github.com/devtron-labs/devtron/client/events" - "github.com/devtron-labs/devtron/internal/middleware" - "github.com/devtron-labs/devtron/internal/sql/models" repository3 "github.com/devtron-labs/devtron/internal/sql/repository" appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" repository4 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" bean4 "github.com/devtron-labs/devtron/pkg/app/bean" "github.com/devtron-labs/devtron/pkg/app/status" - statusBean "github.com/devtron-labs/devtron/pkg/app/status/bean" "github.com/devtron-labs/devtron/pkg/attributes" "github.com/devtron-labs/devtron/pkg/auth/user" - bean2 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/read" pipeline2 "github.com/devtron-labs/devtron/pkg/build/pipeline" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" @@ -60,49 +49,43 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "github.com/devtron-labs/devtron/pkg/deployment/manifest" - bean10 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/bean" - bean5 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" "github.com/devtron-labs/devtron/pkg/deployment/manifest/publish" - "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/adapter" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" - "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/helper" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/userDeploymentRequest/service" - clientErrors "github.com/devtron-labs/devtron/pkg/errors" "github.com/devtron-labs/devtron/pkg/eventProcessor/out" "github.com/devtron-labs/devtron/pkg/executor" "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" - k8s2 "github.com/devtron-labs/devtron/pkg/k8s" "github.com/devtron-labs/devtron/pkg/pipeline" - bean8 "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/history" "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/plugin" security2 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning" read2 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/read" - repository6 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/repository" "github.com/devtron-labs/devtron/pkg/sql" "github.com/devtron-labs/devtron/pkg/variables" "github.com/devtron-labs/devtron/pkg/workflow/cd" globalUtil "github.com/devtron-labs/devtron/util" util2 "github.com/devtron-labs/devtron/util/event" "github.com/devtron-labs/devtron/util/rbac" - "github.com/go-pg/pg" - "go.opentelemetry.io/otel" "go.uber.org/zap" - "golang.org/x/exp/slices" - "google.golang.org/grpc/codes" - status2 "google.golang.org/grpc/status" - "helm.sh/helm/v3/pkg/chart" - "net/http" "os" - "path" - "regexp" - "strconv" - "strings" "time" ) +/* +files in this package are - +HandlerService.go - containing If and impl with common used code +HandlerService_ent.go - containing ent If and impl with common used code +deployStageHandlerCode.go - code related to deploy stage trigger +deployStageHandlerCode_ent.go - ent code related to deploy stage trigger +preStageHandlerCode.go - code related to pre stage trigger +preStageHandlerCode_ent.go - ent code related to pre stage trigger +postStageHandlerCode.go - code related to post stage trigger +postStageHandlerCode_ent.go - ent code related to post stage trigger +prePostWfAndLogsCode.go - code containing pre/post wf handling(abort) and logs related code +*/ + type HandlerService interface { TriggerPostStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) TriggerPreStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) @@ -318,1112 +301,6 @@ func NewHandlerServiceImpl(logger *zap.SugaredLogger, return impl, nil } -func (impl *HandlerServiceImpl) TriggerStageForBulk(triggerRequest bean.TriggerRequest) error { - - preStage, err := impl.pipelineStageService.GetCdStageByCdPipelineIdAndStageType(triggerRequest.Pipeline.Id, repository.PIPELINE_STAGE_TYPE_PRE_CD, false) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in fetching CD pipeline stage", "cdPipelineId", triggerRequest.Pipeline.Id, "stage ", repository.PIPELINE_STAGE_TYPE_PRE_CD, "err", err) - return err - } - - // handle corrupt data (https://github.com/devtron-labs/devtron/issues/3826) - err, deleted := impl.deleteCorruptedPipelineStage(preStage, triggerRequest.TriggeredBy) - if err != nil { - impl.logger.Errorw("error in deleteCorruptedPipelineStage ", "cdPipelineId", triggerRequest.Pipeline.Id, "err", err, "preStage", preStage, "triggeredBy", triggerRequest.TriggeredBy) - return err - } - - triggerRequest.TriggerContext.Context = context.Background() - if len(triggerRequest.Pipeline.PreStageConfig) > 0 || (preStage != nil && !deleted) { - // pre stage exists - impl.logger.Debugw("trigger pre stage for pipeline", "artifactId", triggerRequest.Artifact.Id, "pipelineId", triggerRequest.Pipeline.Id) - triggerRequest.RefCdWorkflowRunnerId = 0 - err = impl.preStageHandlingForTriggerStageInBulk(&triggerRequest) - if err != nil { - return err - } - _, err = impl.TriggerPreStage(triggerRequest) // TODO handle error here - return err - } else { - // trigger deployment - impl.logger.Debugw("trigger cd for pipeline", "artifactId", triggerRequest.Artifact.Id, "pipelineId", triggerRequest.Pipeline.Id) - err = impl.TriggerAutomaticDeployment(triggerRequest) - return err - } -} - -func (impl *HandlerServiceImpl) getCdPipelineForManualCdTrigger(ctx context.Context, pipelineId int) (*pipelineConfig.Pipeline, error) { - _, span := otel.Tracer("HandlerService").Start(ctx, "getCdPipelineForManualCdTrigger") - defer span.End() - cdPipeline, err := impl.pipelineRepository.FindById(pipelineId) - if err != nil { - impl.logger.Errorw("manual trigger request with invalid pipelineId, ManualCdTrigger", "pipelineId", pipelineId, "err", err) - return nil, err - } - //checking if namespace exist or not - clusterIdToNsMap := map[int]string{ - cdPipeline.Environment.ClusterId: cdPipeline.Environment.Namespace, - } - - err = impl.helmAppService.CheckIfNsExistsForClusterIds(clusterIdToNsMap) - if err != nil { - impl.logger.Errorw("manual trigger request with invalid namespace, ManualCdTrigger", "pipelineId", pipelineId, "err", err) - return nil, err - } - return cdPipeline, nil -} - -func (impl *HandlerServiceImpl) validateDeploymentTriggerRequest(ctx context.Context, validateDeploymentTriggerObj *bean.ValidateDeploymentTriggerObj) error { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.validateDeploymentTriggerRequest") - defer span.End() - // custom GitOps repo url validation --> Start - err := impl.handleCustomGitOpsRepoValidation(validateDeploymentTriggerObj.Runner, validateDeploymentTriggerObj.CdPipeline, validateDeploymentTriggerObj.DeploymentConfig, validateDeploymentTriggerObj.TriggeredBy) - if err != nil { - impl.logger.Errorw("custom GitOps repository validation error, TriggerStage", "err", err) - return err - } - // custom GitOps repo url validation --> Ends - var isVulnerable bool - // if request is for rollback then bypass vulnerability validation - if !validateDeploymentTriggerObj.IsDeploymentTypeRollback() { - // checking vulnerability for deploying image - vulnerabilityCheckRequest := adapter.GetVulnerabilityCheckRequest(validateDeploymentTriggerObj.CdPipeline, validateDeploymentTriggerObj.ImageDigest) - isVulnerable, err = impl.imageScanService.GetArtifactVulnerabilityStatus(newCtx, vulnerabilityCheckRequest) - if err != nil { - impl.logger.Errorw("error in getting Artifact vulnerability status, ManualCdTrigger", "err", err) - return err - } - } - - if isVulnerable == true { - // if image vulnerable, update timeline status and return - if err = impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(validateDeploymentTriggerObj.Runner, errors.New(cdWorkflow.FOUND_VULNERABILITY), validateDeploymentTriggerObj.TriggeredBy); err != nil { - impl.logger.Errorw("error while updating current runner status to failed, TriggerDeployment", "wfrId", validateDeploymentTriggerObj.Runner.Id, "err", err) - } - return fmt.Errorf("found vulnerability for image digest %s", validateDeploymentTriggerObj.ImageDigest) - } - return nil -} - -// TODO: write a wrapper to handle auto and manual trigger -func (impl *HandlerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerContext, overrideRequest *bean3.ValuesOverrideRequest) (int, string, *bean4.ManifestPushTemplate, error) { - - triggerContext.TriggerType = bean.Manual - // setting triggeredAt variable to have consistent data for various audit log places in db for deployment time - triggeredAt := time.Now() - - releaseId := 0 - ctx := triggerContext.Context - cdPipeline, err := impl.getCdPipelineForManualCdTrigger(ctx, overrideRequest.PipelineId) - if err != nil { - if overrideRequest.WfrId != 0 { - err2 := impl.cdWorkflowCommonService.MarkDeploymentFailedForRunnerId(overrideRequest.WfrId, err, overrideRequest.UserId) - if err2 != nil { - impl.logger.Errorw("error while updating current runner status to failed, ManualCdTrigger", "cdWfr", overrideRequest.WfrId, "err2", err2) - } - } - return 0, "", nil, err - } - envDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(cdPipeline.AppId, cdPipeline.EnvironmentId) - if err != nil { - impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", cdPipeline.AppId, "envId", cdPipeline.EnvironmentId, "err", err) - return 0, "", nil, err - } - - adapter.SetPipelineFieldsInOverrideRequest(overrideRequest, cdPipeline, envDeploymentConfig) - ciArtifactId := overrideRequest.CiArtifactId - - _, span := otel.Tracer("orchestrator").Start(ctx, "ciArtifactRepository.Get") - artifact, err := impl.ciArtifactRepository.Get(ciArtifactId) - span.End() - if err != nil { - impl.logger.Errorw("error in getting CiArtifact", "CiArtifactId", overrideRequest.CiArtifactId, "err", err) - return 0, "", nil, err - } - - if artifact.IsMigrationRequired() { - // Migration of deprecated DataSource Type - migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(artifact.Id) - if migrationErr != nil { - impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id) - } - } - - _, imageTag, err := artifact.ExtractImageRepoAndTag() - if err != nil { - impl.logger.Errorw("error in getting image tag and repo", "err", err) - } - helmPackageName := fmt.Sprintf("%s-%s-%s", cdPipeline.App.AppName, cdPipeline.Environment.Name, imageTag) - var manifestPushTemplate *bean4.ManifestPushTemplate - - switch overrideRequest.CdWorkflowType { - case bean3.CD_WORKFLOW_TYPE_PRE: - var cdWf *pipelineConfig.CdWorkflow - if overrideRequest.CdWorkflowId == 0 { - cdWf = &pipelineConfig.CdWorkflow{ - CiArtifactId: artifact.Id, - PipelineId: cdPipeline.Id, - AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: 1, UpdatedOn: triggeredAt, UpdatedBy: 1}, - } - err := impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) - if err != nil { - return 0, "", nil, err - } - } else { - cdWf, err = impl.cdWorkflowRepository.FindById(overrideRequest.CdWorkflowId) - if err != nil { - impl.logger.Errorw("error in TriggerPreStage, ManualCdTrigger", "err", err) - return 0, "", nil, err - } - } - overrideRequest.CdWorkflowId = cdWf.Id - - _, span = otel.Tracer("orchestrator").Start(ctx, "TriggerPreStage") - triggerRequest := bean.TriggerRequest{ - CdWf: cdWf, - Artifact: artifact, - Pipeline: cdPipeline, - TriggeredBy: overrideRequest.UserId, - ApplyAuth: false, - TriggerContext: triggerContext, - RefCdWorkflowRunnerId: 0, - CdWorkflowRunnerId: overrideRequest.WfrId, - } - manifestPushTemplate, err = impl.TriggerPreStage(triggerRequest) - span.End() - if err != nil { - impl.logger.Errorw("error in TriggerPreStage, ManualCdTrigger", "err", err) - return 0, "", nil, err - } - case bean3.CD_WORKFLOW_TYPE_DEPLOY: - if overrideRequest.DeploymentType == models.DEPLOYMENTTYPE_UNKNOWN { - overrideRequest.DeploymentType = models.DEPLOYMENTTYPE_DEPLOY - } - - cdWf, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, overrideRequest.CdWorkflowId, bean3.CD_WORKFLOW_TYPE_PRE) - if err != nil && !util.IsErrNoRows(err) { - impl.logger.Errorw("error in getting cdWorkflow, ManualCdTrigger", "CdWorkflowId", overrideRequest.CdWorkflowId, "err", err) - return 0, "", nil, err - } - - cdWorkflowId := cdWf.CdWorkflowId - if cdWf.CdWorkflowId == 0 { - cdWf := &pipelineConfig.CdWorkflow{ - CiArtifactId: overrideRequest.CiArtifactId, - PipelineId: overrideRequest.PipelineId, - AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, - } - err := impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) - if err != nil { - impl.logger.Errorw("error in creating cdWorkflow, ManualCdTrigger", "PipelineId", overrideRequest.PipelineId, "err", err) - return 0, "", nil, err - } - cdWorkflowId = cdWf.Id - } - - runner := &pipelineConfig.CdWorkflowRunner{ - Name: cdPipeline.Name, - WorkflowType: bean3.CD_WORKFLOW_TYPE_DEPLOY, - ExecutorType: cdWorkflow.WORKFLOW_EXECUTOR_TYPE_AWF, - Status: cdWorkflow.WorkflowInitiated, //deployment Initiated for manual trigger - TriggeredBy: overrideRequest.UserId, - StartedOn: triggeredAt, - Namespace: impl.config.GetDefaultNamespace(), - CdWorkflowId: cdWorkflowId, - AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, - ReferenceId: triggerContext.ReferenceId, - } - savedWfr, err := impl.cdWorkflowRunnerService.SaveCDWorkflowRunnerWithStage(runner) - if err != nil { - impl.logger.Errorw("err in creating cdWorkflowRunner, ManualCdTrigger", "cdWorkflowId", cdWorkflowId, "err", err) - return 0, "", nil, err - } - runner.CdWorkflow = &pipelineConfig.CdWorkflow{ - Pipeline: cdPipeline, - } - overrideRequest.WfrId = savedWfr.Id - overrideRequest.CdWorkflowId = cdWorkflowId - // creating cd pipeline status timeline for deployment initialisation - timeline := impl.pipelineStatusTimelineService.NewDevtronAppPipelineStatusTimelineDbObject(runner.Id, timelineStatus.TIMELINE_STATUS_DEPLOYMENT_INITIATED, timelineStatus.TIMELINE_DESCRIPTION_DEPLOYMENT_INITIATED, overrideRequest.UserId) - _, span = otel.Tracer("orchestrator").Start(ctx, "cdPipelineStatusTimelineRepo.SaveTimelineForACDHelmApps") - _, err = impl.pipelineStatusTimelineService.SaveTimelineIfNotAlreadyPresent(timeline, nil) - - span.End() - if err != nil { - impl.logger.Errorw("error in creating timeline status for deployment initiation, ManualCdTrigger", "err", err, "timeline", timeline) - } - if isNotHibernateRequest(overrideRequest.DeploymentType) { - validateReqObj := adapter.NewValidateDeploymentTriggerObj(runner, cdPipeline, artifact.ImageDigest, envDeploymentConfig, overrideRequest.UserId, overrideRequest.IsRollbackDeployment) - validationErr := impl.validateDeploymentTriggerRequest(ctx, validateReqObj) - if validationErr != nil { - impl.logger.Errorw("validation error deployment request", "cdWfr", runner.Id, "err", validationErr) - return 0, "", nil, validationErr - } - } - // Deploy the release - var releaseErr error - releaseId, manifestPushTemplate, releaseErr = impl.handleCDTriggerRelease(ctx, overrideRequest, envDeploymentConfig, triggeredAt, overrideRequest.UserId) - // if releaseErr found, then the mark current deployment Failed and return - if releaseErr != nil { - err := impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, releaseErr, overrideRequest.UserId) - if err != nil { - impl.logger.Errorw("error while updating current runner status to failed", "cdWfr", runner.Id, "err", err) - } - return 0, "", nil, releaseErr - } - - case bean3.CD_WORKFLOW_TYPE_POST: - cdWfRunner, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, overrideRequest.CdWorkflowId, bean3.CD_WORKFLOW_TYPE_DEPLOY) - if err != nil && !util.IsErrNoRows(err) { - impl.logger.Errorw("err in getting cdWorkflowRunner, ManualCdTrigger", "cdWorkflowId", overrideRequest.CdWorkflowId, "err", err) - return 0, "", nil, err - } - - var cdWf *pipelineConfig.CdWorkflow - if cdWfRunner.CdWorkflowId == 0 { - cdWf = &pipelineConfig.CdWorkflow{ - CiArtifactId: ciArtifactId, - PipelineId: overrideRequest.PipelineId, - AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, - } - err := impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) - if err != nil { - impl.logger.Errorw("error in creating cdWorkflow, ManualCdTrigger", "CdWorkflowId", overrideRequest.CdWorkflowId, "err", err) - return 0, "", nil, err - } - overrideRequest.CdWorkflowId = cdWf.Id - } else { - _, span = otel.Tracer("orchestrator").Start(ctx, "cdWorkflowRepository.FindById") - cdWf, err = impl.cdWorkflowRepository.FindById(overrideRequest.CdWorkflowId) - span.End() - if err != nil && !util.IsErrNoRows(err) { - impl.logger.Errorw("error in getting cdWorkflow, ManualCdTrigger", "CdWorkflowId", overrideRequest.CdWorkflowId, "err", err) - return 0, "", nil, err - } - } - _, span = otel.Tracer("orchestrator").Start(ctx, "TriggerPostStage") - triggerRequest := bean.TriggerRequest{ - CdWf: cdWf, - Pipeline: cdPipeline, - TriggeredBy: overrideRequest.UserId, - RefCdWorkflowRunnerId: 0, - TriggerContext: triggerContext, - CdWorkflowRunnerId: overrideRequest.WfrId, - } - manifestPushTemplate, err = impl.TriggerPostStage(triggerRequest) - span.End() - if err != nil { - impl.logger.Errorw("error in TriggerPostStage, ManualCdTrigger", "CdWorkflowId", cdWf.Id, "err", err) - return 0, "", nil, err - } - default: - impl.logger.Errorw("invalid CdWorkflowType, ManualCdTrigger", "CdWorkflowType", overrideRequest.CdWorkflowType, "err", err) - return 0, "", nil, fmt.Errorf("invalid CdWorkflowType %s for the trigger request", string(overrideRequest.CdWorkflowType)) - } - return releaseId, helmPackageName, manifestPushTemplate, err -} - -func isNotHibernateRequest(deploymentType models.DeploymentType) bool { - return deploymentType != models.DEPLOYMENTTYPE_STOP && deploymentType != models.DEPLOYMENTTYPE_START -} - -// TODO: write a wrapper to handle auto and manual trigger -func (impl *HandlerServiceImpl) TriggerAutomaticDeployment(request bean.TriggerRequest) error { - // in case of manual trigger auth is already applied and for auto triggers there is no need for auth check here - triggeredBy := request.TriggeredBy - pipeline := request.Pipeline - artifact := request.Artifact - - //setting triggeredAt variable to have consistent data for various audit log places in db for deployment time - triggeredAt := time.Now() - cdWf := request.CdWf - ctx := context.Background() - - if cdWf == nil || (cdWf != nil && cdWf.CiArtifactId != artifact.Id) { - // cdWf != nil && cdWf.CiArtifactId != artifact.Id for auto trigger case when deployment is triggered with image generated by plugin - cdWf = &pipelineConfig.CdWorkflow{ - CiArtifactId: artifact.Id, - PipelineId: pipeline.Id, - AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: 1, UpdatedOn: triggeredAt, UpdatedBy: 1}, - } - err := impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) - if err != nil { - return err - } - } - - runner := &pipelineConfig.CdWorkflowRunner{ - Name: pipeline.Name, - WorkflowType: bean3.CD_WORKFLOW_TYPE_DEPLOY, - ExecutorType: cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM, - Status: cdWorkflow.WorkflowInitiated, // deployment Initiated for auto trigger - TriggeredBy: triggeredBy, - StartedOn: triggeredAt, - Namespace: impl.config.GetDefaultNamespace(), - CdWorkflowId: cdWf.Id, - AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: triggeredBy, UpdatedOn: triggeredAt, UpdatedBy: triggeredBy}, - ReferenceId: request.TriggerContext.ReferenceId, - } - savedWfr, err := impl.cdWorkflowRunnerService.SaveCDWorkflowRunnerWithStage(runner) - if err != nil { - return err - } - runner.CdWorkflow = &pipelineConfig.CdWorkflow{ - Pipeline: pipeline, - } - // creating cd pipeline status timeline for deployment initialisation - timeline := &pipelineConfig.PipelineStatusTimeline{ - CdWorkflowRunnerId: runner.Id, - Status: timelineStatus.TIMELINE_STATUS_DEPLOYMENT_INITIATED, - StatusDetail: "Deployment initiated successfully.", - StatusTime: time.Now(), - } - timeline.CreateAuditLog(1) - err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil) - if err != nil { - impl.logger.Errorw("error in creating timeline status for deployment initiation", "err", err, "timeline", timeline) - } - envDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(pipeline.AppId, pipeline.EnvironmentId) - if err != nil { - impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) - return err - } - validationErr := impl.validateDeploymentTriggerRequest(ctx, adapter.NewValidateDeploymentTriggerObj(runner, pipeline, artifact.ImageDigest, envDeploymentConfig, triggeredBy, false)) - if validationErr != nil { - impl.logger.Errorw("validation error deployment request", "cdWfr", runner.Id, "err", validationErr) - return validationErr - } - releaseErr := impl.TriggerCD(ctx, artifact, cdWf.Id, savedWfr.Id, pipeline, envDeploymentConfig, triggeredAt, triggeredBy) - // if releaseErr found, then the mark current deployment Failed and return - if releaseErr != nil { - err := impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, releaseErr, triggeredBy) - if err != nil { - impl.logger.Errorw("error while updating current runner status to failed, updatePreviousDeploymentStatus", "cdWfr", runner.Id, "err", err) - } - return releaseErr - } - return nil -} - -func (impl *HandlerServiceImpl) TriggerCD(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) error { - impl.logger.Debugw("automatic pipeline trigger attempt async", "artifactId", artifact.Id) - err := impl.triggerReleaseAsync(ctx, artifact, cdWorkflowId, wfrId, pipeline, envDeploymentConfig, triggeredAt, triggeredBy) - if err != nil { - impl.logger.Errorw("error in cd trigger", "err", err) - return err - } - return err -} - -func (impl *HandlerServiceImpl) triggerReleaseAsync(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) error { - err := impl.validateAndTrigger(ctx, pipeline, envDeploymentConfig, artifact, cdWorkflowId, wfrId, triggeredAt, triggeredBy) - if err != nil { - impl.logger.Errorw("error in trigger for pipeline", "pipelineId", strconv.Itoa(pipeline.Id)) - } - impl.logger.Debugw("trigger attempted for all pipeline ", "artifactId", artifact.Id) - return err -} - -func (impl *HandlerServiceImpl) validateAndTrigger(ctx context.Context, p *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time, triggeredBy int32) error { - //TODO: verify this logic - object := impl.enforcerUtil.GetAppRBACNameByAppId(p.AppId) - envApp := strings.Split(object, "/") - if len(envApp) != 2 { - impl.logger.Error("invalid req, app and env not found from rbac") - return errors.New("invalid req, app and env not found from rbac") - } - err := impl.releasePipeline(ctx, p, envDeploymentConfig, artifact, cdWorkflowId, wfrId, triggeredAt, triggeredBy) - return err -} - -func (impl *HandlerServiceImpl) releasePipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time, triggeredBy int32) error { - startTime := time.Now() - defer func() { - impl.logger.Debugw("auto trigger release process completed", "timeTaken", time.Since(startTime), "cdPipelineId", pipeline.Id, "artifactId", artifact.Id, "wfrId", wfrId) - }() - impl.logger.Debugw("auto triggering release for", "cdPipelineId", pipeline.Id, "artifactId", artifact.Id, "wfrId", wfrId) - pipeline, err := impl.pipelineRepository.FindById(pipeline.Id) - if err != nil { - impl.logger.Errorw("error in fetching pipeline by pipelineId", "err", err) - return err - } - - request := &bean3.ValuesOverrideRequest{ - PipelineId: pipeline.Id, - UserId: artifact.CreatedBy, - CiArtifactId: artifact.Id, - AppId: pipeline.AppId, - CdWorkflowId: cdWorkflowId, - ForceTrigger: true, - DeploymentWithConfig: bean3.DEPLOYMENT_CONFIG_TYPE_LAST_SAVED, - WfrId: wfrId, - } - - adapter.SetPipelineFieldsInOverrideRequest(request, pipeline, envDeploymentConfig) - - id, _, err := impl.handleCDTriggerRelease(ctx, request, envDeploymentConfig, triggeredAt, triggeredBy) - if err != nil { - impl.logger.Errorw("error in auto cd pipeline trigger", "pipelineId", pipeline.Id, "artifactId", artifact.Id, "err", err) - } else { - impl.logger.Infow("pipeline successfully triggered", "cdPipelineId", pipeline.Id, "artifactId", artifact.Id, "releaseId", id) - } - return err -} - -func (impl *HandlerServiceImpl) triggerAsyncRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, - envDeploymentConfig *bean9.DeploymentConfig, userDeploymentRequestId int, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.triggerAsyncRelease") - defer span.End() - // build merged values and save PCO history for the release - valuesOverrideResponse, err := impl.manifestCreationService.GetValuesOverrideForTrigger(newCtx, overrideRequest, envDeploymentConfig, triggeredAt) - // auditDeploymentTriggerHistory is performed irrespective of GetValuesOverrideForTrigger error - for auditing purposes - historyErr := impl.auditDeploymentTriggerHistory(overrideRequest.WfrId, valuesOverrideResponse, newCtx, triggeredAt, deployedBy) - if historyErr != nil { - impl.logger.Errorw("error in auditing deployment trigger history", "cdWfrId", overrideRequest.WfrId, "err", err) - return releaseNo, manifestPushTemplate, err - } - // handling GetValuesOverrideForTrigger error - if err != nil { - impl.logger.Errorw("error in fetching values for trigger", "err", err) - return releaseNo, manifestPushTemplate, err - } - // asynchronous mode of Helm/ArgoCd installation starts - return impl.workflowEventPublishService.TriggerAsyncRelease(userDeploymentRequestId, overrideRequest, valuesOverrideResponse, newCtx, deployedBy) -} - -func (impl *HandlerServiceImpl) handleCDTriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.handleCDTriggerRelease") - defer span.End() - // Handling for auto trigger - if overrideRequest.UserId == 0 { - overrideRequest.UserId = deployedBy - } - tx, err := impl.transactionUtilImpl.StartTx() - if err != nil { - impl.logger.Errorw("error in starting transaction to update userDeploymentRequest", "error", err) - return releaseNo, manifestPushTemplate, err - } - defer impl.transactionUtilImpl.RollbackTx(tx) - newDeploymentRequest := adapter.NewUserDeploymentRequest(overrideRequest, triggeredAt, overrideRequest.UserId) - // creating new user deployment request - userDeploymentRequestId, err := impl.userDeploymentRequestService.SaveNewDeployment(newCtx, tx, newDeploymentRequest) - if err != nil { - impl.logger.Errorw("error in saving new userDeploymentRequest", "overrideRequest", overrideRequest, "err", err) - return releaseNo, manifestPushTemplate, err - } - timeline := impl.pipelineStatusTimelineService.NewDevtronAppPipelineStatusTimelineDbObject(overrideRequest.WfrId, timelineStatus.TIMELINE_STATUS_DEPLOYMENT_REQUEST_VALIDATED, timelineStatus.TIMELINE_DESCRIPTION_DEPLOYMENT_REQUEST_VALIDATED, deployedBy) - // creating cd pipeline status timeline for deployment trigger request validated - _, err = impl.pipelineStatusTimelineService.SaveTimelineIfNotAlreadyPresent(timeline, tx) - err = impl.transactionUtilImpl.CommitTx(tx) - if err != nil { - impl.logger.Errorw("error in committing transaction to update userDeploymentRequest", "error", err) - return userDeploymentRequestId, manifestPushTemplate, err - } - isAsyncMode, err := impl.isDevtronAsyncInstallModeEnabled(overrideRequest) - if err != nil { - impl.logger.Errorw("error in checking async mode for devtron app", "err", err, "deploymentType", overrideRequest.DeploymentType, - "forceSyncDeployment", overrideRequest.ForceSyncDeployment, "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId) - return userDeploymentRequestId, manifestPushTemplate, err - } - if envDeploymentConfig.IsEmpty() { - deploymentConfig, dbErr := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(overrideRequest.AppId, overrideRequest.EnvId) - if dbErr != nil { - impl.logger.Errorw("error in getting deployment config by appId and envId", "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId, "err", dbErr) - return releaseNo, manifestPushTemplate, dbErr - } - envDeploymentConfig = deploymentConfig - } - if isAsyncMode { - return impl.triggerAsyncRelease(newCtx, overrideRequest, envDeploymentConfig, userDeploymentRequestId, triggeredAt, deployedBy) - } - // synchronous mode of installation starts - return impl.TriggerRelease(newCtx, overrideRequest, envDeploymentConfig, triggeredAt, deployedBy) -} - -func (impl *HandlerServiceImpl) auditDeploymentTriggerHistory(cdWfrId int, valuesOverrideResponse *app.ValuesOverrideResponse, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (err error) { - if valuesOverrideResponse.Pipeline == nil || valuesOverrideResponse.EnvOverride == nil { - impl.logger.Warnw("unable to save histories for deployment trigger, invalid valuesOverrideResponse received", "cdWfrId", cdWfrId) - return nil - } - err1 := impl.deployedConfigurationHistoryService.CreateHistoriesForDeploymentTrigger(ctx, valuesOverrideResponse.Pipeline, valuesOverrideResponse.PipelineStrategy, valuesOverrideResponse.EnvOverride, triggeredAt, triggeredBy) - if err1 != nil { - impl.logger.Errorw("error in saving histories for deployment trigger", "err", err1, "pipelineId", valuesOverrideResponse.Pipeline.Id, "cdWfrId", cdWfrId) - return nil - } - return nil -} - -// TriggerRelease will trigger Install/Upgrade request for Devtron App releases synchronously -func (impl *HandlerServiceImpl) TriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, - envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.TriggerRelease") - defer span.End() - triggerEvent, skipRequest, err := impl.buildTriggerEventForOverrideRequest(overrideRequest, triggeredAt) - if err != nil { - return releaseNo, manifestPushTemplate, err - } - impl.logger.Debugw("processing TriggerRelease", "wfrId", overrideRequest.WfrId, "triggerEvent", triggerEvent) - // request has already been served, skipping - if skipRequest { - impl.logger.Infow("request already served, skipping", "wfrId", overrideRequest.WfrId) - return releaseNo, manifestPushTemplate, nil - } - // build merged values and save PCO history for the release - valuesOverrideResponse, builtChartPath, err := impl.manifestCreationService.BuildManifestForTrigger(newCtx, overrideRequest, envDeploymentConfig, triggeredAt) - - // auditDeploymentTriggerHistory is performed irrespective of BuildManifestForTrigger error - for auditing purposes - historyErr := impl.auditDeploymentTriggerHistory(overrideRequest.WfrId, valuesOverrideResponse, newCtx, triggeredAt, triggeredBy) - if historyErr != nil { - impl.logger.Errorw("error in auditing deployment trigger history", "cdWfrId", overrideRequest.WfrId, "err", err) - return releaseNo, manifestPushTemplate, err - } - if err != nil { - impl.logger.Errorw("error in building merged manifest for trigger", "err", err) - impl.manifestGenerationFailedTimelineHandling(triggerEvent, overrideRequest, err) - return releaseNo, manifestPushTemplate, err - } - helmManifest, err := impl.getHelmManifestForTriggerRelease(ctx, triggerEvent, overrideRequest, valuesOverrideResponse, builtChartPath) - if err != nil { - impl.logger.Errorw("error, getHelmManifestForTriggerRelease", "err", err) - return releaseNo, manifestPushTemplate, err - } - impl.logger.Debugw("triggering pipeline for release", "wfrId", overrideRequest.WfrId, "builtChartPath", builtChartPath) - releaseNo, err = impl.triggerPipeline(overrideRequest, valuesOverrideResponse, builtChartPath, triggerEvent, newCtx) - if err != nil { - return 0, manifestPushTemplate, err - } - - err = impl.triggerReleaseSuccessHandling(triggerEvent, overrideRequest, valuesOverrideResponse, helmManifest) - if err != nil { - impl.logger.Errorw("error, triggerReleaseSuccessHandling", "triggerEvent", triggerEvent, "err", err) - return releaseNo, manifestPushTemplate, err - } - // creating cd pipeline status timeline for deployment triggered - for successfully triggered requests - timeline := impl.pipelineStatusTimelineService.NewDevtronAppPipelineStatusTimelineDbObject(overrideRequest.WfrId, timelineStatus.TIMELINE_STATUS_DEPLOYMENT_TRIGGERED, timelineStatus.TIMELINE_DESCRIPTION_DEPLOYMENT_COMPLETED, overrideRequest.UserId) - _, dbErr := impl.pipelineStatusTimelineService.SaveTimelineIfNotAlreadyPresent(timeline, nil) - if dbErr != nil { - impl.logger.Errorw("error in creating timeline status for deployment completed", "err", dbErr, "timeline", timeline) - } - impl.logger.Debugw("triggered pipeline for release successfully", "wfrId", overrideRequest.WfrId, "builtChartPath", builtChartPath) - return releaseNo, valuesOverrideResponse.ManifestPushTemplate, nil -} - -func (impl *HandlerServiceImpl) performGitOps(ctx context.Context, - overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, - builtChartPath string, triggerEvent bean.TriggerEvent) error { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.performGitOps") - defer span.End() - // update workflow runner status, used in app workflow view - err := impl.cdWorkflowCommonService.UpdateNonTerminalStatusInRunner(newCtx, overrideRequest.WfrId, overrideRequest.UserId, cdWorkflow.WorkflowInProgress) - if err != nil { - impl.logger.Errorw("error in updating the workflow runner status", "err", err) - return err - } - manifestPushTemplate, err := impl.buildManifestPushTemplate(overrideRequest, valuesOverrideResponse, builtChartPath) - if err != nil { - impl.logger.Errorw("error in building manifest push template", "err", err) - return err - } - manifestPushService := impl.getManifestPushService(triggerEvent.ManifestStorageType) - manifestPushResponse := manifestPushService.PushChart(newCtx, manifestPushTemplate) - if manifestPushResponse.Error != nil { - impl.logger.Errorw("error in pushing manifest to git/helm", "err", manifestPushResponse.Error, "git_repo_url", manifestPushTemplate.RepoUrl) - return manifestPushResponse.Error - } - if manifestPushResponse.IsNewGitRepoConfigured() { - // Update GitOps repo url after repo new repo created - valuesOverrideResponse.DeploymentConfig.SetRepoURL(manifestPushResponse.NewGitRepoUrl) - } - valuesOverrideResponse.ManifestPushTemplate = manifestPushTemplate - return nil -} - -func (impl *HandlerServiceImpl) buildTriggerEventForOverrideRequest(overrideRequest *bean3.ValuesOverrideRequest, triggeredAt time.Time) (triggerEvent bean.TriggerEvent, skipRequest bool, err error) { - triggerEvent = helper.NewTriggerEvent(overrideRequest.DeploymentAppType, triggeredAt, overrideRequest.UserId) - request := statusBean.NewTimelineGetRequest(). - WithCdWfrId(overrideRequest.WfrId). - ExcludingStatuses(timelineStatus.TIMELINE_STATUS_UNABLE_TO_FETCH_STATUS, - timelineStatus.TIMELINE_STATUS_KUBECTL_APPLY_STARTED, - timelineStatus.TIMELINE_STATUS_KUBECTL_APPLY_SYNCED) - timelineStatuses, err := impl.pipelineStatusTimelineService.GetTimelineStatusesFor(request) - if err != nil { - impl.logger.Errorw("error in getting last timeline status by cdWfrId", "cdWfrId", overrideRequest.WfrId, "err", err) - return triggerEvent, skipRequest, err - } else if !slices.Contains(timelineStatuses, timelineStatus.TIMELINE_STATUS_DEPLOYMENT_REQUEST_VALIDATED) { - impl.logger.Errorw("pre-condition missing: timeline for deployment request validation", "cdWfrId", overrideRequest.WfrId, "timelineStatuses", timelineStatuses) - return triggerEvent, skipRequest, fmt.Errorf("pre-condition missing: timeline for deployment request validation") - } else if timelineStatus.ContainsTerminalTimelineStatus(timelineStatuses) { - impl.logger.Info("deployment is already terminated", "cdWfrId", overrideRequest.WfrId, "timelineStatuses", timelineStatuses) - skipRequest = true - return triggerEvent, skipRequest, nil - } else if slices.Contains(timelineStatuses, timelineStatus.TIMELINE_STATUS_DEPLOYMENT_TRIGGERED) { - impl.logger.Info("deployment has been performed. skipping", "cdWfrId", overrideRequest.WfrId, "timelineStatuses", timelineStatuses) - skipRequest = true - return triggerEvent, skipRequest, nil - } - if slices.Contains(timelineStatuses, timelineStatus.TIMELINE_STATUS_GIT_COMMIT) || - slices.Contains(timelineStatuses, timelineStatus.TIMELINE_STATUS_ARGOCD_SYNC_INITIATED) { - // git commit has already been performed - triggerEvent.PerformChartPush = false - } - if slices.Contains(timelineStatuses, timelineStatus.TIMELINE_STATUS_ARGOCD_SYNC_COMPLETED) { - // ArgoCd sync has already been performed - triggerEvent.DeployArgoCdApp = false - } - return triggerEvent, skipRequest, nil -} - -func (impl *HandlerServiceImpl) triggerPipeline(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, triggerEvent bean.TriggerEvent, ctx context.Context) (releaseNo int, err error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.triggerPipeline") - defer span.End() - if triggerEvent.PerformChartPush { - impl.logger.Debugw("performing chart push operation in triggerPipeline", "cdWfrId", overrideRequest.WfrId) - err = impl.performGitOps(newCtx, overrideRequest, valuesOverrideResponse, builtChartPath, triggerEvent) - if err != nil { - impl.logger.Errorw("error in performing GitOps", "cdWfrId", overrideRequest.WfrId, "err", err) - return releaseNo, err - } - impl.logger.Debugw("chart push operation completed successfully", "cdWfrId", overrideRequest.WfrId) - } - - if triggerEvent.PerformDeploymentOnCluster { - err = impl.deployApp(newCtx, overrideRequest, valuesOverrideResponse, triggerEvent) - if err != nil { - impl.logger.Errorw("error in deploying app", "err", err) - return releaseNo, err - } - } - - go impl.writeCDTriggerEvent(overrideRequest, valuesOverrideResponse.Artifact, valuesOverrideResponse.PipelineOverride.PipelineReleaseCounter, valuesOverrideResponse.PipelineOverride.Id, overrideRequest.WfrId) - - _ = impl.markImageScanDeployed(newCtx, overrideRequest.AppId, overrideRequest.EnvId, overrideRequest.ClusterId, - valuesOverrideResponse.Artifact.ImageDigest, valuesOverrideResponse.Artifact.ScanEnabled, valuesOverrideResponse.Artifact.Image) - - middleware.CdTriggerCounter.WithLabelValues(overrideRequest.AppName, overrideRequest.EnvName).Inc() - - // Update previous deployment runner status (in transaction): Failed - dbErr := impl.cdWorkflowCommonService.SupersedePreviousDeployments(newCtx, overrideRequest.WfrId, overrideRequest.PipelineId, triggerEvent.TriggeredAt, overrideRequest.UserId) - if dbErr != nil { - impl.logger.Errorw("error while update previous cd workflow runners", "err", dbErr, "currentRunnerId", overrideRequest.WfrId, "pipelineId", overrideRequest.PipelineId) - return releaseNo, dbErr - } - return valuesOverrideResponse.PipelineOverride.PipelineReleaseCounter, nil -} - -func (impl *HandlerServiceImpl) buildManifestPushTemplate(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string) (*bean4.ManifestPushTemplate, error) { - - manifestPushTemplate := &bean4.ManifestPushTemplate{ - WorkflowRunnerId: overrideRequest.WfrId, - AppId: overrideRequest.AppId, - ChartRefId: valuesOverrideResponse.EnvOverride.Chart.ChartRefId, - EnvironmentId: valuesOverrideResponse.EnvOverride.Environment.Id, - EnvironmentName: valuesOverrideResponse.EnvOverride.Environment.Namespace, - UserId: overrideRequest.UserId, - PipelineOverrideId: valuesOverrideResponse.PipelineOverride.Id, - AppName: overrideRequest.AppName, - TargetEnvironmentId: valuesOverrideResponse.EnvOverride.TargetEnvironment, - BuiltChartPath: builtChartPath, - MergedValues: valuesOverrideResponse.MergedValues, - } - - manifestPushConfig, err := impl.manifestPushConfigRepository.GetManifestPushConfigByAppIdAndEnvId(overrideRequest.AppId, overrideRequest.EnvId) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in fetching manifest push config from db", "err", err) - return manifestPushTemplate, err - } - - if manifestPushConfig != nil && manifestPushConfig.Id != 0 { - if manifestPushConfig.StorageType == bean2.ManifestStorageGit { - // need to implement for git repo push - // currently manifest push config doesn't have git push config. GitOps config is derived from charts, chart_env_config_override and chart_ref table - } else { - err2 := impl.buildManifestPushTemplateForNonGitStorageType(overrideRequest, valuesOverrideResponse, builtChartPath, err, manifestPushConfig, manifestPushTemplate) - if err2 != nil { - return manifestPushTemplate, err2 - } - } - } else { - manifestPushTemplate.ChartReferenceTemplate = valuesOverrideResponse.EnvOverride.Chart.ReferenceTemplate - manifestPushTemplate.ChartName = valuesOverrideResponse.EnvOverride.Chart.ChartName - manifestPushTemplate.ChartVersion = valuesOverrideResponse.EnvOverride.Chart.ChartVersion - manifestPushTemplate.ChartLocation = valuesOverrideResponse.DeploymentConfig.GetChartLocation() - manifestPushTemplate.RepoUrl = valuesOverrideResponse.DeploymentConfig.GetRepoURL() - manifestPushTemplate.TargetRevision = valuesOverrideResponse.DeploymentConfig.GetTargetRevision() - manifestPushTemplate.ValuesFilePath = valuesOverrideResponse.DeploymentConfig.GetValuesFilePath() - manifestPushTemplate.ReleaseMode = valuesOverrideResponse.DeploymentConfig.ReleaseMode - manifestPushTemplate.IsCustomGitRepository = common.IsCustomGitOpsRepo(valuesOverrideResponse.DeploymentConfig.ConfigType) - manifestPushTemplate.IsArgoSyncSupported = valuesOverrideResponse.DeploymentConfig.IsArgoAppSyncAndRefreshSupported() - } - return manifestPushTemplate, nil -} - -func (impl *HandlerServiceImpl) deployApp(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, triggerEvent bean.TriggerEvent) error { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.deployApp") - defer span.End() - var referenceChartByte []byte - var err error - - if util.IsAcdApp(overrideRequest.DeploymentAppType) && triggerEvent.DeployArgoCdApp { - err = impl.deployArgoCdApp(newCtx, overrideRequest, valuesOverrideResponse) - if err != nil { - impl.logger.Errorw("error in deploying app on ArgoCd", "err", err) - return err - } - } else if util.IsHelmApp(overrideRequest.DeploymentAppType) { - _, referenceChartByte, err = impl.createHelmAppForCdPipeline(newCtx, overrideRequest, valuesOverrideResponse) - if err != nil { - impl.logger.Errorw("error in creating or updating helm application for cd pipeline", "err", err) - return err - } - } - impl.postDeployHook(overrideRequest, valuesOverrideResponse, referenceChartByte, err) - return nil -} - -func (impl *HandlerServiceImpl) createHelmAppForCdPipeline(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse) (bool, []byte, error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.createHelmAppForCdPipeline") - defer span.End() - pipelineModel := valuesOverrideResponse.Pipeline - envOverride := valuesOverrideResponse.EnvOverride - mergeAndSave := valuesOverrideResponse.MergedValues - chartMetaData, helmRevisionHistory, releaseIdentifier, err := impl.getHelmHistoryLimitAndChartMetadataForHelmAppCreation(ctx, valuesOverrideResponse) - if err != nil { - impl.logger.Errorw("error, getHelmHistoryLimitAndChartMetadataForHelmAppCreation", "valuesOverrideResponse", valuesOverrideResponse, "err", err) - return false, nil, err - } - referenceTemplate := envOverride.Chart.ReferenceTemplate - referenceTemplatePath := path.Join(bean5.RefChartDirPath, referenceTemplate) - var referenceChartByte []byte - if util.IsHelmApp(valuesOverrideResponse.DeploymentConfig.DeploymentAppType) { - sanitizedK8sVersion, err := impl.getSanitizedK8sVersion(referenceTemplate) - if err != nil { - return false, nil, err - } - referenceChartByte, err = impl.getReferenceChartByteForHelmTypeApp(envOverride, chartMetaData, referenceTemplatePath, overrideRequest, valuesOverrideResponse) - if err != nil { - impl.logger.Errorw("error, getReferenceChartByteForHelmTypeApp", "envOverride", envOverride, "err", err) - return false, nil, err - } - if pipelineModel.DeploymentAppCreated { - req := &gRPC.UpgradeReleaseRequest{ - ReleaseIdentifier: releaseIdentifier, - ValuesYaml: mergeAndSave, - HistoryMax: helmRevisionHistory, - ChartContent: &gRPC.ChartContent{Content: referenceChartByte}, - } - if len(sanitizedK8sVersion) > 0 { - req.K8SVersion = sanitizedK8sVersion - } - if impl.isDevtronAsyncHelmInstallModeEnabled(overrideRequest.ForceSyncDeployment) { - req.RunInCtx = true - } - // For cases where helm release was not found, kubelink will install the same configuration - updateApplicationResponse, err := impl.helmAppClient.UpdateApplication(newCtx, req) - if err != nil { - impl.logger.Errorw("error in updating helm application for cd pipelineModel", "err", err) - if util.IsErrorContextCancelled(err) { - return false, nil, cdWorkflow.ErrorDeploymentSuperseded - } else if util.IsErrorContextDeadlineExceeded(err) { - return false, nil, context.DeadlineExceeded - } - apiError := clientErrors.ConvertToApiError(err) - if apiError != nil { - return false, nil, apiError - } - return false, nil, err - } else { - impl.logger.Debugw("updated helm application", "response", updateApplicationResponse, "isSuccess", updateApplicationResponse.Success) - } - - } else { - - helmResponse, err := impl.helmInstallReleaseWithCustomChart(newCtx, releaseIdentifier, referenceChartByte, - mergeAndSave, sanitizedK8sVersion, overrideRequest.ForceSyncDeployment) - - // For connection related errors, no need to update the db - if err != nil && strings.Contains(err.Error(), "connection error") { - impl.logger.Errorw("error in helm install custom chart", "err", err) - return false, nil, err - } - - // IMP: update cd pipelineModel to mark deployment app created, even if helm install fails - // If the helm install fails, it still creates the app in failed state, so trying to - // re-create the app results in error from helm that cannot re-use name which is still in use - _, pgErr := impl.updatePipeline(pipelineModel, overrideRequest.UserId) - - if err != nil { - impl.logger.Errorw("error in helm install custom chart", "err", err) - if pgErr != nil { - impl.logger.Errorw("failed to update deployment app created flag in pipelineModel table", "err", err) - } - if util.IsErrorContextCancelled(err) { - return false, nil, cdWorkflow.ErrorDeploymentSuperseded - } else if util.IsErrorContextDeadlineExceeded(err) { - return false, nil, context.DeadlineExceeded - } - apiError := clientErrors.ConvertToApiError(err) - if apiError != nil { - return false, nil, apiError - } - return false, nil, err - } - - if pgErr != nil { - impl.logger.Errorw("failed to update deployment app created flag in pipelineModel table", "err", err) - return false, nil, err - } - - impl.logger.Debugw("received helm release response", "helmResponse", helmResponse, "isSuccess", helmResponse.Success) - } - - //update workflow runner status, used in app workflow view - err = impl.cdWorkflowCommonService.UpdateNonTerminalStatusInRunner(newCtx, overrideRequest.WfrId, overrideRequest.UserId, cdWorkflow.WorkflowInProgress) - if err != nil { - impl.logger.Errorw("error in updating the workflow runner status, createHelmAppForCdPipeline", "err", err) - return false, nil, err - } - } - return true, referenceChartByte, nil -} - -func (impl *HandlerServiceImpl) getHelmHistoryLimitAndChartMetadataForHelmAppCreation(ctx context.Context, - valuesOverrideResponse *app.ValuesOverrideResponse) (*chart.Metadata, int32, *gRPC.ReleaseIdentifier, error) { - pipelineModel := valuesOverrideResponse.Pipeline - envOverride := valuesOverrideResponse.EnvOverride - - var chartMetaData *chart.Metadata - releaseName := pipelineModel.DeploymentAppName - //getting cluster by id - cluster, err := impl.clusterRepository.FindById(envOverride.Environment.ClusterId) - if err != nil { - impl.logger.Errorw("error in getting cluster by id", "clusterId", envOverride.Environment.ClusterId, "err", err) - return nil, 0, nil, err - } else if cluster == nil { - impl.logger.Errorw("error in getting cluster by id, found nil object", "clusterId", envOverride.Environment.ClusterId) - return nil, 0, nil, err - } - - clusterConfig := impl.getClusterGRPCConfig(*cluster) - - releaseIdentifier := &gRPC.ReleaseIdentifier{ - ReleaseName: releaseName, - ReleaseNamespace: envOverride.Namespace, - ClusterConfig: clusterConfig, - } - - var helmRevisionHistory int32 - if valuesOverrideResponse.DeploymentConfig.ReleaseMode == util.PIPELINE_RELEASE_MODE_LINK { - detail, err := impl.helmAppClient.GetReleaseDetails(ctx, releaseIdentifier) - if err != nil { - impl.logger.Errorw("error in fetching release details", "clusterId", clusterConfig.ClusterId, "namespace", envOverride.Namespace, "releaseName", releaseName, "err", err) - return nil, 0, nil, err - } - chartMetaData = &chart.Metadata{ - Name: detail.ChartName, - Version: detail.ChartVersion, - } - //not modifying revision history in case of linked release - helmRevisionHistory = impl.helmAppService.GetRevisionHistoryMaxValue(bean6.SOURCE_LINKED_HELM_APP) - } else { - chartMetaData = &chart.Metadata{ - Name: pipelineModel.App.AppName, - Version: envOverride.Chart.ChartVersion, - } - helmRevisionHistory = impl.helmAppService.GetRevisionHistoryMaxValue(bean6.SOURCE_DEVTRON_APP) - } - - return chartMetaData, helmRevisionHistory, releaseIdentifier, nil -} - -func (impl *HandlerServiceImpl) deployArgoCdApp(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, - valuesOverrideResponse *app.ValuesOverrideResponse) error { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.deployArgoCdApp") - defer span.End() - name, err := impl.createArgoApplicationIfRequired(newCtx, valuesOverrideResponse.EnvOverride, valuesOverrideResponse.Pipeline, valuesOverrideResponse.DeploymentConfig, overrideRequest.UserId) - if err != nil { - impl.logger.Errorw("acd application create error on cd trigger", "err", err, "req", overrideRequest) - return err - } - impl.logger.Debugw("ArgoCd application created", "name", name) - updateAppInArgoCd, err := impl.updateArgoPipeline(newCtx, valuesOverrideResponse.Pipeline, valuesOverrideResponse.EnvOverride, valuesOverrideResponse.DeploymentConfig) - if err != nil { - impl.logger.Errorw("error in updating argocd app ", "err", err) - return err - } - if valuesOverrideResponse.DeploymentConfig.IsArgoAppSyncAndRefreshSupported() { - syncTime := time.Now() - targetRevision := valuesOverrideResponse.DeploymentConfig.GetTargetRevision() - err = impl.argoClientWrapperService.SyncArgoCDApplicationIfNeededAndRefresh(newCtx, valuesOverrideResponse.Pipeline.DeploymentAppName, targetRevision) - if err != nil { - impl.logger.Errorw("error in getting argo application with normal refresh", "argoAppName", valuesOverrideResponse.Pipeline.DeploymentAppName) - return fmt.Errorf("%s. err: %s", bean.ARGOCD_SYNC_ERROR, util.GetClientErrorDetailedMessage(err)) - } - if impl.ACDConfig.IsManualSyncEnabled() { - timeline := &pipelineConfig.PipelineStatusTimeline{ - CdWorkflowRunnerId: overrideRequest.WfrId, - StatusTime: syncTime, - Status: timelineStatus.TIMELINE_STATUS_ARGOCD_SYNC_COMPLETED, - StatusDetail: timelineStatus.TIMELINE_DESCRIPTION_ARGOCD_SYNC_COMPLETED, - } - timeline.CreateAuditLog(overrideRequest.UserId) - _, err = impl.pipelineStatusTimelineService.SaveTimelineIfNotAlreadyPresent(timeline, nil) - if err != nil { - impl.logger.Errorw("error in saving pipeline status timeline", "err", err) - } - } - } - if updateAppInArgoCd { - impl.logger.Debug("argo-cd successfully updated") - } else { - impl.logger.Debug("argo-cd failed to update, ignoring it") - } - return nil -} - -// update repoUrl, revision and argo app sync mode (auto/manual) if needed -func (impl *HandlerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envOverride *bean10.EnvConfigOverride, deploymentConfig *bean9.DeploymentConfig) (bool, error) { - if !deploymentConfig.IsArgoAppPatchSupported() { - impl.logger.Infow("argo app patch not supported", "pipelineId", pipeline.Id, "pipelineName", pipeline.Name) - return false, nil - } - if ctx == nil { - impl.logger.Errorw("err in syncing ACD, ctx is NULL", "pipelineId", pipeline.Id, "pipelineName", pipeline.Name) - return false, nil - } - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.updateArgoPipeline") - defer span.End() - argoAppName := pipeline.DeploymentAppName - impl.logger.Infow("received payload, updateArgoPipeline", "appId", pipeline.AppId, "pipelineName", pipeline.Name, "envId", envOverride.TargetEnvironment, "argoAppName", argoAppName) - argoApplication, err := impl.argoClientWrapperService.GetArgoAppByName(newCtx, argoAppName) - if err != nil { - impl.logger.Errorw("unable to get ArgoCd app", "app", argoAppName, "pipeline", pipeline.Name, "err", err) - return false, err - } - // if status, ok:=status.FromError(err);ok{ - appStatus, _ := status2.FromError(err) - if appStatus.Code() == codes.OK { - impl.logger.Debugw("argo app exists", "app", argoAppName, "pipeline", pipeline.Name) - if impl.argoClientWrapperService.IsArgoAppPatchRequired(argoApplication.Spec.Source, deploymentConfig.GetRepoURL(), deploymentConfig.GetTargetRevision(), deploymentConfig.GetChartLocation()) { - patchRequestDto := &bean7.ArgoCdAppPatchReqDto{ - ArgoAppName: argoAppName, - ChartLocation: deploymentConfig.GetChartLocation(), - GitRepoUrl: deploymentConfig.GetRepoURL(), - TargetRevision: deploymentConfig.GetTargetRevision(), - PatchType: bean7.PatchTypeMerge, - } - url, err := impl.gitOperationService.GetRepoUrlWithUserName(deploymentConfig.GetRepoURL()) - if err != nil { - return false, err - } - patchRequestDto.GitRepoUrl = url - err = impl.argoClientWrapperService.PatchArgoCdApp(newCtx, patchRequestDto) - if err != nil { - impl.logger.Errorw("error in patching argo pipeline", "err", err, "req", patchRequestDto) - return false, err - } - if deploymentConfig.GetRepoURL() != argoApplication.Spec.Source.RepoURL { - impl.logger.Infow("patching argo application's repo url", "argoAppName", argoAppName) - } - if deploymentConfig.GetTargetRevision() != argoApplication.Spec.Source.TargetRevision { - impl.logger.Infow("patching argo application's revision", "argoAppName", argoAppName) - } - impl.logger.Debugw("pipeline update req", "res", patchRequestDto) - } else { - impl.logger.Debug("pipeline no need to update ") - } - err := impl.argoClientWrapperService.UpdateArgoCDSyncModeIfNeeded(newCtx, argoApplication) - if err != nil { - impl.logger.Errorw("error in updating argocd sync mode", "err", err) - return false, err - } - return true, nil - } else if appStatus.Code() == codes.NotFound { - impl.logger.Errorw("argo app not found", "app", argoAppName, "pipeline", pipeline.Name) - return false, nil - } else { - impl.logger.Errorw("err in checking application on argoCD", "err", err, "pipeline", pipeline.Name) - return false, err - } -} - -func (impl *HandlerServiceImpl) createArgoApplicationIfRequired(ctx context.Context, envConfigOverride *bean10.EnvConfigOverride, - pipeline *pipelineConfig.Pipeline, deploymentConfig *bean9.DeploymentConfig, userId int32) (string, error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.createArgoApplicationIfRequired") - defer span.End() - envModel, err := impl.envRepository.FindById(envConfigOverride.TargetEnvironment) - if err != nil { - return "", err - } - argoAppName := pipeline.DeploymentAppName - if !deploymentConfig.IsArgoAppCreationRequired(pipeline.DeploymentAppCreated) { - return argoAppName, nil - } else { - impl.logger.Debugw("new pipeline found", "pipeline", pipeline) - // create - appNamespace := envConfigOverride.Namespace - if appNamespace == "" { - appNamespace = "default" - } - namespace := argocdServer.DevtronInstalationNs - - appRequest := &argocdServer.AppTemplate{ - ApplicationName: argoAppName, - Namespace: namespace, - TargetNamespace: appNamespace, - TargetServer: envModel.Cluster.ServerUrl, - Project: "default", - ValuesFile: helper.GetValuesFileForEnv(envModel.Id), - RepoPath: deploymentConfig.GetChartLocation(), - RepoUrl: deploymentConfig.GetRepoURL(), - AutoSyncEnabled: impl.ACDConfig.ArgoCDAutoSyncEnabled, - } - appRequest.RepoUrl, err = impl.gitOperationService.GetRepoUrlWithUserName(appRequest.RepoUrl) - if err != nil { - return "", err - } - createdArgoAppName, err := impl.argoK8sClient.CreateAcdApp(newCtx, appRequest, argocdServer.ARGOCD_APPLICATION_TEMPLATE) - if err != nil { - return "", err - } - // update cd pipeline to mark deployment app created - _, err = impl.updatePipeline(pipeline, userId) - if err != nil { - impl.logger.Errorw("error in update cd pipeline for deployment app created or not", "err", err) - return "", err - } - return createdArgoAppName, nil - } -} - -func (impl *HandlerServiceImpl) updatePipeline(pipeline *pipelineConfig.Pipeline, userId int32) (bool, error) { - err := impl.pipelineRepository.SetDeploymentAppCreatedInPipeline(true, pipeline.Id, userId) - if err != nil { - impl.logger.Errorw("error on updating cd pipeline for setting deployment app created", "err", err) - return false, err - } - return true, nil -} - -// helmInstallReleaseWithCustomChart performs helm install with custom chart -func (impl *HandlerServiceImpl) helmInstallReleaseWithCustomChart(ctx context.Context, releaseIdentifier *gRPC.ReleaseIdentifier, - referenceChartByte []byte, valuesYaml, k8sServerVersion string, forceSync bool) (*gRPC.HelmInstallCustomResponse, error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.helmInstallReleaseWithCustomChart") - defer span.End() - helmInstallRequest := gRPC.HelmInstallCustomRequest{ - ValuesYaml: valuesYaml, - ChartContent: &gRPC.ChartContent{Content: referenceChartByte}, - ReleaseIdentifier: releaseIdentifier, - } - if len(k8sServerVersion) > 0 { - helmInstallRequest.K8SVersion = k8sServerVersion - } - if impl.isDevtronAsyncHelmInstallModeEnabled(forceSync) { - helmInstallRequest.RunInCtx = true - } - // Request exec - return impl.helmAppClient.InstallReleaseWithCustomChart(newCtx, &helmInstallRequest) -} - func (impl *HandlerServiceImpl) writeCDTriggerEvent(overrideRequest *bean3.ValuesOverrideRequest, artifact *repository3.CiArtifact, releaseId, pipelineOverrideId, wfrId int) { event, err := impl.eventFactory.Build(util2.Trigger, &overrideRequest.PipelineId, overrideRequest.AppId, &overrideRequest.EnvId, util2.CD) @@ -1466,184 +343,3 @@ func (impl *HandlerServiceImpl) writeCDTriggerEvent(overrideRequest *bean3.Value impl.logger.Errorw("error in writing cd trigger event", "err", err) } } - -func (impl *HandlerServiceImpl) markImageScanDeployed(ctx context.Context, appId, envId, clusterId int, - imageDigest string, isScanEnabled bool, image string) error { - _, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.markImageScanDeployed") - defer span.End() - // TODO KB: send NATS event for self consumption - impl.logger.Debugw("mark image scan deployed for devtron app, from cd auto or manual trigger", "imageDigest", imageDigest) - executionHistory, err := impl.imageScanHistoryReadService.FindByImageAndDigest(imageDigest, image) - if err != nil && !errors.Is(err, pg.ErrNoRows) { - impl.logger.Errorw("error in fetching execution history", "err", err) - return err - } - if errors.Is(err, pg.ErrNoRows) || executionHistory == nil || executionHistory.Id == 0 { - if isScanEnabled { - // There should ImageScanHistory for ScanEnabled artifacts - impl.logger.Errorw("no execution history found for digest", "digest", imageDigest) - return fmt.Errorf("no execution history found for digest - %s", imageDigest) - } else { - // For ScanDisabled artifacts it should be an expected condition - impl.logger.Infow("no execution history found for digest", "digest", imageDigest) - return nil - } - } - impl.logger.Debugw("saving image_scan_deploy_info for cd auto or manual trigger", "executionHistory", executionHistory) - var ids []int - ids = append(ids, executionHistory.Id) - - ot, err := impl.imageScanDeployInfoReadService.FetchByAppIdAndEnvId(appId, envId, []string{repository6.ScanObjectType_APP}) - - if err == pg.ErrNoRows && !isScanEnabled { - // ignoring if no rows are found and scan is disabled - return nil - } - - if err != nil && err != pg.ErrNoRows { - return err - } else if err == pg.ErrNoRows && isScanEnabled { - imageScanDeployInfo := &repository6.ImageScanDeployInfo{ - ImageScanExecutionHistoryId: ids, - ScanObjectMetaId: appId, - ObjectType: repository6.ScanObjectType_APP, - EnvId: envId, - ClusterId: clusterId, - AuditLog: sql.AuditLog{ - CreatedOn: time.Now(), - CreatedBy: 1, - UpdatedOn: time.Now(), - UpdatedBy: 1, - }, - } - impl.logger.Debugw("mark image scan deployed for normal app, from cd auto or manual trigger", "imageScanDeployInfo", imageScanDeployInfo) - err = impl.imageScanDeployInfoService.Save(imageScanDeployInfo) - if err != nil { - impl.logger.Errorw("error in creating deploy info", "err", err) - } - } else { - // Updating Execution history for Latest Deployment to fetch out security Vulnerabilities for latest deployed info - if isScanEnabled { - ot.ImageScanExecutionHistoryId = ids - } else { - arr := []int{-1} - ot.ImageScanExecutionHistoryId = arr - } - err = impl.imageScanDeployInfoService.Update(ot) - if err != nil { - impl.logger.Errorw("error in updating deploy info for latest deployed image", "err", err) - } - } - return err -} - -func (impl *HandlerServiceImpl) isDevtronAsyncHelmInstallModeEnabled(forceSync bool) bool { - return impl.globalEnvVariables.EnableAsyncHelmInstallDevtronChart && !forceSync -} - -func (impl *HandlerServiceImpl) isDevtronAsyncInstallModeEnabled(overrideRequest *bean3.ValuesOverrideRequest) (bool, error) { - if util.IsHelmApp(overrideRequest.DeploymentAppType) { - return impl.isDevtronAsyncHelmInstallModeEnabled(overrideRequest.ForceSyncDeployment), nil - } else if util.IsAcdApp(overrideRequest.DeploymentAppType) { - return impl.isDevtronAsyncArgoCdInstallModeEnabledForApp(overrideRequest.AppId, - overrideRequest.EnvId, overrideRequest.ForceSyncDeployment) - } - return false, nil -} - -func (impl *HandlerServiceImpl) deleteCorruptedPipelineStage(pipelineStage *repository.PipelineStage, triggeredBy int32) (error, bool) { - if pipelineStage != nil { - stageReq := &bean8.PipelineStageDto{ - Id: pipelineStage.Id, - Type: pipelineStage.Type, - } - err, deleted := impl.pipelineStageService.DeletePipelineStageIfReq(stageReq, triggeredBy) - if err != nil { - impl.logger.Errorw("error in deleting the corrupted pipeline stage", "err", err, "pipelineStageReq", stageReq) - return err, false - } - return nil, deleted - } - return nil, false -} - -func (impl *HandlerServiceImpl) handleCustomGitOpsRepoValidation(runner *pipelineConfig.CdWorkflowRunner, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredBy int32) error { - if !util.IsAcdApp(pipeline.DeploymentAppName) { - return nil - } - isGitOpsConfigured := false - gitOpsConfig, err := impl.gitOpsConfigReadService.GetGitOpsConfigActive() - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error while getting active GitOpsConfig", "err", err) - } - if gitOpsConfig != nil && gitOpsConfig.Id > 0 { - isGitOpsConfigured = true - } - if isGitOpsConfigured && gitOpsConfig.AllowCustomRepository { - //chart, err := impl.chartRepository.FindLatestChartForAppByAppId(pipeline.AppId) - //if err != nil { - // impl.logger.Errorw("error in fetching latest chart for app by appId", "err", err, "appId", pipeline.AppId) - // return err - //} - if gitOps.IsGitOpsRepoNotConfigured(envDeploymentConfig.GetRepoURL()) { - if err = impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, errors.New(cdWorkflow.GITOPS_REPO_NOT_CONFIGURED), triggeredBy); err != nil { - impl.logger.Errorw("error while updating current runner status to failed, TriggerDeployment", "wfrId", runner.Id, "err", err) - } - apiErr := &util.ApiError{ - HttpStatusCode: http.StatusConflict, - UserMessage: cdWorkflow.GITOPS_REPO_NOT_CONFIGURED, - InternalMessage: cdWorkflow.GITOPS_REPO_NOT_CONFIGURED, - } - return apiErr - } - } - return nil -} - -func (impl *HandlerServiceImpl) getSanitizedK8sVersion(referenceTemplate string) (string, error) { - var sanitizedK8sVersion string - //handle specific case for all cronjob charts from cronjob-chart_1-2-0 to cronjob-chart_1-5-0 where semverCompare - //comparison func has wrong api version mentioned, so for already installed charts via these charts that comparison - //is always false, handles the gh issue:- https://github.com/devtron-labs/devtron/issues/4860 - cronJobChartRegex := regexp.MustCompile(bean.CronJobChartRegexExpression) - if cronJobChartRegex.MatchString(referenceTemplate) { - k8sServerVersion, err := impl.K8sUtil.GetKubeVersion() - if err != nil { - impl.logger.Errorw("exception caught in getting k8sServerVersion", "err", err) - return "", err - } - sanitizedK8sVersion = k8s2.StripPrereleaseFromK8sVersion(k8sServerVersion.String()) - } - return sanitizedK8sVersion, nil -} - -func (impl *HandlerServiceImpl) getReferenceChartByteForHelmTypeApp(envOverride *bean10.EnvConfigOverride, - chartMetaData *chart.Metadata, referenceTemplatePath string, overrideRequest *bean3.ValuesOverrideRequest, - valuesOverrideResponse *app.ValuesOverrideResponse) ([]byte, error) { - referenceChartByte := envOverride.Chart.ReferenceChart - // here updating reference chart into database. - if len(envOverride.Chart.ReferenceChart) == 0 { - refChartByte, err := impl.chartTemplateService.GetByteArrayRefChart(chartMetaData, referenceTemplatePath) - if err != nil { - impl.logger.Errorw("ref chart commit error on cd trigger", "err", err, "req", overrideRequest) - return nil, err - } - ch := envOverride.Chart - ch.ReferenceChart = refChartByte - ch.UpdatedOn = time.Now() - ch.UpdatedBy = overrideRequest.UserId - err = impl.chartRepository.Update(ch) - if err != nil { - impl.logger.Errorw("chart update error", "err", err, "req", overrideRequest) - return nil, err - } - referenceChartByte = refChartByte - } - var err error - referenceChartByte, err = impl.overrideReferenceChartByteForHelmTypeApp(valuesOverrideResponse, chartMetaData, referenceTemplatePath, referenceChartByte) - if err != nil { - impl.logger.Errorw("ref chart commit error on cd trigger", "err", err, "req", overrideRequest) - return nil, err - } - return referenceChartByte, nil -} diff --git a/pkg/deployment/trigger/devtronApps/HandlerService_ent1.go b/pkg/deployment/trigger/devtronApps/deployStageHandlerCode_ent.go similarity index 84% rename from pkg/deployment/trigger/devtronApps/HandlerService_ent1.go rename to pkg/deployment/trigger/devtronApps/deployStageHandlerCode_ent.go index 24b7dab264..347f980a07 100644 --- a/pkg/deployment/trigger/devtronApps/HandlerService_ent1.go +++ b/pkg/deployment/trigger/devtronApps/deployStageHandlerCode_ent.go @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package devtronApps import ( diff --git a/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go b/pkg/deployment/trigger/devtronApps/postStageHandlerCode.go similarity index 100% rename from pkg/deployment/trigger/devtronApps/PostStageTriggerService.go rename to pkg/deployment/trigger/devtronApps/postStageHandlerCode.go diff --git a/pkg/deployment/trigger/devtronApps/PostStageTriggerService_ent.go b/pkg/deployment/trigger/devtronApps/postStageHandlerCode_ent.go similarity index 100% rename from pkg/deployment/trigger/devtronApps/PostStageTriggerService_ent.go rename to pkg/deployment/trigger/devtronApps/postStageHandlerCode_ent.go diff --git a/pkg/deployment/trigger/devtronApps/PrePostCommonService.go b/pkg/deployment/trigger/devtronApps/prePostWfAndLogsCode.go similarity index 100% rename from pkg/deployment/trigger/devtronApps/PrePostCommonService.go rename to pkg/deployment/trigger/devtronApps/prePostWfAndLogsCode.go diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService_ent.go b/pkg/deployment/trigger/devtronApps/preStageHandlerCode_ent.go similarity index 100% rename from pkg/deployment/trigger/devtronApps/PreStageTriggerService_ent.go rename to pkg/deployment/trigger/devtronApps/preStageHandlerCode_ent.go diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go b/pkg/deployment/trigger/devtronApps/preStageHandlerService.go similarity index 100% rename from pkg/deployment/trigger/devtronApps/PreStageTriggerService.go rename to pkg/deployment/trigger/devtronApps/preStageHandlerService.go From 43fb95b70f9de2c6738a0aaa12c93fc6a8ff1b40 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 8 Apr 2025 12:34:38 +0530 Subject: [PATCH 16/17] deployStage unstacked file --- .../devtronApps/deployStageHandlerCode.go | 1352 +++++++++++++++++ 1 file changed, 1352 insertions(+) create mode 100644 pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go diff --git a/pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go b/pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go new file mode 100644 index 0000000000..c0d8a204a1 --- /dev/null +++ b/pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go @@ -0,0 +1,1352 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package devtronApps + +import ( + "context" + "errors" + "fmt" + bean3 "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/api/bean/gitOps" + bean6 "github.com/devtron-labs/devtron/api/helm-app/bean" + "github.com/devtron-labs/devtron/api/helm-app/gRPC" + "github.com/devtron-labs/devtron/client/argocdServer" + bean7 "github.com/devtron-labs/devtron/client/argocdServer/bean" + "github.com/devtron-labs/devtron/internal/middleware" + "github.com/devtron-labs/devtron/internal/sql/models" + repository3 "github.com/devtron-labs/devtron/internal/sql/repository" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" + "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/app" + bean4 "github.com/devtron-labs/devtron/pkg/app/bean" + statusBean "github.com/devtron-labs/devtron/pkg/app/status/bean" + bean2 "github.com/devtron-labs/devtron/pkg/bean" + "github.com/devtron-labs/devtron/pkg/deployment/common" + bean9 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" + bean10 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/bean" + bean5 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/adapter" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/helper" + clientErrors "github.com/devtron-labs/devtron/pkg/errors" + k8s2 "github.com/devtron-labs/devtron/pkg/k8s" + bean8 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/repository" + repository6 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/repository" + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/go-pg/pg" + "go.opentelemetry.io/otel" + "golang.org/x/exp/slices" + "google.golang.org/grpc/codes" + status2 "google.golang.org/grpc/status" + "helm.sh/helm/v3/pkg/chart" + "net/http" + "path" + "regexp" + "strconv" + "strings" + "time" +) + +func (impl *HandlerServiceImpl) TriggerStageForBulk(triggerRequest bean.TriggerRequest) error { + + preStage, err := impl.pipelineStageService.GetCdStageByCdPipelineIdAndStageType(triggerRequest.Pipeline.Id, repository.PIPELINE_STAGE_TYPE_PRE_CD, false) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching CD pipeline stage", "cdPipelineId", triggerRequest.Pipeline.Id, "stage ", repository.PIPELINE_STAGE_TYPE_PRE_CD, "err", err) + return err + } + + // handle corrupt data (https://github.com/devtron-labs/devtron/issues/3826) + err, deleted := impl.deleteCorruptedPipelineStage(preStage, triggerRequest.TriggeredBy) + if err != nil { + impl.logger.Errorw("error in deleteCorruptedPipelineStage ", "cdPipelineId", triggerRequest.Pipeline.Id, "err", err, "preStage", preStage, "triggeredBy", triggerRequest.TriggeredBy) + return err + } + + triggerRequest.TriggerContext.Context = context.Background() + if len(triggerRequest.Pipeline.PreStageConfig) > 0 || (preStage != nil && !deleted) { + // pre stage exists + impl.logger.Debugw("trigger pre stage for pipeline", "artifactId", triggerRequest.Artifact.Id, "pipelineId", triggerRequest.Pipeline.Id) + triggerRequest.RefCdWorkflowRunnerId = 0 + err = impl.preStageHandlingForTriggerStageInBulk(&triggerRequest) + if err != nil { + return err + } + _, err = impl.TriggerPreStage(triggerRequest) // TODO handle error here + return err + } else { + // trigger deployment + impl.logger.Debugw("trigger cd for pipeline", "artifactId", triggerRequest.Artifact.Id, "pipelineId", triggerRequest.Pipeline.Id) + err = impl.TriggerAutomaticDeployment(triggerRequest) + return err + } +} + +func (impl *HandlerServiceImpl) getCdPipelineForManualCdTrigger(ctx context.Context, pipelineId int) (*pipelineConfig.Pipeline, error) { + _, span := otel.Tracer("HandlerService").Start(ctx, "getCdPipelineForManualCdTrigger") + defer span.End() + cdPipeline, err := impl.pipelineRepository.FindById(pipelineId) + if err != nil { + impl.logger.Errorw("manual trigger request with invalid pipelineId, ManualCdTrigger", "pipelineId", pipelineId, "err", err) + return nil, err + } + //checking if namespace exist or not + clusterIdToNsMap := map[int]string{ + cdPipeline.Environment.ClusterId: cdPipeline.Environment.Namespace, + } + + err = impl.helmAppService.CheckIfNsExistsForClusterIds(clusterIdToNsMap) + if err != nil { + impl.logger.Errorw("manual trigger request with invalid namespace, ManualCdTrigger", "pipelineId", pipelineId, "err", err) + return nil, err + } + return cdPipeline, nil +} + +func (impl *HandlerServiceImpl) validateDeploymentTriggerRequest(ctx context.Context, validateDeploymentTriggerObj *bean.ValidateDeploymentTriggerObj) error { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.validateDeploymentTriggerRequest") + defer span.End() + // custom GitOps repo url validation --> Start + err := impl.handleCustomGitOpsRepoValidation(validateDeploymentTriggerObj.Runner, validateDeploymentTriggerObj.CdPipeline, validateDeploymentTriggerObj.DeploymentConfig, validateDeploymentTriggerObj.TriggeredBy) + if err != nil { + impl.logger.Errorw("custom GitOps repository validation error, TriggerStage", "err", err) + return err + } + // custom GitOps repo url validation --> Ends + var isVulnerable bool + // if request is for rollback then bypass vulnerability validation + if !validateDeploymentTriggerObj.IsDeploymentTypeRollback() { + // checking vulnerability for deploying image + vulnerabilityCheckRequest := adapter.GetVulnerabilityCheckRequest(validateDeploymentTriggerObj.CdPipeline, validateDeploymentTriggerObj.ImageDigest) + isVulnerable, err = impl.imageScanService.GetArtifactVulnerabilityStatus(newCtx, vulnerabilityCheckRequest) + if err != nil { + impl.logger.Errorw("error in getting Artifact vulnerability status, ManualCdTrigger", "err", err) + return err + } + } + + if isVulnerable == true { + // if image vulnerable, update timeline status and return + if err = impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(validateDeploymentTriggerObj.Runner, errors.New(cdWorkflow.FOUND_VULNERABILITY), validateDeploymentTriggerObj.TriggeredBy); err != nil { + impl.logger.Errorw("error while updating current runner status to failed, TriggerDeployment", "wfrId", validateDeploymentTriggerObj.Runner.Id, "err", err) + } + return fmt.Errorf("found vulnerability for image digest %s", validateDeploymentTriggerObj.ImageDigest) + } + return nil +} + +// TODO: write a wrapper to handle auto and manual trigger +func (impl *HandlerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerContext, overrideRequest *bean3.ValuesOverrideRequest) (int, string, *bean4.ManifestPushTemplate, error) { + + triggerContext.TriggerType = bean.Manual + // setting triggeredAt variable to have consistent data for various audit log places in db for deployment time + triggeredAt := time.Now() + + releaseId := 0 + ctx := triggerContext.Context + cdPipeline, err := impl.getCdPipelineForManualCdTrigger(ctx, overrideRequest.PipelineId) + if err != nil { + if overrideRequest.WfrId != 0 { + err2 := impl.cdWorkflowCommonService.MarkDeploymentFailedForRunnerId(overrideRequest.WfrId, err, overrideRequest.UserId) + if err2 != nil { + impl.logger.Errorw("error while updating current runner status to failed, ManualCdTrigger", "cdWfr", overrideRequest.WfrId, "err2", err2) + } + } + return 0, "", nil, err + } + envDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(cdPipeline.AppId, cdPipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", cdPipeline.AppId, "envId", cdPipeline.EnvironmentId, "err", err) + return 0, "", nil, err + } + + adapter.SetPipelineFieldsInOverrideRequest(overrideRequest, cdPipeline, envDeploymentConfig) + ciArtifactId := overrideRequest.CiArtifactId + + _, span := otel.Tracer("orchestrator").Start(ctx, "ciArtifactRepository.Get") + artifact, err := impl.ciArtifactRepository.Get(ciArtifactId) + span.End() + if err != nil { + impl.logger.Errorw("error in getting CiArtifact", "CiArtifactId", overrideRequest.CiArtifactId, "err", err) + return 0, "", nil, err + } + + if artifact.IsMigrationRequired() { + // Migration of deprecated DataSource Type + migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(artifact.Id) + if migrationErr != nil { + impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id) + } + } + + _, imageTag, err := artifact.ExtractImageRepoAndTag() + if err != nil { + impl.logger.Errorw("error in getting image tag and repo", "err", err) + } + helmPackageName := fmt.Sprintf("%s-%s-%s", cdPipeline.App.AppName, cdPipeline.Environment.Name, imageTag) + var manifestPushTemplate *bean4.ManifestPushTemplate + + switch overrideRequest.CdWorkflowType { + case bean3.CD_WORKFLOW_TYPE_PRE: + var cdWf *pipelineConfig.CdWorkflow + if overrideRequest.CdWorkflowId == 0 { + cdWf = &pipelineConfig.CdWorkflow{ + CiArtifactId: artifact.Id, + PipelineId: cdPipeline.Id, + AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: 1, UpdatedOn: triggeredAt, UpdatedBy: 1}, + } + err := impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) + if err != nil { + return 0, "", nil, err + } + } else { + cdWf, err = impl.cdWorkflowRepository.FindById(overrideRequest.CdWorkflowId) + if err != nil { + impl.logger.Errorw("error in TriggerPreStage, ManualCdTrigger", "err", err) + return 0, "", nil, err + } + } + overrideRequest.CdWorkflowId = cdWf.Id + + _, span = otel.Tracer("orchestrator").Start(ctx, "TriggerPreStage") + triggerRequest := bean.TriggerRequest{ + CdWf: cdWf, + Artifact: artifact, + Pipeline: cdPipeline, + TriggeredBy: overrideRequest.UserId, + ApplyAuth: false, + TriggerContext: triggerContext, + RefCdWorkflowRunnerId: 0, + CdWorkflowRunnerId: overrideRequest.WfrId, + } + manifestPushTemplate, err = impl.TriggerPreStage(triggerRequest) + span.End() + if err != nil { + impl.logger.Errorw("error in TriggerPreStage, ManualCdTrigger", "err", err) + return 0, "", nil, err + } + case bean3.CD_WORKFLOW_TYPE_DEPLOY: + if overrideRequest.DeploymentType == models.DEPLOYMENTTYPE_UNKNOWN { + overrideRequest.DeploymentType = models.DEPLOYMENTTYPE_DEPLOY + } + + cdWf, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, overrideRequest.CdWorkflowId, bean3.CD_WORKFLOW_TYPE_PRE) + if err != nil && !util.IsErrNoRows(err) { + impl.logger.Errorw("error in getting cdWorkflow, ManualCdTrigger", "CdWorkflowId", overrideRequest.CdWorkflowId, "err", err) + return 0, "", nil, err + } + + cdWorkflowId := cdWf.CdWorkflowId + if cdWf.CdWorkflowId == 0 { + cdWf := &pipelineConfig.CdWorkflow{ + CiArtifactId: overrideRequest.CiArtifactId, + PipelineId: overrideRequest.PipelineId, + AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, + } + err := impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) + if err != nil { + impl.logger.Errorw("error in creating cdWorkflow, ManualCdTrigger", "PipelineId", overrideRequest.PipelineId, "err", err) + return 0, "", nil, err + } + cdWorkflowId = cdWf.Id + } + + runner := &pipelineConfig.CdWorkflowRunner{ + Name: cdPipeline.Name, + WorkflowType: bean3.CD_WORKFLOW_TYPE_DEPLOY, + ExecutorType: cdWorkflow.WORKFLOW_EXECUTOR_TYPE_AWF, + Status: cdWorkflow.WorkflowInitiated, //deployment Initiated for manual trigger + TriggeredBy: overrideRequest.UserId, + StartedOn: triggeredAt, + Namespace: impl.config.GetDefaultNamespace(), + CdWorkflowId: cdWorkflowId, + AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, + ReferenceId: triggerContext.ReferenceId, + } + savedWfr, err := impl.cdWorkflowRunnerService.SaveCDWorkflowRunnerWithStage(runner) + if err != nil { + impl.logger.Errorw("err in creating cdWorkflowRunner, ManualCdTrigger", "cdWorkflowId", cdWorkflowId, "err", err) + return 0, "", nil, err + } + runner.CdWorkflow = &pipelineConfig.CdWorkflow{ + Pipeline: cdPipeline, + } + overrideRequest.WfrId = savedWfr.Id + overrideRequest.CdWorkflowId = cdWorkflowId + // creating cd pipeline status timeline for deployment initialisation + timeline := impl.pipelineStatusTimelineService.NewDevtronAppPipelineStatusTimelineDbObject(runner.Id, timelineStatus.TIMELINE_STATUS_DEPLOYMENT_INITIATED, timelineStatus.TIMELINE_DESCRIPTION_DEPLOYMENT_INITIATED, overrideRequest.UserId) + _, span = otel.Tracer("orchestrator").Start(ctx, "cdPipelineStatusTimelineRepo.SaveTimelineForACDHelmApps") + _, err = impl.pipelineStatusTimelineService.SaveTimelineIfNotAlreadyPresent(timeline, nil) + + span.End() + if err != nil { + impl.logger.Errorw("error in creating timeline status for deployment initiation, ManualCdTrigger", "err", err, "timeline", timeline) + } + if isNotHibernateRequest(overrideRequest.DeploymentType) { + validateReqObj := adapter.NewValidateDeploymentTriggerObj(runner, cdPipeline, artifact.ImageDigest, envDeploymentConfig, overrideRequest.UserId, overrideRequest.IsRollbackDeployment) + validationErr := impl.validateDeploymentTriggerRequest(ctx, validateReqObj) + if validationErr != nil { + impl.logger.Errorw("validation error deployment request", "cdWfr", runner.Id, "err", validationErr) + return 0, "", nil, validationErr + } + } + // Deploy the release + var releaseErr error + releaseId, manifestPushTemplate, releaseErr = impl.handleCDTriggerRelease(ctx, overrideRequest, envDeploymentConfig, triggeredAt, overrideRequest.UserId) + // if releaseErr found, then the mark current deployment Failed and return + if releaseErr != nil { + err := impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, releaseErr, overrideRequest.UserId) + if err != nil { + impl.logger.Errorw("error while updating current runner status to failed", "cdWfr", runner.Id, "err", err) + } + return 0, "", nil, releaseErr + } + + case bean3.CD_WORKFLOW_TYPE_POST: + cdWfRunner, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, overrideRequest.CdWorkflowId, bean3.CD_WORKFLOW_TYPE_DEPLOY) + if err != nil && !util.IsErrNoRows(err) { + impl.logger.Errorw("err in getting cdWorkflowRunner, ManualCdTrigger", "cdWorkflowId", overrideRequest.CdWorkflowId, "err", err) + return 0, "", nil, err + } + + var cdWf *pipelineConfig.CdWorkflow + if cdWfRunner.CdWorkflowId == 0 { + cdWf = &pipelineConfig.CdWorkflow{ + CiArtifactId: ciArtifactId, + PipelineId: overrideRequest.PipelineId, + AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, + } + err := impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) + if err != nil { + impl.logger.Errorw("error in creating cdWorkflow, ManualCdTrigger", "CdWorkflowId", overrideRequest.CdWorkflowId, "err", err) + return 0, "", nil, err + } + overrideRequest.CdWorkflowId = cdWf.Id + } else { + _, span = otel.Tracer("orchestrator").Start(ctx, "cdWorkflowRepository.FindById") + cdWf, err = impl.cdWorkflowRepository.FindById(overrideRequest.CdWorkflowId) + span.End() + if err != nil && !util.IsErrNoRows(err) { + impl.logger.Errorw("error in getting cdWorkflow, ManualCdTrigger", "CdWorkflowId", overrideRequest.CdWorkflowId, "err", err) + return 0, "", nil, err + } + } + _, span = otel.Tracer("orchestrator").Start(ctx, "TriggerPostStage") + triggerRequest := bean.TriggerRequest{ + CdWf: cdWf, + Pipeline: cdPipeline, + TriggeredBy: overrideRequest.UserId, + RefCdWorkflowRunnerId: 0, + TriggerContext: triggerContext, + CdWorkflowRunnerId: overrideRequest.WfrId, + } + manifestPushTemplate, err = impl.TriggerPostStage(triggerRequest) + span.End() + if err != nil { + impl.logger.Errorw("error in TriggerPostStage, ManualCdTrigger", "CdWorkflowId", cdWf.Id, "err", err) + return 0, "", nil, err + } + default: + impl.logger.Errorw("invalid CdWorkflowType, ManualCdTrigger", "CdWorkflowType", overrideRequest.CdWorkflowType, "err", err) + return 0, "", nil, fmt.Errorf("invalid CdWorkflowType %s for the trigger request", string(overrideRequest.CdWorkflowType)) + } + return releaseId, helmPackageName, manifestPushTemplate, err +} + +func isNotHibernateRequest(deploymentType models.DeploymentType) bool { + return deploymentType != models.DEPLOYMENTTYPE_STOP && deploymentType != models.DEPLOYMENTTYPE_START +} + +// TODO: write a wrapper to handle auto and manual trigger +func (impl *HandlerServiceImpl) TriggerAutomaticDeployment(request bean.TriggerRequest) error { + // in case of manual trigger auth is already applied and for auto triggers there is no need for auth check here + triggeredBy := request.TriggeredBy + pipeline := request.Pipeline + artifact := request.Artifact + + //setting triggeredAt variable to have consistent data for various audit log places in db for deployment time + triggeredAt := time.Now() + cdWf := request.CdWf + ctx := context.Background() + + if cdWf == nil || (cdWf != nil && cdWf.CiArtifactId != artifact.Id) { + // cdWf != nil && cdWf.CiArtifactId != artifact.Id for auto trigger case when deployment is triggered with image generated by plugin + cdWf = &pipelineConfig.CdWorkflow{ + CiArtifactId: artifact.Id, + PipelineId: pipeline.Id, + AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: 1, UpdatedOn: triggeredAt, UpdatedBy: 1}, + } + err := impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) + if err != nil { + return err + } + } + + runner := &pipelineConfig.CdWorkflowRunner{ + Name: pipeline.Name, + WorkflowType: bean3.CD_WORKFLOW_TYPE_DEPLOY, + ExecutorType: cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM, + Status: cdWorkflow.WorkflowInitiated, // deployment Initiated for auto trigger + TriggeredBy: triggeredBy, + StartedOn: triggeredAt, + Namespace: impl.config.GetDefaultNamespace(), + CdWorkflowId: cdWf.Id, + AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: triggeredBy, UpdatedOn: triggeredAt, UpdatedBy: triggeredBy}, + ReferenceId: request.TriggerContext.ReferenceId, + } + savedWfr, err := impl.cdWorkflowRunnerService.SaveCDWorkflowRunnerWithStage(runner) + if err != nil { + return err + } + runner.CdWorkflow = &pipelineConfig.CdWorkflow{ + Pipeline: pipeline, + } + // creating cd pipeline status timeline for deployment initialisation + timeline := &pipelineConfig.PipelineStatusTimeline{ + CdWorkflowRunnerId: runner.Id, + Status: timelineStatus.TIMELINE_STATUS_DEPLOYMENT_INITIATED, + StatusDetail: "Deployment initiated successfully.", + StatusTime: time.Now(), + } + timeline.CreateAuditLog(1) + err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil) + if err != nil { + impl.logger.Errorw("error in creating timeline status for deployment initiation", "err", err, "timeline", timeline) + } + envDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(pipeline.AppId, pipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) + return err + } + validationErr := impl.validateDeploymentTriggerRequest(ctx, adapter.NewValidateDeploymentTriggerObj(runner, pipeline, artifact.ImageDigest, envDeploymentConfig, triggeredBy, false)) + if validationErr != nil { + impl.logger.Errorw("validation error deployment request", "cdWfr", runner.Id, "err", validationErr) + return validationErr + } + releaseErr := impl.TriggerCD(ctx, artifact, cdWf.Id, savedWfr.Id, pipeline, envDeploymentConfig, triggeredAt, triggeredBy) + // if releaseErr found, then the mark current deployment Failed and return + if releaseErr != nil { + err := impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, releaseErr, triggeredBy) + if err != nil { + impl.logger.Errorw("error while updating current runner status to failed, updatePreviousDeploymentStatus", "cdWfr", runner.Id, "err", err) + } + return releaseErr + } + return nil +} + +func (impl *HandlerServiceImpl) TriggerCD(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) error { + impl.logger.Debugw("automatic pipeline trigger attempt async", "artifactId", artifact.Id) + err := impl.triggerReleaseAsync(ctx, artifact, cdWorkflowId, wfrId, pipeline, envDeploymentConfig, triggeredAt, triggeredBy) + if err != nil { + impl.logger.Errorw("error in cd trigger", "err", err) + return err + } + return err +} + +func (impl *HandlerServiceImpl) triggerReleaseAsync(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) error { + err := impl.validateAndTrigger(ctx, pipeline, envDeploymentConfig, artifact, cdWorkflowId, wfrId, triggeredAt, triggeredBy) + if err != nil { + impl.logger.Errorw("error in trigger for pipeline", "pipelineId", strconv.Itoa(pipeline.Id)) + } + impl.logger.Debugw("trigger attempted for all pipeline ", "artifactId", artifact.Id) + return err +} + +func (impl *HandlerServiceImpl) validateAndTrigger(ctx context.Context, p *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time, triggeredBy int32) error { + //TODO: verify this logic + object := impl.enforcerUtil.GetAppRBACNameByAppId(p.AppId) + envApp := strings.Split(object, "/") + if len(envApp) != 2 { + impl.logger.Error("invalid req, app and env not found from rbac") + return errors.New("invalid req, app and env not found from rbac") + } + err := impl.releasePipeline(ctx, p, envDeploymentConfig, artifact, cdWorkflowId, wfrId, triggeredAt, triggeredBy) + return err +} + +func (impl *HandlerServiceImpl) releasePipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time, triggeredBy int32) error { + startTime := time.Now() + defer func() { + impl.logger.Debugw("auto trigger release process completed", "timeTaken", time.Since(startTime), "cdPipelineId", pipeline.Id, "artifactId", artifact.Id, "wfrId", wfrId) + }() + impl.logger.Debugw("auto triggering release for", "cdPipelineId", pipeline.Id, "artifactId", artifact.Id, "wfrId", wfrId) + pipeline, err := impl.pipelineRepository.FindById(pipeline.Id) + if err != nil { + impl.logger.Errorw("error in fetching pipeline by pipelineId", "err", err) + return err + } + + request := &bean3.ValuesOverrideRequest{ + PipelineId: pipeline.Id, + UserId: artifact.CreatedBy, + CiArtifactId: artifact.Id, + AppId: pipeline.AppId, + CdWorkflowId: cdWorkflowId, + ForceTrigger: true, + DeploymentWithConfig: bean3.DEPLOYMENT_CONFIG_TYPE_LAST_SAVED, + WfrId: wfrId, + } + + adapter.SetPipelineFieldsInOverrideRequest(request, pipeline, envDeploymentConfig) + + id, _, err := impl.handleCDTriggerRelease(ctx, request, envDeploymentConfig, triggeredAt, triggeredBy) + if err != nil { + impl.logger.Errorw("error in auto cd pipeline trigger", "pipelineId", pipeline.Id, "artifactId", artifact.Id, "err", err) + } else { + impl.logger.Infow("pipeline successfully triggered", "cdPipelineId", pipeline.Id, "artifactId", artifact.Id, "releaseId", id) + } + return err +} + +func (impl *HandlerServiceImpl) triggerAsyncRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, + envDeploymentConfig *bean9.DeploymentConfig, userDeploymentRequestId int, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.triggerAsyncRelease") + defer span.End() + // build merged values and save PCO history for the release + valuesOverrideResponse, err := impl.manifestCreationService.GetValuesOverrideForTrigger(newCtx, overrideRequest, envDeploymentConfig, triggeredAt) + // auditDeploymentTriggerHistory is performed irrespective of GetValuesOverrideForTrigger error - for auditing purposes + historyErr := impl.auditDeploymentTriggerHistory(overrideRequest.WfrId, valuesOverrideResponse, newCtx, triggeredAt, deployedBy) + if historyErr != nil { + impl.logger.Errorw("error in auditing deployment trigger history", "cdWfrId", overrideRequest.WfrId, "err", err) + return releaseNo, manifestPushTemplate, err + } + // handling GetValuesOverrideForTrigger error + if err != nil { + impl.logger.Errorw("error in fetching values for trigger", "err", err) + return releaseNo, manifestPushTemplate, err + } + // asynchronous mode of Helm/ArgoCd installation starts + return impl.workflowEventPublishService.TriggerAsyncRelease(userDeploymentRequestId, overrideRequest, valuesOverrideResponse, newCtx, deployedBy) +} + +func (impl *HandlerServiceImpl) handleCDTriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.handleCDTriggerRelease") + defer span.End() + // Handling for auto trigger + if overrideRequest.UserId == 0 { + overrideRequest.UserId = deployedBy + } + tx, err := impl.transactionUtilImpl.StartTx() + if err != nil { + impl.logger.Errorw("error in starting transaction to update userDeploymentRequest", "error", err) + return releaseNo, manifestPushTemplate, err + } + defer impl.transactionUtilImpl.RollbackTx(tx) + newDeploymentRequest := adapter.NewUserDeploymentRequest(overrideRequest, triggeredAt, overrideRequest.UserId) + // creating new user deployment request + userDeploymentRequestId, err := impl.userDeploymentRequestService.SaveNewDeployment(newCtx, tx, newDeploymentRequest) + if err != nil { + impl.logger.Errorw("error in saving new userDeploymentRequest", "overrideRequest", overrideRequest, "err", err) + return releaseNo, manifestPushTemplate, err + } + timeline := impl.pipelineStatusTimelineService.NewDevtronAppPipelineStatusTimelineDbObject(overrideRequest.WfrId, timelineStatus.TIMELINE_STATUS_DEPLOYMENT_REQUEST_VALIDATED, timelineStatus.TIMELINE_DESCRIPTION_DEPLOYMENT_REQUEST_VALIDATED, deployedBy) + // creating cd pipeline status timeline for deployment trigger request validated + _, err = impl.pipelineStatusTimelineService.SaveTimelineIfNotAlreadyPresent(timeline, tx) + err = impl.transactionUtilImpl.CommitTx(tx) + if err != nil { + impl.logger.Errorw("error in committing transaction to update userDeploymentRequest", "error", err) + return userDeploymentRequestId, manifestPushTemplate, err + } + isAsyncMode, err := impl.isDevtronAsyncInstallModeEnabled(overrideRequest) + if err != nil { + impl.logger.Errorw("error in checking async mode for devtron app", "err", err, "deploymentType", overrideRequest.DeploymentType, + "forceSyncDeployment", overrideRequest.ForceSyncDeployment, "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId) + return userDeploymentRequestId, manifestPushTemplate, err + } + if envDeploymentConfig.IsEmpty() { + deploymentConfig, dbErr := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(overrideRequest.AppId, overrideRequest.EnvId) + if dbErr != nil { + impl.logger.Errorw("error in getting deployment config by appId and envId", "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId, "err", dbErr) + return releaseNo, manifestPushTemplate, dbErr + } + envDeploymentConfig = deploymentConfig + } + if isAsyncMode { + return impl.triggerAsyncRelease(newCtx, overrideRequest, envDeploymentConfig, userDeploymentRequestId, triggeredAt, deployedBy) + } + // synchronous mode of installation starts + return impl.TriggerRelease(newCtx, overrideRequest, envDeploymentConfig, triggeredAt, deployedBy) +} + +func (impl *HandlerServiceImpl) auditDeploymentTriggerHistory(cdWfrId int, valuesOverrideResponse *app.ValuesOverrideResponse, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (err error) { + if valuesOverrideResponse.Pipeline == nil || valuesOverrideResponse.EnvOverride == nil { + impl.logger.Warnw("unable to save histories for deployment trigger, invalid valuesOverrideResponse received", "cdWfrId", cdWfrId) + return nil + } + err1 := impl.deployedConfigurationHistoryService.CreateHistoriesForDeploymentTrigger(ctx, valuesOverrideResponse.Pipeline, valuesOverrideResponse.PipelineStrategy, valuesOverrideResponse.EnvOverride, triggeredAt, triggeredBy) + if err1 != nil { + impl.logger.Errorw("error in saving histories for deployment trigger", "err", err1, "pipelineId", valuesOverrideResponse.Pipeline.Id, "cdWfrId", cdWfrId) + return nil + } + return nil +} + +// TriggerRelease will trigger Install/Upgrade request for Devtron App releases synchronously +func (impl *HandlerServiceImpl) TriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, + envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.TriggerRelease") + defer span.End() + triggerEvent, skipRequest, err := impl.buildTriggerEventForOverrideRequest(overrideRequest, triggeredAt) + if err != nil { + return releaseNo, manifestPushTemplate, err + } + impl.logger.Debugw("processing TriggerRelease", "wfrId", overrideRequest.WfrId, "triggerEvent", triggerEvent) + // request has already been served, skipping + if skipRequest { + impl.logger.Infow("request already served, skipping", "wfrId", overrideRequest.WfrId) + return releaseNo, manifestPushTemplate, nil + } + // build merged values and save PCO history for the release + valuesOverrideResponse, builtChartPath, err := impl.manifestCreationService.BuildManifestForTrigger(newCtx, overrideRequest, envDeploymentConfig, triggeredAt) + + // auditDeploymentTriggerHistory is performed irrespective of BuildManifestForTrigger error - for auditing purposes + historyErr := impl.auditDeploymentTriggerHistory(overrideRequest.WfrId, valuesOverrideResponse, newCtx, triggeredAt, triggeredBy) + if historyErr != nil { + impl.logger.Errorw("error in auditing deployment trigger history", "cdWfrId", overrideRequest.WfrId, "err", err) + return releaseNo, manifestPushTemplate, err + } + if err != nil { + impl.logger.Errorw("error in building merged manifest for trigger", "err", err) + impl.manifestGenerationFailedTimelineHandling(triggerEvent, overrideRequest, err) + return releaseNo, manifestPushTemplate, err + } + helmManifest, err := impl.getHelmManifestForTriggerRelease(ctx, triggerEvent, overrideRequest, valuesOverrideResponse, builtChartPath) + if err != nil { + impl.logger.Errorw("error, getHelmManifestForTriggerRelease", "err", err) + return releaseNo, manifestPushTemplate, err + } + impl.logger.Debugw("triggering pipeline for release", "wfrId", overrideRequest.WfrId, "builtChartPath", builtChartPath) + releaseNo, err = impl.triggerPipeline(overrideRequest, valuesOverrideResponse, builtChartPath, triggerEvent, newCtx) + if err != nil { + return 0, manifestPushTemplate, err + } + + err = impl.triggerReleaseSuccessHandling(triggerEvent, overrideRequest, valuesOverrideResponse, helmManifest) + if err != nil { + impl.logger.Errorw("error, triggerReleaseSuccessHandling", "triggerEvent", triggerEvent, "err", err) + return releaseNo, manifestPushTemplate, err + } + // creating cd pipeline status timeline for deployment triggered - for successfully triggered requests + timeline := impl.pipelineStatusTimelineService.NewDevtronAppPipelineStatusTimelineDbObject(overrideRequest.WfrId, timelineStatus.TIMELINE_STATUS_DEPLOYMENT_TRIGGERED, timelineStatus.TIMELINE_DESCRIPTION_DEPLOYMENT_COMPLETED, overrideRequest.UserId) + _, dbErr := impl.pipelineStatusTimelineService.SaveTimelineIfNotAlreadyPresent(timeline, nil) + if dbErr != nil { + impl.logger.Errorw("error in creating timeline status for deployment completed", "err", dbErr, "timeline", timeline) + } + impl.logger.Debugw("triggered pipeline for release successfully", "wfrId", overrideRequest.WfrId, "builtChartPath", builtChartPath) + return releaseNo, valuesOverrideResponse.ManifestPushTemplate, nil +} + +func (impl *HandlerServiceImpl) performGitOps(ctx context.Context, + overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, + builtChartPath string, triggerEvent bean.TriggerEvent) error { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.performGitOps") + defer span.End() + // update workflow runner status, used in app workflow view + err := impl.cdWorkflowCommonService.UpdateNonTerminalStatusInRunner(newCtx, overrideRequest.WfrId, overrideRequest.UserId, cdWorkflow.WorkflowInProgress) + if err != nil { + impl.logger.Errorw("error in updating the workflow runner status", "err", err) + return err + } + manifestPushTemplate, err := impl.buildManifestPushTemplate(overrideRequest, valuesOverrideResponse, builtChartPath) + if err != nil { + impl.logger.Errorw("error in building manifest push template", "err", err) + return err + } + manifestPushService := impl.getManifestPushService(triggerEvent.ManifestStorageType) + manifestPushResponse := manifestPushService.PushChart(newCtx, manifestPushTemplate) + if manifestPushResponse.Error != nil { + impl.logger.Errorw("error in pushing manifest to git/helm", "err", manifestPushResponse.Error, "git_repo_url", manifestPushTemplate.RepoUrl) + return manifestPushResponse.Error + } + if manifestPushResponse.IsNewGitRepoConfigured() { + // Update GitOps repo url after repo new repo created + valuesOverrideResponse.DeploymentConfig.SetRepoURL(manifestPushResponse.NewGitRepoUrl) + } + valuesOverrideResponse.ManifestPushTemplate = manifestPushTemplate + return nil +} + +func (impl *HandlerServiceImpl) buildTriggerEventForOverrideRequest(overrideRequest *bean3.ValuesOverrideRequest, triggeredAt time.Time) (triggerEvent bean.TriggerEvent, skipRequest bool, err error) { + triggerEvent = helper.NewTriggerEvent(overrideRequest.DeploymentAppType, triggeredAt, overrideRequest.UserId) + request := statusBean.NewTimelineGetRequest(). + WithCdWfrId(overrideRequest.WfrId). + ExcludingStatuses(timelineStatus.TIMELINE_STATUS_UNABLE_TO_FETCH_STATUS, + timelineStatus.TIMELINE_STATUS_KUBECTL_APPLY_STARTED, + timelineStatus.TIMELINE_STATUS_KUBECTL_APPLY_SYNCED) + timelineStatuses, err := impl.pipelineStatusTimelineService.GetTimelineStatusesFor(request) + if err != nil { + impl.logger.Errorw("error in getting last timeline status by cdWfrId", "cdWfrId", overrideRequest.WfrId, "err", err) + return triggerEvent, skipRequest, err + } else if !slices.Contains(timelineStatuses, timelineStatus.TIMELINE_STATUS_DEPLOYMENT_REQUEST_VALIDATED) { + impl.logger.Errorw("pre-condition missing: timeline for deployment request validation", "cdWfrId", overrideRequest.WfrId, "timelineStatuses", timelineStatuses) + return triggerEvent, skipRequest, fmt.Errorf("pre-condition missing: timeline for deployment request validation") + } else if timelineStatus.ContainsTerminalTimelineStatus(timelineStatuses) { + impl.logger.Info("deployment is already terminated", "cdWfrId", overrideRequest.WfrId, "timelineStatuses", timelineStatuses) + skipRequest = true + return triggerEvent, skipRequest, nil + } else if slices.Contains(timelineStatuses, timelineStatus.TIMELINE_STATUS_DEPLOYMENT_TRIGGERED) { + impl.logger.Info("deployment has been performed. skipping", "cdWfrId", overrideRequest.WfrId, "timelineStatuses", timelineStatuses) + skipRequest = true + return triggerEvent, skipRequest, nil + } + if slices.Contains(timelineStatuses, timelineStatus.TIMELINE_STATUS_GIT_COMMIT) || + slices.Contains(timelineStatuses, timelineStatus.TIMELINE_STATUS_ARGOCD_SYNC_INITIATED) { + // git commit has already been performed + triggerEvent.PerformChartPush = false + } + if slices.Contains(timelineStatuses, timelineStatus.TIMELINE_STATUS_ARGOCD_SYNC_COMPLETED) { + // ArgoCd sync has already been performed + triggerEvent.DeployArgoCdApp = false + } + return triggerEvent, skipRequest, nil +} + +func (impl *HandlerServiceImpl) triggerPipeline(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, triggerEvent bean.TriggerEvent, ctx context.Context) (releaseNo int, err error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.triggerPipeline") + defer span.End() + if triggerEvent.PerformChartPush { + impl.logger.Debugw("performing chart push operation in triggerPipeline", "cdWfrId", overrideRequest.WfrId) + err = impl.performGitOps(newCtx, overrideRequest, valuesOverrideResponse, builtChartPath, triggerEvent) + if err != nil { + impl.logger.Errorw("error in performing GitOps", "cdWfrId", overrideRequest.WfrId, "err", err) + return releaseNo, err + } + impl.logger.Debugw("chart push operation completed successfully", "cdWfrId", overrideRequest.WfrId) + } + + if triggerEvent.PerformDeploymentOnCluster { + err = impl.deployApp(newCtx, overrideRequest, valuesOverrideResponse, triggerEvent) + if err != nil { + impl.logger.Errorw("error in deploying app", "err", err) + return releaseNo, err + } + } + + go impl.writeCDTriggerEvent(overrideRequest, valuesOverrideResponse.Artifact, valuesOverrideResponse.PipelineOverride.PipelineReleaseCounter, valuesOverrideResponse.PipelineOverride.Id, overrideRequest.WfrId) + + _ = impl.markImageScanDeployed(newCtx, overrideRequest.AppId, overrideRequest.EnvId, overrideRequest.ClusterId, + valuesOverrideResponse.Artifact.ImageDigest, valuesOverrideResponse.Artifact.ScanEnabled, valuesOverrideResponse.Artifact.Image) + + middleware.CdTriggerCounter.WithLabelValues(overrideRequest.AppName, overrideRequest.EnvName).Inc() + + // Update previous deployment runner status (in transaction): Failed + dbErr := impl.cdWorkflowCommonService.SupersedePreviousDeployments(newCtx, overrideRequest.WfrId, overrideRequest.PipelineId, triggerEvent.TriggeredAt, overrideRequest.UserId) + if dbErr != nil { + impl.logger.Errorw("error while update previous cd workflow runners", "err", dbErr, "currentRunnerId", overrideRequest.WfrId, "pipelineId", overrideRequest.PipelineId) + return releaseNo, dbErr + } + return valuesOverrideResponse.PipelineOverride.PipelineReleaseCounter, nil +} + +func (impl *HandlerServiceImpl) buildManifestPushTemplate(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string) (*bean4.ManifestPushTemplate, error) { + + manifestPushTemplate := &bean4.ManifestPushTemplate{ + WorkflowRunnerId: overrideRequest.WfrId, + AppId: overrideRequest.AppId, + ChartRefId: valuesOverrideResponse.EnvOverride.Chart.ChartRefId, + EnvironmentId: valuesOverrideResponse.EnvOverride.Environment.Id, + EnvironmentName: valuesOverrideResponse.EnvOverride.Environment.Namespace, + UserId: overrideRequest.UserId, + PipelineOverrideId: valuesOverrideResponse.PipelineOverride.Id, + AppName: overrideRequest.AppName, + TargetEnvironmentId: valuesOverrideResponse.EnvOverride.TargetEnvironment, + BuiltChartPath: builtChartPath, + MergedValues: valuesOverrideResponse.MergedValues, + } + + manifestPushConfig, err := impl.manifestPushConfigRepository.GetManifestPushConfigByAppIdAndEnvId(overrideRequest.AppId, overrideRequest.EnvId) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching manifest push config from db", "err", err) + return manifestPushTemplate, err + } + + if manifestPushConfig != nil && manifestPushConfig.Id != 0 { + if manifestPushConfig.StorageType == bean2.ManifestStorageGit { + // need to implement for git repo push + // currently manifest push config doesn't have git push config. GitOps config is derived from charts, chart_env_config_override and chart_ref table + } else { + err2 := impl.buildManifestPushTemplateForNonGitStorageType(overrideRequest, valuesOverrideResponse, builtChartPath, err, manifestPushConfig, manifestPushTemplate) + if err2 != nil { + return manifestPushTemplate, err2 + } + } + } else { + manifestPushTemplate.ChartReferenceTemplate = valuesOverrideResponse.EnvOverride.Chart.ReferenceTemplate + manifestPushTemplate.ChartName = valuesOverrideResponse.EnvOverride.Chart.ChartName + manifestPushTemplate.ChartVersion = valuesOverrideResponse.EnvOverride.Chart.ChartVersion + manifestPushTemplate.ChartLocation = valuesOverrideResponse.DeploymentConfig.GetChartLocation() + manifestPushTemplate.RepoUrl = valuesOverrideResponse.DeploymentConfig.GetRepoURL() + manifestPushTemplate.TargetRevision = valuesOverrideResponse.DeploymentConfig.GetTargetRevision() + manifestPushTemplate.ValuesFilePath = valuesOverrideResponse.DeploymentConfig.GetValuesFilePath() + manifestPushTemplate.ReleaseMode = valuesOverrideResponse.DeploymentConfig.ReleaseMode + manifestPushTemplate.IsCustomGitRepository = common.IsCustomGitOpsRepo(valuesOverrideResponse.DeploymentConfig.ConfigType) + manifestPushTemplate.IsArgoSyncSupported = valuesOverrideResponse.DeploymentConfig.IsArgoAppSyncAndRefreshSupported() + } + return manifestPushTemplate, nil +} + +func (impl *HandlerServiceImpl) deployApp(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, triggerEvent bean.TriggerEvent) error { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.deployApp") + defer span.End() + var referenceChartByte []byte + var err error + + if util.IsAcdApp(overrideRequest.DeploymentAppType) && triggerEvent.DeployArgoCdApp { + err = impl.deployArgoCdApp(newCtx, overrideRequest, valuesOverrideResponse) + if err != nil { + impl.logger.Errorw("error in deploying app on ArgoCd", "err", err) + return err + } + } else if util.IsHelmApp(overrideRequest.DeploymentAppType) { + _, referenceChartByte, err = impl.createHelmAppForCdPipeline(newCtx, overrideRequest, valuesOverrideResponse) + if err != nil { + impl.logger.Errorw("error in creating or updating helm application for cd pipeline", "err", err) + return err + } + } + impl.postDeployHook(overrideRequest, valuesOverrideResponse, referenceChartByte, err) + return nil +} + +func (impl *HandlerServiceImpl) createHelmAppForCdPipeline(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse) (bool, []byte, error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.createHelmAppForCdPipeline") + defer span.End() + pipelineModel := valuesOverrideResponse.Pipeline + envOverride := valuesOverrideResponse.EnvOverride + mergeAndSave := valuesOverrideResponse.MergedValues + chartMetaData, helmRevisionHistory, releaseIdentifier, err := impl.getHelmHistoryLimitAndChartMetadataForHelmAppCreation(ctx, valuesOverrideResponse) + if err != nil { + impl.logger.Errorw("error, getHelmHistoryLimitAndChartMetadataForHelmAppCreation", "valuesOverrideResponse", valuesOverrideResponse, "err", err) + return false, nil, err + } + referenceTemplate := envOverride.Chart.ReferenceTemplate + referenceTemplatePath := path.Join(bean5.RefChartDirPath, referenceTemplate) + var referenceChartByte []byte + if util.IsHelmApp(valuesOverrideResponse.DeploymentConfig.DeploymentAppType) { + sanitizedK8sVersion, err := impl.getSanitizedK8sVersion(referenceTemplate) + if err != nil { + return false, nil, err + } + referenceChartByte, err = impl.getReferenceChartByteForHelmTypeApp(envOverride, chartMetaData, referenceTemplatePath, overrideRequest, valuesOverrideResponse) + if err != nil { + impl.logger.Errorw("error, getReferenceChartByteForHelmTypeApp", "envOverride", envOverride, "err", err) + return false, nil, err + } + if pipelineModel.DeploymentAppCreated { + req := &gRPC.UpgradeReleaseRequest{ + ReleaseIdentifier: releaseIdentifier, + ValuesYaml: mergeAndSave, + HistoryMax: helmRevisionHistory, + ChartContent: &gRPC.ChartContent{Content: referenceChartByte}, + } + if len(sanitizedK8sVersion) > 0 { + req.K8SVersion = sanitizedK8sVersion + } + if impl.isDevtronAsyncHelmInstallModeEnabled(overrideRequest.ForceSyncDeployment) { + req.RunInCtx = true + } + // For cases where helm release was not found, kubelink will install the same configuration + updateApplicationResponse, err := impl.helmAppClient.UpdateApplication(newCtx, req) + if err != nil { + impl.logger.Errorw("error in updating helm application for cd pipelineModel", "err", err) + if util.IsErrorContextCancelled(err) { + return false, nil, cdWorkflow.ErrorDeploymentSuperseded + } else if util.IsErrorContextDeadlineExceeded(err) { + return false, nil, context.DeadlineExceeded + } + apiError := clientErrors.ConvertToApiError(err) + if apiError != nil { + return false, nil, apiError + } + return false, nil, err + } else { + impl.logger.Debugw("updated helm application", "response", updateApplicationResponse, "isSuccess", updateApplicationResponse.Success) + } + + } else { + + helmResponse, err := impl.helmInstallReleaseWithCustomChart(newCtx, releaseIdentifier, referenceChartByte, + mergeAndSave, sanitizedK8sVersion, overrideRequest.ForceSyncDeployment) + + // For connection related errors, no need to update the db + if err != nil && strings.Contains(err.Error(), "connection error") { + impl.logger.Errorw("error in helm install custom chart", "err", err) + return false, nil, err + } + + // IMP: update cd pipelineModel to mark deployment app created, even if helm install fails + // If the helm install fails, it still creates the app in failed state, so trying to + // re-create the app results in error from helm that cannot re-use name which is still in use + _, pgErr := impl.updatePipeline(pipelineModel, overrideRequest.UserId) + + if err != nil { + impl.logger.Errorw("error in helm install custom chart", "err", err) + if pgErr != nil { + impl.logger.Errorw("failed to update deployment app created flag in pipelineModel table", "err", err) + } + if util.IsErrorContextCancelled(err) { + return false, nil, cdWorkflow.ErrorDeploymentSuperseded + } else if util.IsErrorContextDeadlineExceeded(err) { + return false, nil, context.DeadlineExceeded + } + apiError := clientErrors.ConvertToApiError(err) + if apiError != nil { + return false, nil, apiError + } + return false, nil, err + } + + if pgErr != nil { + impl.logger.Errorw("failed to update deployment app created flag in pipelineModel table", "err", err) + return false, nil, err + } + + impl.logger.Debugw("received helm release response", "helmResponse", helmResponse, "isSuccess", helmResponse.Success) + } + + //update workflow runner status, used in app workflow view + err = impl.cdWorkflowCommonService.UpdateNonTerminalStatusInRunner(newCtx, overrideRequest.WfrId, overrideRequest.UserId, cdWorkflow.WorkflowInProgress) + if err != nil { + impl.logger.Errorw("error in updating the workflow runner status, createHelmAppForCdPipeline", "err", err) + return false, nil, err + } + } + return true, referenceChartByte, nil +} + +func (impl *HandlerServiceImpl) getHelmHistoryLimitAndChartMetadataForHelmAppCreation(ctx context.Context, + valuesOverrideResponse *app.ValuesOverrideResponse) (*chart.Metadata, int32, *gRPC.ReleaseIdentifier, error) { + pipelineModel := valuesOverrideResponse.Pipeline + envOverride := valuesOverrideResponse.EnvOverride + + var chartMetaData *chart.Metadata + releaseName := pipelineModel.DeploymentAppName + //getting cluster by id + cluster, err := impl.clusterRepository.FindById(envOverride.Environment.ClusterId) + if err != nil { + impl.logger.Errorw("error in getting cluster by id", "clusterId", envOverride.Environment.ClusterId, "err", err) + return nil, 0, nil, err + } else if cluster == nil { + impl.logger.Errorw("error in getting cluster by id, found nil object", "clusterId", envOverride.Environment.ClusterId) + return nil, 0, nil, err + } + + clusterConfig := impl.getClusterGRPCConfig(*cluster) + + releaseIdentifier := &gRPC.ReleaseIdentifier{ + ReleaseName: releaseName, + ReleaseNamespace: envOverride.Namespace, + ClusterConfig: clusterConfig, + } + + var helmRevisionHistory int32 + if valuesOverrideResponse.DeploymentConfig.ReleaseMode == util.PIPELINE_RELEASE_MODE_LINK { + detail, err := impl.helmAppClient.GetReleaseDetails(ctx, releaseIdentifier) + if err != nil { + impl.logger.Errorw("error in fetching release details", "clusterId", clusterConfig.ClusterId, "namespace", envOverride.Namespace, "releaseName", releaseName, "err", err) + return nil, 0, nil, err + } + chartMetaData = &chart.Metadata{ + Name: detail.ChartName, + Version: detail.ChartVersion, + } + //not modifying revision history in case of linked release + helmRevisionHistory = impl.helmAppService.GetRevisionHistoryMaxValue(bean6.SOURCE_LINKED_HELM_APP) + } else { + chartMetaData = &chart.Metadata{ + Name: pipelineModel.App.AppName, + Version: envOverride.Chart.ChartVersion, + } + helmRevisionHistory = impl.helmAppService.GetRevisionHistoryMaxValue(bean6.SOURCE_DEVTRON_APP) + } + + return chartMetaData, helmRevisionHistory, releaseIdentifier, nil +} + +func (impl *HandlerServiceImpl) deployArgoCdApp(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, + valuesOverrideResponse *app.ValuesOverrideResponse) error { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.deployArgoCdApp") + defer span.End() + name, err := impl.createArgoApplicationIfRequired(newCtx, valuesOverrideResponse.EnvOverride, valuesOverrideResponse.Pipeline, valuesOverrideResponse.DeploymentConfig, overrideRequest.UserId) + if err != nil { + impl.logger.Errorw("acd application create error on cd trigger", "err", err, "req", overrideRequest) + return err + } + impl.logger.Debugw("ArgoCd application created", "name", name) + updateAppInArgoCd, err := impl.updateArgoPipeline(newCtx, valuesOverrideResponse.Pipeline, valuesOverrideResponse.EnvOverride, valuesOverrideResponse.DeploymentConfig) + if err != nil { + impl.logger.Errorw("error in updating argocd app ", "err", err) + return err + } + if valuesOverrideResponse.DeploymentConfig.IsArgoAppSyncAndRefreshSupported() { + syncTime := time.Now() + targetRevision := valuesOverrideResponse.DeploymentConfig.GetTargetRevision() + err = impl.argoClientWrapperService.SyncArgoCDApplicationIfNeededAndRefresh(newCtx, valuesOverrideResponse.Pipeline.DeploymentAppName, targetRevision) + if err != nil { + impl.logger.Errorw("error in getting argo application with normal refresh", "argoAppName", valuesOverrideResponse.Pipeline.DeploymentAppName) + return fmt.Errorf("%s. err: %s", bean.ARGOCD_SYNC_ERROR, util.GetClientErrorDetailedMessage(err)) + } + if impl.ACDConfig.IsManualSyncEnabled() { + timeline := &pipelineConfig.PipelineStatusTimeline{ + CdWorkflowRunnerId: overrideRequest.WfrId, + StatusTime: syncTime, + Status: timelineStatus.TIMELINE_STATUS_ARGOCD_SYNC_COMPLETED, + StatusDetail: timelineStatus.TIMELINE_DESCRIPTION_ARGOCD_SYNC_COMPLETED, + } + timeline.CreateAuditLog(overrideRequest.UserId) + _, err = impl.pipelineStatusTimelineService.SaveTimelineIfNotAlreadyPresent(timeline, nil) + if err != nil { + impl.logger.Errorw("error in saving pipeline status timeline", "err", err) + } + } + } + if updateAppInArgoCd { + impl.logger.Debug("argo-cd successfully updated") + } else { + impl.logger.Debug("argo-cd failed to update, ignoring it") + } + return nil +} + +// update repoUrl, revision and argo app sync mode (auto/manual) if needed +func (impl *HandlerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envOverride *bean10.EnvConfigOverride, deploymentConfig *bean9.DeploymentConfig) (bool, error) { + if !deploymentConfig.IsArgoAppPatchSupported() { + impl.logger.Infow("argo app patch not supported", "pipelineId", pipeline.Id, "pipelineName", pipeline.Name) + return false, nil + } + if ctx == nil { + impl.logger.Errorw("err in syncing ACD, ctx is NULL", "pipelineId", pipeline.Id, "pipelineName", pipeline.Name) + return false, nil + } + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.updateArgoPipeline") + defer span.End() + argoAppName := pipeline.DeploymentAppName + impl.logger.Infow("received payload, updateArgoPipeline", "appId", pipeline.AppId, "pipelineName", pipeline.Name, "envId", envOverride.TargetEnvironment, "argoAppName", argoAppName) + argoApplication, err := impl.argoClientWrapperService.GetArgoAppByName(newCtx, argoAppName) + if err != nil { + impl.logger.Errorw("unable to get ArgoCd app", "app", argoAppName, "pipeline", pipeline.Name, "err", err) + return false, err + } + // if status, ok:=status.FromError(err);ok{ + appStatus, _ := status2.FromError(err) + if appStatus.Code() == codes.OK { + impl.logger.Debugw("argo app exists", "app", argoAppName, "pipeline", pipeline.Name) + if impl.argoClientWrapperService.IsArgoAppPatchRequired(argoApplication.Spec.Source, deploymentConfig.GetRepoURL(), deploymentConfig.GetTargetRevision(), deploymentConfig.GetChartLocation()) { + patchRequestDto := &bean7.ArgoCdAppPatchReqDto{ + ArgoAppName: argoAppName, + ChartLocation: deploymentConfig.GetChartLocation(), + GitRepoUrl: deploymentConfig.GetRepoURL(), + TargetRevision: deploymentConfig.GetTargetRevision(), + PatchType: bean7.PatchTypeMerge, + } + url, err := impl.gitOperationService.GetRepoUrlWithUserName(deploymentConfig.GetRepoURL()) + if err != nil { + return false, err + } + patchRequestDto.GitRepoUrl = url + err = impl.argoClientWrapperService.PatchArgoCdApp(newCtx, patchRequestDto) + if err != nil { + impl.logger.Errorw("error in patching argo pipeline", "err", err, "req", patchRequestDto) + return false, err + } + if deploymentConfig.GetRepoURL() != argoApplication.Spec.Source.RepoURL { + impl.logger.Infow("patching argo application's repo url", "argoAppName", argoAppName) + } + if deploymentConfig.GetTargetRevision() != argoApplication.Spec.Source.TargetRevision { + impl.logger.Infow("patching argo application's revision", "argoAppName", argoAppName) + } + impl.logger.Debugw("pipeline update req", "res", patchRequestDto) + } else { + impl.logger.Debug("pipeline no need to update ") + } + err := impl.argoClientWrapperService.UpdateArgoCDSyncModeIfNeeded(newCtx, argoApplication) + if err != nil { + impl.logger.Errorw("error in updating argocd sync mode", "err", err) + return false, err + } + return true, nil + } else if appStatus.Code() == codes.NotFound { + impl.logger.Errorw("argo app not found", "app", argoAppName, "pipeline", pipeline.Name) + return false, nil + } else { + impl.logger.Errorw("err in checking application on argoCD", "err", err, "pipeline", pipeline.Name) + return false, err + } +} + +func (impl *HandlerServiceImpl) createArgoApplicationIfRequired(ctx context.Context, envConfigOverride *bean10.EnvConfigOverride, + pipeline *pipelineConfig.Pipeline, deploymentConfig *bean9.DeploymentConfig, userId int32) (string, error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.createArgoApplicationIfRequired") + defer span.End() + envModel, err := impl.envRepository.FindById(envConfigOverride.TargetEnvironment) + if err != nil { + return "", err + } + argoAppName := pipeline.DeploymentAppName + if !deploymentConfig.IsArgoAppCreationRequired(pipeline.DeploymentAppCreated) { + return argoAppName, nil + } else { + impl.logger.Debugw("new pipeline found", "pipeline", pipeline) + // create + appNamespace := envConfigOverride.Namespace + if appNamespace == "" { + appNamespace = "default" + } + namespace := argocdServer.DevtronInstalationNs + + appRequest := &argocdServer.AppTemplate{ + ApplicationName: argoAppName, + Namespace: namespace, + TargetNamespace: appNamespace, + TargetServer: envModel.Cluster.ServerUrl, + Project: "default", + ValuesFile: helper.GetValuesFileForEnv(envModel.Id), + RepoPath: deploymentConfig.GetChartLocation(), + RepoUrl: deploymentConfig.GetRepoURL(), + AutoSyncEnabled: impl.ACDConfig.ArgoCDAutoSyncEnabled, + } + appRequest.RepoUrl, err = impl.gitOperationService.GetRepoUrlWithUserName(appRequest.RepoUrl) + if err != nil { + return "", err + } + createdArgoAppName, err := impl.argoK8sClient.CreateAcdApp(newCtx, appRequest, argocdServer.ARGOCD_APPLICATION_TEMPLATE) + if err != nil { + return "", err + } + // update cd pipeline to mark deployment app created + _, err = impl.updatePipeline(pipeline, userId) + if err != nil { + impl.logger.Errorw("error in update cd pipeline for deployment app created or not", "err", err) + return "", err + } + return createdArgoAppName, nil + } +} + +func (impl *HandlerServiceImpl) updatePipeline(pipeline *pipelineConfig.Pipeline, userId int32) (bool, error) { + err := impl.pipelineRepository.SetDeploymentAppCreatedInPipeline(true, pipeline.Id, userId) + if err != nil { + impl.logger.Errorw("error on updating cd pipeline for setting deployment app created", "err", err) + return false, err + } + return true, nil +} + +// helmInstallReleaseWithCustomChart performs helm install with custom chart +func (impl *HandlerServiceImpl) helmInstallReleaseWithCustomChart(ctx context.Context, releaseIdentifier *gRPC.ReleaseIdentifier, + referenceChartByte []byte, valuesYaml, k8sServerVersion string, forceSync bool) (*gRPC.HelmInstallCustomResponse, error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.helmInstallReleaseWithCustomChart") + defer span.End() + helmInstallRequest := gRPC.HelmInstallCustomRequest{ + ValuesYaml: valuesYaml, + ChartContent: &gRPC.ChartContent{Content: referenceChartByte}, + ReleaseIdentifier: releaseIdentifier, + } + if len(k8sServerVersion) > 0 { + helmInstallRequest.K8SVersion = k8sServerVersion + } + if impl.isDevtronAsyncHelmInstallModeEnabled(forceSync) { + helmInstallRequest.RunInCtx = true + } + // Request exec + return impl.helmAppClient.InstallReleaseWithCustomChart(newCtx, &helmInstallRequest) +} + +func (impl *HandlerServiceImpl) markImageScanDeployed(ctx context.Context, appId, envId, clusterId int, + imageDigest string, isScanEnabled bool, image string) error { + _, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.markImageScanDeployed") + defer span.End() + // TODO KB: send NATS event for self consumption + impl.logger.Debugw("mark image scan deployed for devtron app, from cd auto or manual trigger", "imageDigest", imageDigest) + executionHistory, err := impl.imageScanHistoryReadService.FindByImageAndDigest(imageDigest, image) + if err != nil && !errors.Is(err, pg.ErrNoRows) { + impl.logger.Errorw("error in fetching execution history", "err", err) + return err + } + if errors.Is(err, pg.ErrNoRows) || executionHistory == nil || executionHistory.Id == 0 { + if isScanEnabled { + // There should ImageScanHistory for ScanEnabled artifacts + impl.logger.Errorw("no execution history found for digest", "digest", imageDigest) + return fmt.Errorf("no execution history found for digest - %s", imageDigest) + } else { + // For ScanDisabled artifacts it should be an expected condition + impl.logger.Infow("no execution history found for digest", "digest", imageDigest) + return nil + } + } + impl.logger.Debugw("saving image_scan_deploy_info for cd auto or manual trigger", "executionHistory", executionHistory) + var ids []int + ids = append(ids, executionHistory.Id) + + ot, err := impl.imageScanDeployInfoReadService.FetchByAppIdAndEnvId(appId, envId, []string{repository6.ScanObjectType_APP}) + + if err == pg.ErrNoRows && !isScanEnabled { + // ignoring if no rows are found and scan is disabled + return nil + } + + if err != nil && err != pg.ErrNoRows { + return err + } else if err == pg.ErrNoRows && isScanEnabled { + imageScanDeployInfo := &repository6.ImageScanDeployInfo{ + ImageScanExecutionHistoryId: ids, + ScanObjectMetaId: appId, + ObjectType: repository6.ScanObjectType_APP, + EnvId: envId, + ClusterId: clusterId, + AuditLog: sql.AuditLog{ + CreatedOn: time.Now(), + CreatedBy: 1, + UpdatedOn: time.Now(), + UpdatedBy: 1, + }, + } + impl.logger.Debugw("mark image scan deployed for normal app, from cd auto or manual trigger", "imageScanDeployInfo", imageScanDeployInfo) + err = impl.imageScanDeployInfoService.Save(imageScanDeployInfo) + if err != nil { + impl.logger.Errorw("error in creating deploy info", "err", err) + } + } else { + // Updating Execution history for Latest Deployment to fetch out security Vulnerabilities for latest deployed info + if isScanEnabled { + ot.ImageScanExecutionHistoryId = ids + } else { + arr := []int{-1} + ot.ImageScanExecutionHistoryId = arr + } + err = impl.imageScanDeployInfoService.Update(ot) + if err != nil { + impl.logger.Errorw("error in updating deploy info for latest deployed image", "err", err) + } + } + return err +} + +func (impl *HandlerServiceImpl) isDevtronAsyncHelmInstallModeEnabled(forceSync bool) bool { + return impl.globalEnvVariables.EnableAsyncHelmInstallDevtronChart && !forceSync +} + +func (impl *HandlerServiceImpl) isDevtronAsyncInstallModeEnabled(overrideRequest *bean3.ValuesOverrideRequest) (bool, error) { + if util.IsHelmApp(overrideRequest.DeploymentAppType) { + return impl.isDevtronAsyncHelmInstallModeEnabled(overrideRequest.ForceSyncDeployment), nil + } else if util.IsAcdApp(overrideRequest.DeploymentAppType) { + return impl.isDevtronAsyncArgoCdInstallModeEnabledForApp(overrideRequest.AppId, + overrideRequest.EnvId, overrideRequest.ForceSyncDeployment) + } + return false, nil +} + +func (impl *HandlerServiceImpl) deleteCorruptedPipelineStage(pipelineStage *repository.PipelineStage, triggeredBy int32) (error, bool) { + if pipelineStage != nil { + stageReq := &bean8.PipelineStageDto{ + Id: pipelineStage.Id, + Type: pipelineStage.Type, + } + err, deleted := impl.pipelineStageService.DeletePipelineStageIfReq(stageReq, triggeredBy) + if err != nil { + impl.logger.Errorw("error in deleting the corrupted pipeline stage", "err", err, "pipelineStageReq", stageReq) + return err, false + } + return nil, deleted + } + return nil, false +} + +func (impl *HandlerServiceImpl) handleCustomGitOpsRepoValidation(runner *pipelineConfig.CdWorkflowRunner, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredBy int32) error { + if !util.IsAcdApp(pipeline.DeploymentAppName) { + return nil + } + isGitOpsConfigured := false + gitOpsConfig, err := impl.gitOpsConfigReadService.GetGitOpsConfigActive() + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error while getting active GitOpsConfig", "err", err) + } + if gitOpsConfig != nil && gitOpsConfig.Id > 0 { + isGitOpsConfigured = true + } + if isGitOpsConfigured && gitOpsConfig.AllowCustomRepository { + //chart, err := impl.chartRepository.FindLatestChartForAppByAppId(pipeline.AppId) + //if err != nil { + // impl.logger.Errorw("error in fetching latest chart for app by appId", "err", err, "appId", pipeline.AppId) + // return err + //} + if gitOps.IsGitOpsRepoNotConfigured(envDeploymentConfig.GetRepoURL()) { + if err = impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, errors.New(cdWorkflow.GITOPS_REPO_NOT_CONFIGURED), triggeredBy); err != nil { + impl.logger.Errorw("error while updating current runner status to failed, TriggerDeployment", "wfrId", runner.Id, "err", err) + } + apiErr := &util.ApiError{ + HttpStatusCode: http.StatusConflict, + UserMessage: cdWorkflow.GITOPS_REPO_NOT_CONFIGURED, + InternalMessage: cdWorkflow.GITOPS_REPO_NOT_CONFIGURED, + } + return apiErr + } + } + return nil +} + +func (impl *HandlerServiceImpl) getSanitizedK8sVersion(referenceTemplate string) (string, error) { + var sanitizedK8sVersion string + //handle specific case for all cronjob charts from cronjob-chart_1-2-0 to cronjob-chart_1-5-0 where semverCompare + //comparison func has wrong api version mentioned, so for already installed charts via these charts that comparison + //is always false, handles the gh issue:- https://github.com/devtron-labs/devtron/issues/4860 + cronJobChartRegex := regexp.MustCompile(bean.CronJobChartRegexExpression) + if cronJobChartRegex.MatchString(referenceTemplate) { + k8sServerVersion, err := impl.K8sUtil.GetKubeVersion() + if err != nil { + impl.logger.Errorw("exception caught in getting k8sServerVersion", "err", err) + return "", err + } + sanitizedK8sVersion = k8s2.StripPrereleaseFromK8sVersion(k8sServerVersion.String()) + } + return sanitizedK8sVersion, nil +} + +func (impl *HandlerServiceImpl) getReferenceChartByteForHelmTypeApp(envOverride *bean10.EnvConfigOverride, + chartMetaData *chart.Metadata, referenceTemplatePath string, overrideRequest *bean3.ValuesOverrideRequest, + valuesOverrideResponse *app.ValuesOverrideResponse) ([]byte, error) { + referenceChartByte := envOverride.Chart.ReferenceChart + // here updating reference chart into database. + if len(envOverride.Chart.ReferenceChart) == 0 { + refChartByte, err := impl.chartTemplateService.GetByteArrayRefChart(chartMetaData, referenceTemplatePath) + if err != nil { + impl.logger.Errorw("ref chart commit error on cd trigger", "err", err, "req", overrideRequest) + return nil, err + } + ch := envOverride.Chart + ch.ReferenceChart = refChartByte + ch.UpdatedOn = time.Now() + ch.UpdatedBy = overrideRequest.UserId + err = impl.chartRepository.Update(ch) + if err != nil { + impl.logger.Errorw("chart update error", "err", err, "req", overrideRequest) + return nil, err + } + referenceChartByte = refChartByte + } + var err error + referenceChartByte, err = impl.overrideReferenceChartByteForHelmTypeApp(valuesOverrideResponse, chartMetaData, referenceTemplatePath, referenceChartByte) + if err != nil { + impl.logger.Errorw("ref chart commit error on cd trigger", "err", err, "req", overrideRequest) + return nil, err + } + return referenceChartByte, nil +} From 0ac8f1fae65bbdd4b8a361cdc35f8f0ab3db1759 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 8 Apr 2025 13:05:07 +0530 Subject: [PATCH 17/17] file renamed --- .../{preStageHandlerService.go => preStageHandlerCode.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pkg/deployment/trigger/devtronApps/{preStageHandlerService.go => preStageHandlerCode.go} (100%) diff --git a/pkg/deployment/trigger/devtronApps/preStageHandlerService.go b/pkg/deployment/trigger/devtronApps/preStageHandlerCode.go similarity index 100% rename from pkg/deployment/trigger/devtronApps/preStageHandlerService.go rename to pkg/deployment/trigger/devtronApps/preStageHandlerCode.go