diff --git a/App.go b/App.go index 08b240a8b7..649c319298 100644 --- a/App.go +++ b/App.go @@ -23,6 +23,7 @@ import ( "fmt" "github.com/devtron-labs/common-lib/middlewares" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" + posthogTelemetry "github.com/devtron-labs/common-lib/telemetry" "github.com/devtron-labs/devtron/pkg/eventProcessor" "github.com/devtron-labs/devtron/pkg/eventProcessor/in" "log" @@ -31,7 +32,6 @@ import ( "time" "github.com/devtron-labs/devtron/api/util" - "github.com/devtron-labs/devtron/client/telemetry" "github.com/devtron-labs/devtron/otel" "github.com/devtron-labs/devtron/pkg/auth/user" @@ -55,7 +55,7 @@ type App struct { EnforcerV2 *casbinv2.SyncedEnforcer server *http.Server db *pg.DB - posthogClient *telemetry.PosthogClient + posthogClient *posthogTelemetry.PosthogClient // eventProcessor.CentralEventProcessor is used to register event processors centralEventProcessor *eventProcessor.CentralEventProcessor // do not remove this. // used for local dev only @@ -73,7 +73,7 @@ func NewApp(router *router.MuxRouter, enforcer *casbin.SyncedEnforcer, db *pg.DB, sessionManager2 *authMiddleware.SessionManager, - posthogClient *telemetry.PosthogClient, + posthogClient *posthogTelemetry.PosthogClient, loggingMiddleware util.LoggingMiddleware, centralEventProcessor *eventProcessor.CentralEventProcessor, pubSubClient *pubsub.PubSubClientServiceImpl, diff --git a/Wire.go b/Wire.go index ebfb6a943d..f239eb7d06 100644 --- a/Wire.go +++ b/Wire.go @@ -22,7 +22,8 @@ package main import ( "github.com/devtron-labs/authenticator/middleware" cloudProviderIdentifier "github.com/devtron-labs/common-lib/cloud-provider-identifier" - pubsub1 "github.com/devtron-labs/common-lib/pubsub-lib" + pubSub "github.com/devtron-labs/common-lib/pubsub-lib" + posthogTelemetry "github.com/devtron-labs/common-lib/telemetry" util4 "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/api/apiToken" appStoreRestHandler "github.com/devtron-labs/devtron/api/appStore" @@ -144,6 +145,7 @@ import ( "github.com/devtron-labs/devtron/pkg/deploymentGroup" "github.com/devtron-labs/devtron/pkg/dockerRegistry" "github.com/devtron-labs/devtron/pkg/eventProcessor" + "github.com/devtron-labs/devtron/pkg/executor" "github.com/devtron-labs/devtron/pkg/generateManifest" "github.com/devtron-labs/devtron/pkg/gitops" "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" @@ -165,6 +167,7 @@ import ( resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" "github.com/devtron-labs/devtron/pkg/sql" + "github.com/devtron-labs/devtron/pkg/ucid" util3 "github.com/devtron-labs/devtron/pkg/util" "github.com/devtron-labs/devtron/pkg/variables" "github.com/devtron-labs/devtron/pkg/variables/parsers" @@ -219,7 +222,7 @@ func InitializeApp() (*App, error) { userResource.UserResourceWireSet, policyGovernance.PolicyGovernanceWireSet, resourceScan.ScanningResultWireSet, - + executor.ExecutorWireSet, // -------wireset end ---------- // ------- gitSensor.GetConfig, @@ -478,9 +481,6 @@ func InitializeApp() (*App, error) { util.IntValidator, types.GetCiCdConfig, - pipeline.NewWorkflowServiceImpl, - wire.Bind(new(pipeline.WorkflowService), new(*pipeline.WorkflowServiceImpl)), - pipeline.NewCiServiceImpl, wire.Bind(new(pipeline.CiService), new(*pipeline.CiServiceImpl)), @@ -501,7 +501,7 @@ func InitializeApp() (*App, error) { pipeline.NewCiLogServiceImpl, wire.Bind(new(pipeline.CiLogService), new(*pipeline.CiLogServiceImpl)), - pubsub1.NewPubSubClientServiceImpl, + pubSub.NewPubSubClientServiceImpl, rbac.NewEnforcerUtilImpl, wire.Bind(new(rbac.EnforcerUtil), new(*rbac.EnforcerUtilImpl)), @@ -696,7 +696,8 @@ func InitializeApp() (*App, error) { wire.Bind(new(router.TelemetryRouter), new(*router.TelemetryRouterImpl)), restHandler.NewTelemetryRestHandlerImpl, wire.Bind(new(restHandler.TelemetryRestHandler), new(*restHandler.TelemetryRestHandlerImpl)), - telemetry.NewPosthogClient, + posthogTelemetry.NewPosthogClient, + ucid.WireSet, cloudProviderIdentifier.NewProviderIdentifierServiceImpl, wire.Bind(new(cloudProviderIdentifier.ProviderIdentifierService), new(*cloudProviderIdentifier.ProviderIdentifierServiceImpl)), diff --git a/api/argoApplication/wire_argoApplication.go b/api/argoApplication/wire_argoApplication.go index e3c7381725..45ef8139db 100644 --- a/api/argoApplication/wire_argoApplication.go +++ b/api/argoApplication/wire_argoApplication.go @@ -17,7 +17,7 @@ package argoApplication import ( - "github.com/devtron-labs/devtron/pkg/argoApplication" + argoApplication3 "github.com/devtron-labs/devtron/pkg/argoApplication" "github.com/devtron-labs/devtron/pkg/argoApplication/read" "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" "github.com/google/wire" @@ -30,9 +30,9 @@ var ArgoApplicationWireSetFull = wire.NewSet( config.NewArgoApplicationConfigServiceImpl, wire.Bind(new(config.ArgoApplicationConfigService), new(*config.ArgoApplicationConfigServiceImpl)), - argoApplication.NewArgoApplicationServiceImpl, - argoApplication.NewArgoApplicationServiceExtendedServiceImpl, - wire.Bind(new(argoApplication.ArgoApplicationService), new(*argoApplication.ArgoApplicationServiceExtendedImpl)), + argoApplication3.NewArgoApplicationServiceImpl, + argoApplication3.NewArgoApplicationServiceExtendedServiceImpl, + wire.Bind(new(argoApplication3.ArgoApplicationService), new(*argoApplication3.ArgoApplicationServiceExtendedImpl)), NewArgoApplicationRestHandlerImpl, wire.Bind(new(ArgoApplicationRestHandler), new(*ArgoApplicationRestHandlerImpl)), @@ -48,8 +48,8 @@ var ArgoApplicationWireSetEA = wire.NewSet( config.NewArgoApplicationConfigServiceImpl, wire.Bind(new(config.ArgoApplicationConfigService), new(*config.ArgoApplicationConfigServiceImpl)), - argoApplication.NewArgoApplicationServiceImpl, - wire.Bind(new(argoApplication.ArgoApplicationService), new(*argoApplication.ArgoApplicationServiceImpl)), + argoApplication3.NewArgoApplicationServiceImpl, + wire.Bind(new(argoApplication3.ArgoApplicationService), new(*argoApplication3.ArgoApplicationServiceImpl)), NewArgoApplicationRestHandlerImpl, wire.Bind(new(ArgoApplicationRestHandler), new(*ArgoApplicationRestHandlerImpl)), diff --git a/api/restHandler/CoreAppRestHandler.go b/api/restHandler/CoreAppRestHandler.go index eff6fba1c0..277405a08b 100644 --- a/api/restHandler/CoreAppRestHandler.go +++ b/api/restHandler/CoreAppRestHandler.go @@ -29,6 +29,7 @@ import ( "github.com/devtron-labs/devtron/pkg/build/git/gitProvider" "github.com/devtron-labs/devtron/pkg/build/git/gitProvider/read" pipelineBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + common2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" bean3 "github.com/devtron-labs/devtron/pkg/chart/bean" read5 "github.com/devtron-labs/devtron/pkg/chart/read" "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" @@ -1688,7 +1689,7 @@ func (handler CoreAppRestHandlerImpl) createCiPipeline(appId int, userId int32, ParentCiPipeline: ciPipelineData.ParentCiPipeline, ParentAppId: ciPipelineData.ParentAppId, LinkedCount: ciPipelineData.LinkedCount, - PipelineType: pipelineBean.PipelineType(ciPipelineData.PipelineType), + PipelineType: common2.PipelineType(ciPipelineData.PipelineType), }, } diff --git a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go index 0bb2a921b8..13b61730e3 100644 --- a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go @@ -25,6 +25,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/constants" "github.com/devtron-labs/devtron/pkg/build/artifacts/imageTagging" bean2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" constants2 "github.com/devtron-labs/devtron/pkg/pipeline/constants" "github.com/devtron-labs/devtron/util/stringsUtil" "golang.org/x/exp/maps" @@ -37,7 +38,6 @@ import ( "github.com/devtron-labs/devtron/util/response/pagination" "github.com/gorilla/schema" - "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/devtron-labs/devtron/api/restHandler/common" "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/sql/repository" @@ -701,8 +701,7 @@ func (handler *PipelineConfigRestHandlerImpl) TriggerCiPipeline(w http.ResponseW handler.Logger.Infow("request payload, TriggerCiPipeline", "payload", ciTriggerRequest) response := make(map[string]string) - resp, err := handler.ciHandler.HandleCIManual(ciTriggerRequest) - + resp, err := handler.ciHandlerService.HandleCIManual(ciTriggerRequest) if errors.Is(err, bean1.ErrImagePathInUse) { handler.Logger.Errorw("service err duplicate image tag, TriggerCiPipeline", "err", err, "payload", ciTriggerRequest) common.WriteJsonResp(w, err, err, http.StatusConflict) @@ -918,7 +917,8 @@ func (handler *PipelineConfigRestHandlerImpl) DownloadCiWorkflowArtifacts(w http return } - file, err := handler.ciHandler.DownloadCiWorkflowArtifacts(pipelineId, buildId) + file, err := handler.ciHandlerService.DownloadCiWorkflowArtifacts(pipelineId, buildId) + defer file.Close() if err != nil { handler.Logger.Errorw("service err, DownloadCiWorkflowArtifacts", "err", err, "pipelineId", pipelineId, "buildId", buildId) if util.IsErrNoRows(err) { @@ -974,8 +974,8 @@ func (handler *PipelineConfigRestHandlerImpl) GetHistoricBuildLogs(w http.Respon common.WriteJsonResp(w, nil, "Unauthorized User", http.StatusForbidden) return } - - resp, err := handler.ciHandler.GetHistoricBuildLogs(workflowId, nil) + // RBAC + resp, err := handler.ciHandlerService.GetHistoricBuildLogs(workflowId, nil) if err != nil { handler.Logger.Errorw("service err, GetHistoricBuildLogs", "err", err, "pipelineId", pipelineId, "workflowId", workflowId) common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) @@ -1114,7 +1114,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetBuildLogs(w http.ResponseWriter return } } - logsReader, cleanUp, err := handler.ciHandler.GetRunningWorkflowLogs(workflowId) + logsReader, cleanUp, err := handler.ciHandlerService.GetRunningWorkflowLogs(workflowId) if err != nil { handler.Logger.Errorw("service err, GetBuildLogs", "err", err, "pipelineId", pipelineId, "workflowId", workflowId, "lastEventId", lastEventId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) @@ -1446,7 +1446,7 @@ func (handler *PipelineConfigRestHandlerImpl) DeleteMaterial(w http.ResponseWrit func (handler *PipelineConfigRestHandlerImpl) HandleWorkflowWebhook(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) - var wfUpdateReq v1alpha1.WorkflowStatus + var wfUpdateReq eventProcessorBean.CiCdStatus err := decoder.Decode(&wfUpdateReq) if err != nil { handler.Logger.Errorw("request err, HandleWorkflowWebhook", "err", err, "payload", wfUpdateReq) @@ -1454,7 +1454,7 @@ func (handler *PipelineConfigRestHandlerImpl) HandleWorkflowWebhook(w http.Respo return } handler.Logger.Infow("request payload, HandleWorkflowWebhook", "payload", wfUpdateReq) - resp, err := handler.ciHandler.UpdateWorkflow(wfUpdateReq) + resp, _, err := handler.ciHandler.UpdateWorkflow(wfUpdateReq) if err != nil { handler.Logger.Errorw("service err, HandleWorkflowWebhook", "err", err, "payload", wfUpdateReq) common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) @@ -1559,7 +1559,7 @@ func (handler *PipelineConfigRestHandlerImpl) CancelWorkflow(w http.ResponseWrit //RBAC - resp, err := handler.ciHandler.CancelBuild(workflowId, forceAbort) + resp, err := handler.ciHandlerService.CancelBuild(workflowId, forceAbort) if err != nil { handler.Logger.Errorw("service err, CancelWorkflow", "err", err, "workflowId", workflowId, "pipelineId", pipelineId) if util.IsErrNoRows(err) { diff --git a/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go index 8437aa7ec4..67e78a7ccf 100644 --- a/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go @@ -1620,7 +1620,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetPrePostDeploymentLogs(w http.Re } // RBAC CHECK - logsReader, cleanUp, err := handler.cdHandler.GetRunningWorkflowLogs(environmentId, pipelineId, workflowId) + logsReader, cleanUp, err := handler.cdHandlerService.GetRunningWorkflowLogs(environmentId, pipelineId, workflowId) if err != nil { handler.Logger.Errorw("service err, GetPrePostDeploymentLogs", "err", err, "appId", appId, "environmentId", environmentId, "pipelineId", pipelineId, "workflowId", workflowId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) @@ -1745,7 +1745,7 @@ func (handler *PipelineConfigRestHandlerImpl) DownloadArtifacts(w http.ResponseW } // RBAC CHECK - file, err := handler.cdHandler.DownloadCdWorkflowArtifacts(buildId) + file, err := handler.cdHandlerService.DownloadCdWorkflowArtifacts(buildId) defer file.Close() if err != nil { @@ -1929,7 +1929,7 @@ func (handler *PipelineConfigRestHandlerImpl) CancelStage(w http.ResponseWriter, } // RBAC - resp, err := handler.cdHandler.CancelStage(workflowRunnerId, forceAbort, userId) + resp, err := handler.cdHandlerService.CancelStage(workflowRunnerId, forceAbort, userId) if err != nil { handler.Logger.Errorw("service err, CancelStage", "err", err, "pipelineId", pipelineId, "workflowRunnerId", workflowRunnerId) if util.IsErrNoRows(err) { diff --git a/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go b/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go index 4d37e03579..74ecde9387 100644 --- a/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go +++ b/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go @@ -26,12 +26,14 @@ import ( read2 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/read" gitProviderRead "github.com/devtron-labs/devtron/pkg/build/git/gitProvider/read" bean3 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/build/trigger" "github.com/devtron-labs/devtron/pkg/chart/gitOpsConfig" read5 "github.com/devtron-labs/devtron/pkg/chart/read" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" validator2 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/validator" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" "github.com/devtron-labs/devtron/pkg/pipeline/draftAwareConfigService" security2 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/read" @@ -140,6 +142,8 @@ type PipelineConfigRestHandlerImpl struct { environmentRepository repository2.EnvironmentRepository chartReadService read5.ChartReadService draftAwareResourceService draftAwareConfigService.DraftAwareConfigService + ciHandlerService trigger.HandlerService + cdHandlerService devtronApps.HandlerService } func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger *zap.SugaredLogger, @@ -175,6 +179,8 @@ func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger EnvironmentRepository repository2.EnvironmentRepository, chartReadService read5.ChartReadService, draftAwareResourceService draftAwareConfigService.DraftAwareConfigService, + ciHandlerService trigger.HandlerService, + cdHandlerService devtronApps.HandlerService, ) *PipelineConfigRestHandlerImpl { envConfig := &PipelineRestHandlerEnvConfig{} err := env.Parse(envConfig) @@ -218,6 +224,8 @@ func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger environmentRepository: EnvironmentRepository, chartReadService: chartReadService, draftAwareResourceService: draftAwareResourceService, + ciHandlerService: ciHandlerService, + cdHandlerService: cdHandlerService, } } diff --git a/api/restHandler/app/pipeline/trigger/PipelineTriggerRestHandler.go b/api/restHandler/app/pipeline/trigger/PipelineTriggerRestHandler.go index 08b43625b1..8fab862852 100644 --- a/api/restHandler/app/pipeline/trigger/PipelineTriggerRestHandler.go +++ b/api/restHandler/app/pipeline/trigger/PipelineTriggerRestHandler.go @@ -66,7 +66,7 @@ type PipelineTriggerRestHandlerImpl struct { deploymentGroupService deploymentGroup.DeploymentGroupService deploymentConfigService pipeline.PipelineDeploymentConfigService deployedAppService deployedApp.DeployedAppService - cdTriggerService devtronApps.TriggerService + cdHandlerService devtronApps.HandlerService workflowEventPublishService out.WorkflowEventPublishService } @@ -75,7 +75,7 @@ func NewPipelineRestHandler(appService app.AppService, userAuthService user.User deploymentGroupService deploymentGroup.DeploymentGroupService, deploymentConfigService pipeline.PipelineDeploymentConfigService, deployedAppService deployedApp.DeployedAppService, - cdTriggerService devtronApps.TriggerService, + cdHandlerService devtronApps.HandlerService, workflowEventPublishService out.WorkflowEventPublishService) *PipelineTriggerRestHandlerImpl { pipelineHandler := &PipelineTriggerRestHandlerImpl{ appService: appService, @@ -88,7 +88,7 @@ func NewPipelineRestHandler(appService app.AppService, userAuthService user.User deploymentGroupService: deploymentGroupService, deploymentConfigService: deploymentConfigService, deployedAppService: deployedAppService, - cdTriggerService: cdTriggerService, + cdHandlerService: cdHandlerService, workflowEventPublishService: workflowEventPublishService, } return pipelineHandler @@ -140,7 +140,7 @@ func (handler PipelineTriggerRestHandlerImpl) OverrideConfig(w http.ResponseWrit triggerContext := bean3.TriggerContext{ Context: ctx, } - mergeResp, helmPackageName, _, err := handler.cdTriggerService.ManualCdTrigger(triggerContext, &overrideRequest) + mergeResp, helmPackageName, _, err := handler.cdHandlerService.ManualCdTrigger(triggerContext, &overrideRequest) span.End() if err != nil { handler.logger.Errorw("request err, OverrideConfig", "err", err, "payload", overrideRequest) diff --git a/client/argocdServer/config/Config.go b/client/argocdServer/config/Config.go index 20b622c38e..2027a2257f 100644 --- a/client/argocdServer/config/Config.go +++ b/client/argocdServer/config/Config.go @@ -71,7 +71,7 @@ func (impl *ArgoCDConfigGetterImpl) GetGRPCConfig() (*bean.ArgoGRPCConfig, error } func (impl *ArgoCDConfigGetterImpl) GetK8sConfig() (*bean.ArgoK8sConfig, error) { - clusterBean, err := impl.clusterReadService.FindOne(bean2.DEFAULT_CLUSTER) + clusterBean, err := impl.clusterReadService.FindOne(bean2.DefaultCluster) if err != nil { impl.logger.Errorw("error in fetching cluster bean from db", "err", err) return nil, err diff --git a/client/cron/CiStatusUpdateCron.go b/client/cron/CiStatusUpdateCron.go index a62d4bc03f..4dbdd7ece5 100644 --- a/client/cron/CiStatusUpdateCron.go +++ b/client/cron/CiStatusUpdateCron.go @@ -21,7 +21,7 @@ import ( "github.com/caarlos0/env" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/app" - "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/workflow/dag" cron2 "github.com/devtron-labs/devtron/util/cron" "github.com/robfig/cron/v3" "go.uber.org/zap" @@ -38,12 +38,13 @@ type CiStatusUpdateCronImpl struct { appService app.AppService ciWorkflowStatusUpdateConfig *CiWorkflowStatusUpdateConfig ciPipelineRepository pipelineConfig.CiPipelineRepository - ciHandler pipeline.CiHandler + workflowDagExecutor dag.WorkflowDagExecutor } func NewCiStatusUpdateCronImpl(logger *zap.SugaredLogger, appService app.AppService, ciWorkflowStatusUpdateConfig *CiWorkflowStatusUpdateConfig, ciPipelineRepository pipelineConfig.CiPipelineRepository, - ciHandler pipeline.CiHandler, cronLogger *cron2.CronLoggerImpl) *CiStatusUpdateCronImpl { + cronLogger *cron2.CronLoggerImpl, + workflowDagExecutor dag.WorkflowDagExecutor) *CiStatusUpdateCronImpl { cron := cron.New( cron.WithChain(cron.Recover(cronLogger))) cron.Start() @@ -53,7 +54,7 @@ func NewCiStatusUpdateCronImpl(logger *zap.SugaredLogger, appService app.AppServ appService: appService, ciWorkflowStatusUpdateConfig: ciWorkflowStatusUpdateConfig, ciPipelineRepository: ciPipelineRepository, - ciHandler: ciHandler, + workflowDagExecutor: workflowDagExecutor, } // execute periodically, update ci workflow status for failed process @@ -87,7 +88,7 @@ func (impl *CiStatusUpdateCronImpl) UpdateCiWorkflowStatusFailedCron() { impl.logger.Errorw("error in converting string to int", "err", err) return } - err = impl.ciHandler.UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild) + err = impl.workflowDagExecutor.UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild) if err != nil { impl.logger.Errorw("error in updating ci workflow status for failed workflows", "err", err) return diff --git a/client/cron/CiTriggerCron.go b/client/cron/CiTriggerCron.go index 147015a840..ddc84da2c7 100644 --- a/client/cron/CiTriggerCron.go +++ b/client/cron/CiTriggerCron.go @@ -22,8 +22,8 @@ import ( repository2 "github.com/devtron-labs/devtron/internal/sql/repository" bean2 "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/bean" - pipelineConfigBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" - "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" + "github.com/devtron-labs/devtron/pkg/build/trigger" "github.com/devtron-labs/devtron/pkg/pipeline/repository" repository3 "github.com/devtron-labs/devtron/pkg/plugin/repository" cron2 "github.com/devtron-labs/devtron/util/cron" @@ -40,13 +40,14 @@ type CiTriggerCronImpl struct { cron *cron.Cron cfg *CiTriggerCronConfig pipelineStageRepository repository.PipelineStageRepository - ciHandler pipeline.CiHandler ciArtifactRepository repository2.CiArtifactRepository globalPluginRepository repository3.GlobalPluginRepository + ciHandlerService trigger.HandlerService } func NewCiTriggerCronImpl(logger *zap.SugaredLogger, cfg *CiTriggerCronConfig, pipelineStageRepository repository.PipelineStageRepository, - ciHandler pipeline.CiHandler, ciArtifactRepository repository2.CiArtifactRepository, globalPluginRepository repository3.GlobalPluginRepository, cronLogger *cron2.CronLoggerImpl) *CiTriggerCronImpl { + ciArtifactRepository repository2.CiArtifactRepository, globalPluginRepository repository3.GlobalPluginRepository, cronLogger *cron2.CronLoggerImpl, + ciHandlerService trigger.HandlerService) *CiTriggerCronImpl { cron := cron.New( cron.WithChain(cron.Recover(cronLogger))) cron.Start() @@ -54,10 +55,10 @@ func NewCiTriggerCronImpl(logger *zap.SugaredLogger, cfg *CiTriggerCronConfig, p logger: logger, cron: cron, pipelineStageRepository: pipelineStageRepository, - ciHandler: ciHandler, cfg: cfg, ciArtifactRepository: ciArtifactRepository, globalPluginRepository: globalPluginRepository, + ciHandlerService: ciHandlerService, } _, err := cron.AddFunc(fmt.Sprintf("@every %dm", cfg.SourceControllerCronTime), impl.TriggerCiCron) @@ -101,9 +102,9 @@ func (impl *CiTriggerCronImpl) TriggerCiCron() { CiPipelineMaterial: ciPipelineMaterials, TriggeredBy: bean2.SYSTEM_USER_ID, InvalidateCache: false, - PipelineType: string(pipelineConfigBean.CI_JOB), + PipelineType: string(common.CI_JOB), } - _, err = impl.ciHandler.HandleCIManual(ciTriggerRequest) + _, err = impl.ciHandlerService.HandleCIManual(ciTriggerRequest) if err != nil { return } diff --git a/client/events/EventBuilder.go b/client/events/EventBuilder.go index 83f7186811..4464d7aa98 100644 --- a/client/events/EventBuilder.go +++ b/client/events/EventBuilder.go @@ -19,6 +19,7 @@ package client import ( "context" "fmt" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" repository4 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "strings" "time" @@ -38,7 +39,7 @@ import ( type EventFactory interface { Build(eventType util.EventType, sourceId *int, appId int, envId *int, pipelineType util.PipelineType) (Event, error) BuildExtraCDData(event Event, wfr *pipelineConfig.CdWorkflowRunner, pipelineOverrideId int, stage bean2.WorkflowType) Event - BuildExtraCIData(event Event, material *MaterialTriggerInfo) Event + BuildExtraCIData(event Event, material *buildBean.MaterialTriggerInfo) Event //BuildFinalData(event Event) *Payload } @@ -173,7 +174,7 @@ func (impl *EventSimpleFactoryImpl) BuildExtraCDData(event Event, wfr *pipelineC return event } -func (impl *EventSimpleFactoryImpl) BuildExtraCIData(event Event, material *MaterialTriggerInfo) Event { +func (impl *EventSimpleFactoryImpl) BuildExtraCIData(event Event, material *buildBean.MaterialTriggerInfo) Event { if material == nil { materialInfo, err := impl.getCiMaterialInfo(event.PipelineId, event.CiArtifactId) if err != nil { @@ -207,8 +208,8 @@ func (impl *EventSimpleFactoryImpl) BuildExtraCIData(event Event, material *Mate return event } -func (impl *EventSimpleFactoryImpl) getCiMaterialInfo(ciPipelineId int, ciArtifactId int) (*MaterialTriggerInfo, error) { - materialTriggerInfo := &MaterialTriggerInfo{} +func (impl *EventSimpleFactoryImpl) getCiMaterialInfo(ciPipelineId int, ciArtifactId int) (*buildBean.MaterialTriggerInfo, error) { + materialTriggerInfo := &buildBean.MaterialTriggerInfo{} if ciPipelineId > 0 { ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(ciPipelineId) if err != nil { @@ -216,13 +217,13 @@ func (impl *EventSimpleFactoryImpl) getCiMaterialInfo(ciPipelineId int, ciArtifa return nil, err } - var ciMaterialsArr []CiPipelineMaterialResponse + var ciMaterialsArr []buildBean.CiPipelineMaterialResponse for _, m := range ciMaterials { if m.GitMaterial == nil { impl.logger.Warnw("git material are empty", "material", m) continue } - res := CiPipelineMaterialResponse{ + res := buildBean.CiPipelineMaterialResponse{ Id: m.Id, GitMaterialId: m.GitMaterialId, GitMaterialName: m.GitMaterial.Name[strings.Index(m.GitMaterial.Name, "-")+1:], diff --git a/client/events/EventClient.go b/client/events/EventClient.go index 5d7590f9b1..b15c23b513 100644 --- a/client/events/EventClient.go +++ b/client/events/EventClient.go @@ -21,20 +21,18 @@ import ( "encoding/json" "errors" "fmt" - bean2 "github.com/devtron-labs/devtron/pkg/attributes/bean" - "github.com/devtron-labs/devtron/pkg/module" - bean3 "github.com/devtron-labs/devtron/pkg/module/bean" - "net/http" - "time" - "github.com/caarlos0/env" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + bean2 "github.com/devtron-labs/devtron/pkg/attributes/bean" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/module" + bean3 "github.com/devtron-labs/devtron/pkg/module/bean" util "github.com/devtron-labs/devtron/util/event" "go.uber.org/zap" + "net/http" ) type EventClientConfig struct { @@ -81,41 +79,19 @@ type Event struct { } type Payload struct { - AppName string `json:"appName"` - EnvName string `json:"envName"` - PipelineName string `json:"pipelineName"` - Source string `json:"source"` - DockerImageUrl string `json:"dockerImageUrl"` - TriggeredBy string `json:"triggeredBy"` - Stage string `json:"stage"` - DeploymentHistoryLink string `json:"deploymentHistoryLink"` - AppDetailLink string `json:"appDetailLink"` - DownloadLink string `json:"downloadLink"` - BuildHistoryLink string `json:"buildHistoryLink"` - MaterialTriggerInfo *MaterialTriggerInfo `json:"material"` - FailureReason string `json:"failureReason"` -} - -type CiPipelineMaterialResponse struct { - Id int `json:"id"` - GitMaterialId int `json:"gitMaterialId"` - GitMaterialUrl string `json:"gitMaterialUrl"` - GitMaterialName string `json:"gitMaterialName"` - Type string `json:"type"` - Value string `json:"value"` - Active bool `json:"active"` - History []*gitSensor.GitCommit `json:"history,omitempty"` - LastFetchTime time.Time `json:"lastFetchTime"` - IsRepoError bool `json:"isRepoError"` - RepoErrorMsg string `json:"repoErrorMsg"` - IsBranchError bool `json:"isBranchError"` - BranchErrorMsg string `json:"branchErrorMsg"` - Url string `json:"url"` -} - -type MaterialTriggerInfo struct { - GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` - CiMaterials []CiPipelineMaterialResponse `json:"ciMaterials"` + AppName string `json:"appName"` + EnvName string `json:"envName"` + PipelineName string `json:"pipelineName"` + Source string `json:"source"` + DockerImageUrl string `json:"dockerImageUrl"` + TriggeredBy string `json:"triggeredBy"` + Stage string `json:"stage"` + DeploymentHistoryLink string `json:"deploymentHistoryLink"` + AppDetailLink string `json:"appDetailLink"` + DownloadLink string `json:"downloadLink"` + BuildHistoryLink string `json:"buildHistoryLink"` + MaterialTriggerInfo *buildBean.MaterialTriggerInfo `json:"material"` + FailureReason string `json:"failureReason"` } type EventRESTClientImpl struct { diff --git a/client/telemetry/TelemetryEventClient.go b/client/telemetry/TelemetryEventClient.go index b6c1979825..bfa4bf15c2 100644 --- a/client/telemetry/TelemetryEventClient.go +++ b/client/telemetry/TelemetryEventClient.go @@ -22,6 +22,7 @@ import ( "encoding/json" "fmt" cloudProviderIdentifier "github.com/devtron-labs/common-lib/cloud-provider-identifier" + posthogTelemetry "github.com/devtron-labs/common-lib/telemetry" "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/api/helm-app/gRPC" installedAppReader "github.com/devtron-labs/devtron/pkg/appStore/installedApp/read" @@ -29,7 +30,9 @@ import ( "github.com/devtron-labs/devtron/pkg/auth/user/bean" bean3 "github.com/devtron-labs/devtron/pkg/cluster/bean" module2 "github.com/devtron-labs/devtron/pkg/module/bean" + ucidService "github.com/devtron-labs/devtron/pkg/ucid" cron3 "github.com/devtron-labs/devtron/util/cron" + "go.opentelemetry.io/otel" "net/http" "time" @@ -43,22 +46,13 @@ import ( util3 "github.com/devtron-labs/devtron/pkg/util" "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" - "github.com/patrickmn/go-cache" "github.com/posthog/posthog-go" "github.com/robfig/cron/v3" "github.com/tidwall/gjson" "go.uber.org/zap" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "k8s.io/api/core/v1" - v12 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/version" ) -const LOGIN_COUNT_CONST = "login-count" -const SKIPPED_ONBOARDING_CONST = "SkippedOnboarding" -const ADMIN_EMAIL_ID_CONST = "admin" - type TelemetryEventClientImpl struct { cron *cron.Cron logger *zap.SugaredLogger @@ -69,7 +63,8 @@ type TelemetryEventClientImpl struct { userService user2.UserService attributeRepo repository.AttributesRepository ssoLoginService sso.SSOLoginService - PosthogClient *PosthogClient + posthogClient *posthogTelemetry.PosthogClient + ucid ucidService.Service moduleRepository moduleRepo.ModuleRepository serverDataStore *serverDataStore.ServerDataStore userAuditService user2.UserAuditService @@ -93,9 +88,9 @@ type TelemetryEventClient interface { func NewTelemetryEventClientImpl(logger *zap.SugaredLogger, client *http.Client, clusterService cluster.ClusterService, K8sUtil *k8s.K8sServiceImpl, aCDAuthConfig *util3.ACDAuthConfig, userService user2.UserService, attributeRepo repository.AttributesRepository, ssoLoginService sso.SSOLoginService, - PosthogClient *PosthogClient, moduleRepository moduleRepo.ModuleRepository, - serverDataStore *serverDataStore.ServerDataStore, userAuditService user2.UserAuditService, - helmAppClient gRPC.HelmAppClient, + posthog *posthogTelemetry.PosthogClient, ucid ucidService.Service, + moduleRepository moduleRepo.ModuleRepository, serverDataStore *serverDataStore.ServerDataStore, + userAuditService user2.UserAuditService, helmAppClient gRPC.HelmAppClient, cloudProviderIdentifierService cloudProviderIdentifier.ProviderIdentifierService, cronLogger *cron3.CronLoggerImpl, installedAppReadService installedAppReader.InstalledAppReadServiceEA, envVariables *util.EnvironmentVariables, @@ -113,7 +108,8 @@ func NewTelemetryEventClientImpl(logger *zap.SugaredLogger, client *http.Client, userService: userService, attributeRepo: attributeRepo, ssoLoginService: ssoLoginService, - PosthogClient: PosthogClient, + posthogClient: posthog, + ucid: ucid, moduleRepository: moduleRepository, serverDataStore: serverDataStore, userAuditService: userAuditService, @@ -126,13 +122,13 @@ func NewTelemetryEventClientImpl(logger *zap.SugaredLogger, client *http.Client, } watcher.HeartbeatEventForTelemetry() - _, err := cron.AddFunc(SummaryCronExpr, watcher.SummaryEventForTelemetryEA) + _, err := cron.AddFunc(posthogTelemetry.SummaryCronExpr, watcher.SummaryEventForTelemetryEA) if err != nil { logger.Errorw("error in starting summery event", "err", err) return nil, err } - _, err = cron.AddFunc(HeartbeatCronExpr, watcher.HeartbeatEventForTelemetry) + _, err = cron.AddFunc(posthogTelemetry.HeartbeatCronExpr, watcher.HeartbeatEventForTelemetry) if err != nil { logger.Errorw("error in starting heartbeat event", "err", err) return nil, err @@ -157,59 +153,6 @@ func (impl *TelemetryEventClientImpl) StopCron() { impl.cron.Stop() } -type TelemetryEventEA struct { - UCID string `json:"ucid"` //unique client id - Timestamp time.Time `json:"timestamp"` - EventMessage string `json:"eventMessage,omitempty"` - EventType TelemetryEventType `json:"eventType"` - ServerVersion string `json:"serverVersion,omitempty"` - UserCount int `json:"userCount,omitempty"` - ClusterCount int `json:"clusterCount,omitempty"` - HostURL bool `json:"hostURL,omitempty"` - SSOLogin bool `json:"ssoLogin,omitempty"` - DevtronVersion string `json:"devtronVersion,omitempty"` - DevtronMode string `json:"devtronMode,omitempty"` - InstalledIntegrations []string `json:"installedIntegrations,omitempty"` - InstallFailedIntegrations []string `json:"installFailedIntegrations,omitempty"` - InstallTimedOutIntegrations []string `json:"installTimedOutIntegrations,omitempty"` - LastLoginTime time.Time `json:"LastLoginTime,omitempty"` - InstallingIntegrations []string `json:"installingIntegrations,omitempty"` - DevtronReleaseVersion string `json:"devtronReleaseVersion,omitempty"` - HelmAppAccessCounter string `json:"HelmAppAccessCounter,omitempty"` - HelmAppUpdateCounter string `json:"HelmAppUpdateCounter,omitempty"` - ChartStoreVisitCount string `json:"ChartStoreVisitCount,omitempty"` - SkippedOnboarding bool `json:"SkippedOnboarding"` - HelmChartSuccessfulDeploymentCount int `json:"helmChartSuccessfulDeploymentCount,omitempty"` - ExternalHelmAppClusterCount map[int32]int `json:"ExternalHelmAppClusterCount,omitempty"` - ClusterProvider string `json:"clusterProvider,omitempty"` -} - -const DevtronUniqueClientIdConfigMap = "devtron-ucid" -const DevtronUniqueClientIdConfigMapKey = "UCID" -const InstallEventKey = "installEvent" -const UIEventKey = "uiEventKey" - -type TelemetryEventType string - -const ( - Heartbeat TelemetryEventType = "Heartbeat" - InstallationStart TelemetryEventType = "InstallationStart" - InstallationInProgress TelemetryEventType = "InstallationInProgress" - InstallationInterrupt TelemetryEventType = "InstallationInterrupt" - InstallationSuccess TelemetryEventType = "InstallationSuccess" - InstallationFailure TelemetryEventType = "InstallationFailure" - UpgradeStart TelemetryEventType = "UpgradeStart" - UpgradeInProgress TelemetryEventType = "UpgradeInProgress" - UpgradeInterrupt TelemetryEventType = "UpgradeInterrupt" - UpgradeSuccess TelemetryEventType = "UpgradeSuccess" - UpgradeFailure TelemetryEventType = "UpgradeFailure" - Summary TelemetryEventType = "Summary" - InstallationApplicationError TelemetryEventType = "InstallationApplicationError" - DashboardAccessed TelemetryEventType = "DashboardAccessed" - DashboardLoggedIn TelemetryEventType = "DashboardLoggedIn" - SIG_TERM TelemetryEventType = "SIG_TERM" -) - func (impl *TelemetryEventClientImpl) SummaryDetailsForTelemetry() (cluster []bean3.ClusterBean, user []bean.UserInfo, k8sServerVersion *version.Info, hostURL bool, ssoSetup bool, HelmAppAccessCount string, ChartStoreVisitCount string, SkippedOnboarding bool, HelmAppUpdateCounter string, helmChartSuccessfulDeploymentCount int, ExternalHelmAppClusterCount map[int32]int) { @@ -299,9 +242,9 @@ func (impl *TelemetryEventClientImpl) SummaryDetailsForTelemetry() (cluster []be } //getting userData from emailId - userData, err := impl.userAttributesRepository.GetUserDataByEmailId(ADMIN_EMAIL_ID_CONST) + userData, err := impl.userAttributesRepository.GetUserDataByEmailId(AdminEmailIdConst) - SkippedOnboardingValue := gjson.Get(userData, SKIPPED_ONBOARDING_CONST).Str + SkippedOnboardingValue := gjson.Get(userData, SkippedOnboardingConst).Str if SkippedOnboardingValue == "true" { SkippedOnboarding = true @@ -328,13 +271,13 @@ func (impl *TelemetryEventClientImpl) SummaryEventForTelemetryEA() { func (impl *TelemetryEventClientImpl) SendSummaryEvent(eventType string) error { impl.logger.Infow("sending summary event", "eventType", eventType) - ucid, err := impl.getUCID() + ucid, err := impl.getUCIDAndCheckIsOptedOut(context.Background()) if err != nil { impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) return err } - if IsOptOut { + if posthogTelemetry.IsOptOut { impl.logger.Warnw("client is opt-out for telemetry, there will be no events capture", "ucid", ucid) return err } @@ -406,13 +349,13 @@ func (impl *TelemetryEventClientImpl) EnqueuePostHog(ucid string, eventType Tele } func (impl *TelemetryEventClientImpl) SendGenericTelemetryEvent(eventType string, prop map[string]interface{}) error { - ucid, err := impl.getUCID() + ucid, err := impl.getUCIDAndCheckIsOptedOut(context.Background()) if err != nil { impl.logger.Errorw("exception caught inside telemetry generic event", "err", err) return nil } - if IsOptOut { + if posthogTelemetry.IsOptOut { impl.logger.Warnw("client is opt-out for telemetry, there will be no events capture", "ucid", ucid) return nil } @@ -421,15 +364,15 @@ func (impl *TelemetryEventClientImpl) SendGenericTelemetryEvent(eventType string } func (impl *TelemetryEventClientImpl) EnqueueGenericPostHogEvent(ucid string, eventType string, prop map[string]interface{}) error { - if impl.PosthogClient.Client == nil { + if impl.posthogClient.Client == nil { impl.logger.Warn("no posthog client found, creating new") - client, err := impl.retryPosthogClient(PosthogApiKey, PosthogEndpoint) + client, err := impl.retryPosthogClient(posthogTelemetry.PosthogApiKey, posthogTelemetry.PosthogEndpoint) if err == nil { - impl.PosthogClient.Client = client + impl.posthogClient.Client = client } } - if impl.PosthogClient.Client != nil && !impl.globalEnvVariables.IsAirGapEnvironment { - err := impl.PosthogClient.Client.Enqueue(posthog.Capture{ + if impl.posthogClient.Client != nil && !impl.globalEnvVariables.IsAirGapEnvironment { + err := impl.posthogClient.Client.Enqueue(posthog.Capture{ DistinctId: ucid, Event: eventType, Properties: prop, @@ -443,12 +386,12 @@ func (impl *TelemetryEventClientImpl) EnqueueGenericPostHogEvent(ucid string, ev } func (impl *TelemetryEventClientImpl) HeartbeatEventForTelemetry() { - ucid, err := impl.getUCID() + ucid, err := impl.getUCIDAndCheckIsOptedOut(context.Background()) if err != nil { impl.logger.Errorw("exception caught inside telemetry heartbeat event", "err", err) return } - if IsOptOut { + if posthogTelemetry.IsOptOut { impl.logger.Warnw("client is opt-out for telemetry, there will be no events capture", "ucid", ucid) return } @@ -490,21 +433,21 @@ func (impl *TelemetryEventClientImpl) HeartbeatEventForTelemetry() { } func (impl *TelemetryEventClientImpl) GetTelemetryMetaInfo() (*TelemetryMetaInfo, error) { - ucid, err := impl.getUCID() + ucid, err := impl.getUCIDAndCheckIsOptedOut(context.Background()) if err != nil { impl.logger.Errorw("exception while getting unique client id", "error", err) return nil, err } data := &TelemetryMetaInfo{ - Url: PosthogEndpoint, + Url: posthogTelemetry.PosthogEndpoint, UCID: ucid, - ApiKey: PosthogEncodedApiKey, + ApiKey: posthogTelemetry.PosthogEncodedApiKey, } return data, err } func (impl *TelemetryEventClientImpl) SendTelemetryInstallEventEA() (*TelemetryEventType, error) { - ucid, err := impl.getUCID() + ucid, err := impl.getUCIDAndCheckIsOptedOut(context.Background()) if err != nil { impl.logger.Errorw("exception while getting unique client id", "error", err) return nil, err @@ -548,15 +491,19 @@ func (impl *TelemetryEventClientImpl) SendTelemetryInstallEventEA() (*TelemetryE impl.logger.Errorw("Installation EventForTelemetry EA Mode, payload unmarshal error", "error", err) return nil, nil } - cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, DevtronUniqueClientIdConfigMap, client) + cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, ucidService.DevtronUniqueClientIdConfigMap, client) + if err != nil { + impl.logger.Errorw("Installation EventForTelemetry EA Mode, failed to get DevtronUniqueClientIdConfigMap", "error", err) + return nil, err + } datamap := cm.Data - installEventValue, installEventKeyExists := datamap[InstallEventKey] + installEventValue, installEventKeyExists := datamap[ucidService.InstallEventKey] if installEventKeyExists == false || installEventValue == "1" { err = impl.EnqueuePostHog(ucid, InstallationSuccess, prop) if err == nil { - datamap[InstallEventKey] = "2" + datamap[ucidService.InstallEventKey] = "2" cm.Data = datamap _, err = impl.K8sUtil.UpdateConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, cm, client) if err != nil { @@ -570,7 +517,7 @@ func (impl *TelemetryEventClientImpl) SendTelemetryInstallEventEA() (*TelemetryE } func (impl *TelemetryEventClientImpl) SendTelemetryDashboardAccessEvent() error { - ucid, err := impl.getUCID() + ucid, err := impl.getUCIDAndCheckIsOptedOut(context.Background()) if err != nil { impl.logger.Errorw("exception while getting unique client id", "error", err) return err @@ -614,19 +561,19 @@ func (impl *TelemetryEventClientImpl) SendTelemetryDashboardAccessEvent() error impl.logger.Errorw("DashboardAccessed EventForTelemetry, payload unmarshal error", "error", err) return err } - cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, DevtronUniqueClientIdConfigMap, client) + cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, ucidService.DevtronUniqueClientIdConfigMap, client) if err != nil { impl.logger.Errorw("DashboardAccessed EventForTelemetry,failed to get DevtronUniqueClientIdConfigMap", "error", err) return err } datamap := cm.Data - accessEventValue, installEventKeyExists := datamap[UIEventKey] + accessEventValue, installEventKeyExists := datamap[ucidService.UIEventKey] if installEventKeyExists == false || accessEventValue <= "1" { err = impl.EnqueuePostHog(ucid, DashboardAccessed, prop) if err == nil { - datamap[UIEventKey] = "2" + datamap[ucidService.UIEventKey] = "2" cm.Data = datamap _, err = impl.K8sUtil.UpdateConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, cm, client) if err != nil { @@ -640,7 +587,7 @@ func (impl *TelemetryEventClientImpl) SendTelemetryDashboardAccessEvent() error } func (impl *TelemetryEventClientImpl) SendTelemetryDashboardLoggedInEvent() error { - ucid, err := impl.getUCID() + ucid, err := impl.getUCIDAndCheckIsOptedOut(context.Background()) if err != nil { impl.logger.Errorw("exception while getting unique client id", "error", err) return err @@ -684,19 +631,19 @@ func (impl *TelemetryEventClientImpl) SendTelemetryDashboardLoggedInEvent() erro impl.logger.Errorw("DashboardLoggedIn EventForTelemetry, payload unmarshal error", "error", err) return err } - cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, DevtronUniqueClientIdConfigMap, client) + cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, ucidService.DevtronUniqueClientIdConfigMap, client) if err != nil { impl.logger.Errorw("DashboardLoggedIn EventForTelemetry,failed to get DevtronUniqueClientIdConfigMap", "error", err) return err } datamap := cm.Data - accessEventValue, installEventKeyExists := datamap[UIEventKey] + accessEventValue, installEventKeyExists := datamap[ucidService.UIEventKey] if installEventKeyExists == false || accessEventValue != "3" { err = impl.EnqueuePostHog(ucid, DashboardLoggedIn, prop) if err == nil { - datamap[UIEventKey] = "3" + datamap[ucidService.UIEventKey] = "3" cm.Data = datamap _, err = impl.K8sUtil.UpdateConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, cm, client) if err != nil { @@ -709,57 +656,29 @@ func (impl *TelemetryEventClientImpl) SendTelemetryDashboardLoggedInEvent() erro return nil } -type TelemetryMetaInfo struct { - Url string `json:"url,omitempty"` - UCID string `json:"ucid,omitempty"` - ApiKey string `json:"apiKey,omitempty"` -} - -func (impl *TelemetryEventClientImpl) getUCID() (string, error) { - ucid, found := impl.PosthogClient.cache.Get(DevtronUniqueClientIdConfigMapKey) - if found { - return ucid.(string), nil - } else { - client, err := impl.K8sUtil.GetClientForInCluster() - if err != nil { - impl.logger.Errorw("exception while getting unique client id", "error", err) - return "", err - } - - cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, DevtronUniqueClientIdConfigMap, client) - if errStatus, ok := status.FromError(err); !ok || errStatus.Code() == codes.NotFound || errStatus.Code() == codes.Unknown { - // if not found, create new cm - cm = &v1.ConfigMap{ObjectMeta: v12.ObjectMeta{Name: DevtronUniqueClientIdConfigMap}} - data := map[string]string{} - data[DevtronUniqueClientIdConfigMapKey] = util.Generate(16) // generate unique random number - data[InstallEventKey] = "1" // used in operator to detect event is install or upgrade - data[UIEventKey] = "1" - cm.Data = data - _, err = impl.K8sUtil.CreateConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, cm, client) - if err != nil { - impl.logger.Errorw("exception while getting unique client id", "error", err) - return "", err - } - } - dataMap := cm.Data - ucid = dataMap[DevtronUniqueClientIdConfigMapKey] - impl.PosthogClient.cache.Set(DevtronUniqueClientIdConfigMapKey, ucid, cache.DefaultExpiration) - if cm == nil { - impl.logger.Errorw("configmap not found while getting unique client id", "cm", cm) - return ucid.(string), err - } - flag, err := impl.checkForOptOut(ucid.(string)) +func (impl *TelemetryEventClientImpl) getUCIDAndCheckIsOptedOut(ctx context.Context) (string, error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TelemetryEventClientImpl.getUCIDAndCheckIsOptedOut") + defer span.End() + ucid, found, err := impl.ucid.GetUCIDWithCache(impl.posthogClient.GetCache()) + if err != nil { + impl.logger.Errorw("exception while getting unique client id", "error", err) + return "", err + } + if !found { + flag, err := impl.checkForOptOut(newCtx, ucid) if err != nil { impl.logger.Errorw("error sending event to posthog, failed check for opt-out", "err", err) return "", err } - IsOptOut = flag + posthogTelemetry.IsOptOut = flag } - return ucid.(string), nil + return ucid, nil } -func (impl *TelemetryEventClientImpl) checkForOptOut(UCID string) (bool, error) { - decodedUrl, err := base64.StdEncoding.DecodeString(TelemetryOptOutApiBaseUrl) +func (impl *TelemetryEventClientImpl) checkForOptOut(ctx context.Context, UCID string) (bool, error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TelemetryEventClientImpl.checkForOptOut") + defer span.End() + decodedUrl, err := base64.StdEncoding.DecodeString(posthogTelemetry.TelemetryOptOutApiBaseUrl) if err != nil { impl.logger.Errorw("check opt-out list failed, decode error", "err", err) return false, err @@ -767,13 +686,13 @@ func (impl *TelemetryEventClientImpl) checkForOptOut(UCID string) (bool, error) encodedUrl := string(decodedUrl) url := fmt.Sprintf("%s/%s", encodedUrl, UCID) - response, err := util.HttpRequest(url) + response, err := util.HttpRequest(newCtx, url) if err != nil { - impl.logger.Errorw("check opt-out list failed, rest api error", "err", err) - return false, err + // this should be non-blocking call and should not fail the request for ucid getting + impl.logger.Errorw("check opt-out list failed, rest api error", "ucid", UCID, "err", err) } flag := response["result"].(bool) - return flag, err + return flag, nil } func (impl *TelemetryEventClientImpl) retryPosthogClient(PosthogApiKey string, PosthogEndpoint string) (posthog.Client, error) { @@ -785,7 +704,7 @@ func (impl *TelemetryEventClientImpl) retryPosthogClient(PosthogApiKey string, P return client, err } -// returns installedIntegrations, installFailedIntegrations, installTimedOutIntegrations, installingIntegrations +// buildIntegrationsList - returns installedIntegrations, installFailedIntegrations, installTimedOutIntegrations, installingIntegrations func (impl *TelemetryEventClientImpl) buildIntegrationsList() ([]string, []string, []string, []string, error) { impl.logger.Info("building integrations list for telemetry") diff --git a/client/telemetry/TelemetryEventClientExtended.go b/client/telemetry/TelemetryEventClientExtended.go index d185789fef..87de1d237f 100644 --- a/client/telemetry/TelemetryEventClientExtended.go +++ b/client/telemetry/TelemetryEventClientExtended.go @@ -17,8 +17,10 @@ package telemetry import ( + "context" "encoding/json" cloudProviderIdentifier "github.com/devtron-labs/common-lib/cloud-provider-identifier" + posthogTelemetry "github.com/devtron-labs/common-lib/telemetry" util2 "github.com/devtron-labs/common-lib/utils/k8s" client "github.com/devtron-labs/devtron/api/helm-app/gRPC" installedAppReader "github.com/devtron-labs/devtron/pkg/appStore/installedApp/read" @@ -33,6 +35,7 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" moduleRepo "github.com/devtron-labs/devtron/pkg/module/repo" serverDataStore "github.com/devtron-labs/devtron/pkg/server/store" + ucidService "github.com/devtron-labs/devtron/pkg/ucid" util3 "github.com/devtron-labs/devtron/pkg/util" cron3 "github.com/devtron-labs/devtron/util/cron" "net/http" @@ -50,8 +53,6 @@ import ( "go.uber.org/zap" ) -const AppsCount int = 50 - type TelemetryEventClientImplExtended struct { environmentService environment.EnvironmentService appListingRepository repository.AppListingRepository @@ -73,7 +74,7 @@ type TelemetryEventClientImplExtended struct { func NewTelemetryEventClientImplExtended(logger *zap.SugaredLogger, client *http.Client, clusterService cluster.ClusterService, K8sUtil *util2.K8sServiceImpl, aCDAuthConfig *util3.ACDAuthConfig, environmentService environment.EnvironmentService, userService user2.UserService, - appListingRepository repository.AppListingRepository, PosthogClient *PosthogClient, + appListingRepository repository.AppListingRepository, posthog *posthogTelemetry.PosthogClient, ucid ucidService.Service, ciPipelineConfigReadService ciConfig.CiPipelineConfigReadService, pipelineRepository pipelineConfig.PipelineRepository, gitProviderRepository repository3.GitProviderRepository, attributeRepo repository.AttributesRepository, ssoLoginService sso.SSOLoginService, appRepository app.AppRepository, @@ -114,7 +115,8 @@ func NewTelemetryEventClientImplExtended(logger *zap.SugaredLogger, client *http userService: userService, attributeRepo: attributeRepo, ssoLoginService: ssoLoginService, - PosthogClient: PosthogClient, + posthogClient: posthog, + ucid: ucid, moduleRepository: moduleRepository, serverDataStore: serverDataStore, userAuditService: userAuditService, @@ -128,13 +130,13 @@ func NewTelemetryEventClientImplExtended(logger *zap.SugaredLogger, client *http } watcher.HeartbeatEventForTelemetry() - _, err := cron.AddFunc(SummaryCronExpr, watcher.SummaryEventForTelemetry) + _, err := cron.AddFunc(posthogTelemetry.SummaryCronExpr, watcher.SummaryEventForTelemetry) if err != nil { logger.Errorw("error in starting summery event", "err", err) return nil, err } - _, err = cron.AddFunc(HeartbeatCronExpr, watcher.HeartbeatEventForTelemetry) + _, err = cron.AddFunc(posthogTelemetry.HeartbeatCronExpr, watcher.HeartbeatEventForTelemetry) if err != nil { logger.Errorw("error in starting heartbeat event", "err", err) return nil, err @@ -142,65 +144,6 @@ func NewTelemetryEventClientImplExtended(logger *zap.SugaredLogger, client *http return watcher, err } -type TelemetryEventDto struct { - UCID string `json:"ucid"` //unique client id - Timestamp time.Time `json:"timestamp"` - EventMessage string `json:"eventMessage,omitempty"` - EventType TelemetryEventType `json:"eventType"` - ProdAppCount int `json:"prodAppCount,omitempty"` - NonProdAppCount int `json:"nonProdAppCount,omitempty"` - UserCount int `json:"userCount,omitempty"` - EnvironmentCount int `json:"environmentCount,omitempty"` - ClusterCount int `json:"clusterCount,omitempty"` - CiCreatedPerDay int `json:"ciCreatedPerDay"` - CdCreatedPerDay int `json:"cdCreatedPerDay"` - CiDeletedPerDay int `json:"ciDeletedPerDay"` - CdDeletedPerDay int `json:"cdDeletedPerDay"` - CiTriggeredPerDay int `json:"ciTriggeredPerDay"` - CdTriggeredPerDay int `json:"cdTriggeredPerDay"` - HelmChartCount int `json:"helmChartCount,omitempty"` - SecurityScanCountPerDay int `json:"securityScanCountPerDay,omitempty"` - GitAccountsCount int `json:"gitAccountsCount,omitempty"` - GitOpsCount int `json:"gitOpsCount,omitempty"` - RegistryCount int `json:"registryCount,omitempty"` - HostURL bool `json:"hostURL,omitempty"` - SSOLogin bool `json:"ssoLogin,omitempty"` - AppCount int `json:"appCount,omitempty"` - AppsWithGitRepoConfigured int `json:"appsWithGitRepoConfigured,omitempty"` - AppsWithDockerConfigured int `json:"appsWithDockerConfigured,omitempty"` - AppsWithDeploymentTemplateConfigured int `json:"appsWithDeploymentTemplateConfigured,omitempty"` - AppsWithCiPipelineConfigured int `json:"appsWithCiPipelineConfigured,omitempty"` - AppsWithCdPipelineConfigured int `json:"appsWithCdPipelineConfigured,omitempty"` - Build bool `json:"build,omitempty"` - Deployment bool `json:"deployment,omitempty"` - ServerVersion string `json:"serverVersion,omitempty"` - DevtronGitVersion string `json:"devtronGitVersion,omitempty"` - DevtronVersion string `json:"devtronVersion,omitempty"` - DevtronMode string `json:"devtronMode,omitempty"` - InstalledIntegrations []string `json:"installedIntegrations,omitempty"` - InstallFailedIntegrations []string `json:"installFailedIntegrations,omitempty"` - InstallTimedOutIntegrations []string `json:"installTimedOutIntegrations,omitempty"` - InstallingIntegrations []string `json:"installingIntegrations,omitempty"` - DevtronReleaseVersion string `json:"devtronReleaseVersion,omitempty"` - LastLoginTime time.Time `json:"LastLoginTime,omitempty"` - SelfDockerfileCount int `json:"selfDockerfileCount"` - ManagedDockerfileCount int `json:"managedDockerfileCount"` - BuildPackCount int `json:"buildPackCount"` - SelfDockerfileSuccessCount int `json:"selfDockerfileSuccessCount"` - SelfDockerfileFailureCount int `json:"selfDockerfileFailureCount"` - ManagedDockerfileSuccessCount int `json:"managedDockerfileSuccessCount"` - ManagedDockerfileFailureCount int `json:"managedDockerfileFailureCount"` - BuildPackSuccessCount int `json:"buildPackSuccessCount"` - BuildPackFailureCount int `json:"buildPackFailureCount"` - HelmAppAccessCounter string `json:"HelmAppAccessCounter,omitempty"` - ChartStoreVisitCount string `json:"ChartStoreVisitCount,omitempty"` - SkippedOnboarding bool `json:"SkippedOnboarding"` - HelmAppUpdateCounter string `json:"HelmAppUpdateCounter,omitempty"` - HelmChartSuccessfulDeploymentCount int `json:"helmChartSuccessfulDeploymentCount,omitempty"` - ExternalHelmAppClusterCount map[int32]int `json:"ExternalHelmAppClusterCount"` - ClusterProvider string `json:"clusterProvider,omitempty"` -} - func (impl *TelemetryEventClientImplExtended) SummaryEventForTelemetry() { err := impl.SendSummaryEvent(string(Summary)) if err != nil { @@ -210,13 +153,13 @@ func (impl *TelemetryEventClientImplExtended) SummaryEventForTelemetry() { func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string) error { impl.logger.Infow("sending summary event", "eventType", eventType) - ucid, err := impl.getUCID() + ucid, err := impl.getUCIDAndCheckIsOptedOut(context.Background()) if err != nil { impl.logger.Errorw("exception caught inside telemetry summary event while retrieving ucid", "err", err) return err } - if IsOptOut { + if posthogTelemetry.IsOptOut { impl.logger.Warnw("client is opt-out for telemetry, there will be no events capture", "ucid", ucid) return err } diff --git a/client/telemetry/bean.go b/client/telemetry/bean.go new file mode 100644 index 0000000000..f3e95dd0e7 --- /dev/null +++ b/client/telemetry/bean.go @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2020-2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package telemetry + +import "time" + +type TelemetryMetaInfo struct { + Url string `json:"url,omitempty"` + UCID string `json:"ucid,omitempty"` + ApiKey string `json:"apiKey,omitempty"` +} + +const ( + SkippedOnboardingConst = "SkippedOnboarding" + AdminEmailIdConst = "admin" +) + +type TelemetryEventType string + +const ( + Heartbeat TelemetryEventType = "Heartbeat" + InstallationStart TelemetryEventType = "InstallationStart" + InstallationInProgress TelemetryEventType = "InstallationInProgress" + InstallationInterrupt TelemetryEventType = "InstallationInterrupt" + InstallationSuccess TelemetryEventType = "InstallationSuccess" + InstallationFailure TelemetryEventType = "InstallationFailure" + UpgradeStart TelemetryEventType = "UpgradeStart" + UpgradeInProgress TelemetryEventType = "UpgradeInProgress" + UpgradeInterrupt TelemetryEventType = "UpgradeInterrupt" + UpgradeSuccess TelemetryEventType = "UpgradeSuccess" + UpgradeFailure TelemetryEventType = "UpgradeFailure" + Summary TelemetryEventType = "Summary" + InstallationApplicationError TelemetryEventType = "InstallationApplicationError" + DashboardAccessed TelemetryEventType = "DashboardAccessed" + DashboardLoggedIn TelemetryEventType = "DashboardLoggedIn" + SIG_TERM TelemetryEventType = "SIG_TERM" +) + +type TelemetryEventEA struct { + UCID string `json:"ucid"` //unique client id + Timestamp time.Time `json:"timestamp"` + EventMessage string `json:"eventMessage,omitempty"` + EventType TelemetryEventType `json:"eventType"` + ServerVersion string `json:"serverVersion,omitempty"` + UserCount int `json:"userCount,omitempty"` + ClusterCount int `json:"clusterCount,omitempty"` + HostURL bool `json:"hostURL,omitempty"` + SSOLogin bool `json:"ssoLogin,omitempty"` + DevtronVersion string `json:"devtronVersion,omitempty"` + DevtronMode string `json:"devtronMode,omitempty"` + InstalledIntegrations []string `json:"installedIntegrations,omitempty"` + InstallFailedIntegrations []string `json:"installFailedIntegrations,omitempty"` + InstallTimedOutIntegrations []string `json:"installTimedOutIntegrations,omitempty"` + LastLoginTime time.Time `json:"LastLoginTime,omitempty"` + InstallingIntegrations []string `json:"installingIntegrations,omitempty"` + DevtronReleaseVersion string `json:"devtronReleaseVersion,omitempty"` + HelmAppAccessCounter string `json:"HelmAppAccessCounter,omitempty"` + HelmAppUpdateCounter string `json:"HelmAppUpdateCounter,omitempty"` + ChartStoreVisitCount string `json:"ChartStoreVisitCount,omitempty"` + SkippedOnboarding bool `json:"SkippedOnboarding"` + HelmChartSuccessfulDeploymentCount int `json:"helmChartSuccessfulDeploymentCount,omitempty"` + ExternalHelmAppClusterCount map[int32]int `json:"ExternalHelmAppClusterCount,omitempty"` + ClusterProvider string `json:"clusterProvider,omitempty"` +} + +const AppsCount int = 50 + +type TelemetryEventDto struct { + UCID string `json:"ucid"` //unique client id + Timestamp time.Time `json:"timestamp"` + EventMessage string `json:"eventMessage,omitempty"` + EventType TelemetryEventType `json:"eventType"` + ProdAppCount int `json:"prodAppCount,omitempty"` + NonProdAppCount int `json:"nonProdAppCount,omitempty"` + UserCount int `json:"userCount,omitempty"` + EnvironmentCount int `json:"environmentCount,omitempty"` + ClusterCount int `json:"clusterCount,omitempty"` + CiCreatedPerDay int `json:"ciCreatedPerDay"` + CdCreatedPerDay int `json:"cdCreatedPerDay"` + CiDeletedPerDay int `json:"ciDeletedPerDay"` + CdDeletedPerDay int `json:"cdDeletedPerDay"` + CiTriggeredPerDay int `json:"ciTriggeredPerDay"` + CdTriggeredPerDay int `json:"cdTriggeredPerDay"` + HelmChartCount int `json:"helmChartCount,omitempty"` + SecurityScanCountPerDay int `json:"securityScanCountPerDay,omitempty"` + GitAccountsCount int `json:"gitAccountsCount,omitempty"` + GitOpsCount int `json:"gitOpsCount,omitempty"` + RegistryCount int `json:"registryCount,omitempty"` + HostURL bool `json:"hostURL,omitempty"` + SSOLogin bool `json:"ssoLogin,omitempty"` + AppCount int `json:"appCount,omitempty"` + AppsWithGitRepoConfigured int `json:"appsWithGitRepoConfigured,omitempty"` + AppsWithDockerConfigured int `json:"appsWithDockerConfigured,omitempty"` + AppsWithDeploymentTemplateConfigured int `json:"appsWithDeploymentTemplateConfigured,omitempty"` + AppsWithCiPipelineConfigured int `json:"appsWithCiPipelineConfigured,omitempty"` + AppsWithCdPipelineConfigured int `json:"appsWithCdPipelineConfigured,omitempty"` + Build bool `json:"build,omitempty"` + Deployment bool `json:"deployment,omitempty"` + ServerVersion string `json:"serverVersion,omitempty"` + DevtronGitVersion string `json:"devtronGitVersion,omitempty"` + DevtronVersion string `json:"devtronVersion,omitempty"` + DevtronMode string `json:"devtronMode,omitempty"` + InstalledIntegrations []string `json:"installedIntegrations,omitempty"` + InstallFailedIntegrations []string `json:"installFailedIntegrations,omitempty"` + InstallTimedOutIntegrations []string `json:"installTimedOutIntegrations,omitempty"` + InstallingIntegrations []string `json:"installingIntegrations,omitempty"` + DevtronReleaseVersion string `json:"devtronReleaseVersion,omitempty"` + LastLoginTime time.Time `json:"LastLoginTime,omitempty"` + SelfDockerfileCount int `json:"selfDockerfileCount"` + ManagedDockerfileCount int `json:"managedDockerfileCount"` + BuildPackCount int `json:"buildPackCount"` + SelfDockerfileSuccessCount int `json:"selfDockerfileSuccessCount"` + SelfDockerfileFailureCount int `json:"selfDockerfileFailureCount"` + ManagedDockerfileSuccessCount int `json:"managedDockerfileSuccessCount"` + ManagedDockerfileFailureCount int `json:"managedDockerfileFailureCount"` + BuildPackSuccessCount int `json:"buildPackSuccessCount"` + BuildPackFailureCount int `json:"buildPackFailureCount"` + HelmAppAccessCounter string `json:"HelmAppAccessCounter,omitempty"` + ChartStoreVisitCount string `json:"ChartStoreVisitCount,omitempty"` + SkippedOnboarding bool `json:"SkippedOnboarding"` + HelmAppUpdateCounter string `json:"HelmAppUpdateCounter,omitempty"` + HelmChartSuccessfulDeploymentCount int `json:"helmChartSuccessfulDeploymentCount,omitempty"` + ExternalHelmAppClusterCount map[int32]int `json:"ExternalHelmAppClusterCount"` + ClusterProvider string `json:"clusterProvider,omitempty"` +} diff --git a/cmd/external-app/externalApp.go b/cmd/external-app/externalApp.go index b7f90fb2dd..dba06f4eea 100644 --- a/cmd/external-app/externalApp.go +++ b/cmd/external-app/externalApp.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + posthogTelemetry "github.com/devtron-labs/common-lib/telemetry" "net/http" "os" "time" @@ -40,14 +41,14 @@ type App struct { Logger *zap.SugaredLogger server *http.Server telemetry telemetry.TelemetryEventClient - posthogClient *telemetry.PosthogClient + posthogClient *posthogTelemetry.PosthogClient } func NewApp(db *pg.DB, sessionManager *authMiddleware.SessionManager, MuxRouter *MuxRouter, telemetry telemetry.TelemetryEventClient, - posthogClient *telemetry.PosthogClient, + posthogClient *posthogTelemetry.PosthogClient, Logger *zap.SugaredLogger) *App { return &App{ db: db, diff --git a/cmd/external-app/wire.go b/cmd/external-app/wire.go index 8e633f9983..8986381ea6 100644 --- a/cmd/external-app/wire.go +++ b/cmd/external-app/wire.go @@ -22,6 +22,7 @@ package main import ( "github.com/devtron-labs/authenticator/middleware" cloudProviderIdentifier "github.com/devtron-labs/common-lib/cloud-provider-identifier" + posthogTelemetry "github.com/devtron-labs/common-lib/telemetry" util4 "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/api/apiToken" "github.com/devtron-labs/devtron/api/appStore" @@ -87,6 +88,7 @@ import ( "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool" security2 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" "github.com/devtron-labs/devtron/pkg/sql" + "github.com/devtron-labs/devtron/pkg/ucid" util2 "github.com/devtron-labs/devtron/pkg/util" util3 "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/commonEnforcementFunctionsUtil" @@ -132,7 +134,8 @@ func InitializeApp() (*App, error) { util.NewSugardLogger, util.IntValidator, util2.GetACDAuthConfig, - telemetry.NewPosthogClient, + posthogTelemetry.NewPosthogClient, + ucid.WireSet, delete2.NewDeleteServiceImpl, gitMaterial.GitMaterialWireSet, scanTool.ScanToolWireSet, diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index d690e0d96c..43cd02248a 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run -mod=mod github.com/google/wire/cmd/wire +//go:generate go run github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject @@ -11,6 +11,7 @@ import ( "github.com/devtron-labs/authenticator/client" "github.com/devtron-labs/authenticator/middleware" "github.com/devtron-labs/common-lib/cloud-provider-identifier" + "github.com/devtron-labs/common-lib/telemetry" "github.com/devtron-labs/common-lib/utils/grpc" "github.com/devtron-labs/common-lib/utils/k8s" apiToken2 "github.com/devtron-labs/devtron/api/apiToken" @@ -52,7 +53,7 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer/repoCredsK8sClient" "github.com/devtron-labs/devtron/client/dashboard" "github.com/devtron-labs/devtron/client/grafana" - "github.com/devtron-labs/devtron/client/telemetry" + telemetry2 "github.com/devtron-labs/devtron/client/telemetry" repository5 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/appStatus" @@ -130,6 +131,7 @@ import ( "github.com/devtron-labs/devtron/pkg/team/read" repository2 "github.com/devtron-labs/devtron/pkg/team/repository" "github.com/devtron-labs/devtron/pkg/terminal" + "github.com/devtron-labs/devtron/pkg/ucid" "github.com/devtron-labs/devtron/pkg/userResource" util3 "github.com/devtron-labs/devtron/pkg/util" "github.com/devtron-labs/devtron/pkg/webhook/helm" @@ -352,7 +354,7 @@ func InitializeApp() (*App, error) { return nil, err } deletePostProcessorImpl := service2.NewDeletePostProcessorImpl(sugaredLogger) - appStoreDeploymentServiceImpl := service2.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, installedAppDBServiceImpl, appStoreDeploymentDBServiceImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, appRepositoryImpl, eaModeDeploymentServiceImpl, eaModeDeploymentServiceImpl, environmentServiceImpl, helmAppServiceImpl, installedAppVersionHistoryRepositoryImpl, environmentVariables, acdConfig, gitOpsConfigReadServiceImpl, deletePostProcessorImpl, appStoreValidatorImpl, deploymentConfigServiceImpl) + appStoreDeploymentServiceImpl := service2.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, installedAppDBServiceImpl, appStoreDeploymentDBServiceImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, appRepositoryImpl, eaModeDeploymentServiceImpl, eaModeDeploymentServiceImpl, environmentServiceImpl, helmAppServiceImpl, installedAppVersionHistoryRepositoryImpl, environmentVariables, acdConfig, gitOpsConfigReadServiceImpl, deletePostProcessorImpl, appStoreValidatorImpl, deploymentConfigServiceImpl, ociRegistryConfigRepositoryImpl) fluxApplicationServiceImpl := fluxApplication.NewFluxApplicationServiceImpl(sugaredLogger, helmAppReadServiceImpl, clusterServiceImpl, helmAppClientImpl, pumpImpl) k8sResourceHistoryRepositoryImpl := repository10.NewK8sResourceHistoryRepositoryImpl(db, sugaredLogger) k8sResourceHistoryServiceImpl := kubernetesResourceAuditLogs.Newk8sResourceHistoryServiceImpl(k8sResourceHistoryRepositoryImpl, sugaredLogger, appRepositoryImpl, environmentRepositoryImpl) @@ -394,9 +396,10 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } + serviceImpl := ucid.NewServiceImpl(sugaredLogger, k8sServiceImpl, acdAuthConfig) providerIdentifierServiceImpl := providerIdentifier.NewProviderIdentifierServiceImpl(sugaredLogger) userAttributesRepositoryImpl := repository5.NewUserAttributesRepositoryImpl(db) - telemetryEventClientImpl, err := telemetry.NewTelemetryEventClientImpl(sugaredLogger, httpClient, clusterServiceImpl, k8sServiceImpl, acdAuthConfig, userServiceImpl, attributesRepositoryImpl, ssoLoginServiceImpl, posthogClient, moduleRepositoryImpl, serverDataStoreServerDataStore, userAuditServiceImpl, helmAppClientImpl, providerIdentifierServiceImpl, cronLoggerImpl, installedAppReadServiceEAImpl, environmentVariables, userAttributesRepositoryImpl) + telemetryEventClientImpl, err := telemetry2.NewTelemetryEventClientImpl(sugaredLogger, httpClient, clusterServiceImpl, k8sServiceImpl, acdAuthConfig, userServiceImpl, attributesRepositoryImpl, ssoLoginServiceImpl, posthogClient, serviceImpl, moduleRepositoryImpl, serverDataStoreServerDataStore, userAuditServiceImpl, helmAppClientImpl, providerIdentifierServiceImpl, cronLoggerImpl, installedAppReadServiceEAImpl, environmentVariables, userAttributesRepositoryImpl) if err != nil { return nil, err } diff --git a/go.mod b/go.mod index 7b92d82d38..0e23cf5e03 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/go-pg/pg v6.15.1+incompatible github.com/go-resty/resty/v2 v2.7.0 github.com/gogo/protobuf v1.3.2 - github.com/golang-jwt/jwt/v4 v4.5.1 + github.com/golang-jwt/jwt/v4 v4.5.2 github.com/golang/mock v1.6.0 github.com/golang/protobuf v1.5.4 github.com/google/cel-go v0.17.8 @@ -307,8 +307,8 @@ require ( replace ( github.com/argoproj/argo-workflows/v3 v3.5.13 => github.com/devtron-labs/argo-workflows/v3 v3.5.13 - github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250421131910-ad3aa9bb920e - github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421131910-ad3aa9bb920e + github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250422075757-fe1e5dd1b92b + github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250422075757-fe1e5dd1b92b github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127 github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5 k8s.io/api => k8s.io/api v0.29.7 diff --git a/go.sum b/go.sum index 328d6622fa..2824784a2a 100644 --- a/go.sum +++ b/go.sum @@ -829,10 +829,10 @@ github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzq github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/devtron-labs/argo-workflows/v3 v3.5.13 h1:3pINq0gXOSeTw2z/vYe+j80lRpSN5Rp/8mfQORh8SmU= github.com/devtron-labs/argo-workflows/v3 v3.5.13/go.mod h1:/vqxcovDPT4zqr4DjR5v7CF8ggpY1l3TSa2CIG3jmjA= -github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250421131910-ad3aa9bb920e h1:6/RNcoG39xH35ZHb024ERluk2HH3sf+uNWDR5AHb4xs= -github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250421131910-ad3aa9bb920e/go.mod h1:5lv4Wfj5ERhhvDGXe2IeES6qxjvUVCcohaRwKnWBMNo= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421131910-ad3aa9bb920e h1:YkUbwbsa5phIo0ikGvkCOHigF6IQZPfUTE9WcaF4vaM= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421131910-ad3aa9bb920e/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= +github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250422075757-fe1e5dd1b92b h1:b6AkVK1t2NNJcCusTF3w5UN9OwFMS6XEn7MUsSkraTY= +github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250422075757-fe1e5dd1b92b/go.mod h1:FfaLDXN1ZXxyRpnskBqVIYkpkWDCzBmDgIO9xqLnxdQ= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250422075757-fe1e5dd1b92b h1:PyI1pxuq7KiSbM2ObApJBcMSRkjXkMfNBZGzy9EJGQI= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250422075757-fe1e5dd1b92b/go.mod h1:zkNShlkcHxsmnL0gKNbs0uyRL8lZonGKr5Km63uTLI0= github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU= github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y= github.com/devtron-labs/protos v0.0.3-0.20250323220609-ecf8a0f7305e h1:U6UdYbW8a7xn5IzFPd8cywjVVPfutGJCudjePAfL/Hs= @@ -993,8 +993,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= -github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= diff --git a/internal/middleware/instrument.go b/internal/middleware/instrument.go index 0bbb8693e3..7fcc0d9ebf 100644 --- a/internal/middleware/instrument.go +++ b/internal/middleware/instrument.go @@ -117,6 +117,11 @@ var TerminalSessionDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Help: "duration of each terminal session", }, []string{"podName", "namespace", "clusterId"}) +var ReTriggerFailedCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "re_trigger_failed_counter", + Help: "ci/ pre cd/ post cd workflow re-trigger failed counter", +}, []string{"workflowType", "workflowId"}) + // prometheusMiddleware implements mux.MiddlewareFunc. func PrometheusMiddleware(next http.Handler) http.Handler { // prometheus.MustRegister(requestCounter) diff --git a/internal/sql/repository/pipelineConfig/CIPipelineRepository_ent.go b/internal/sql/repository/pipelineConfig/CIPipelineRepository_ent.go new file mode 100644 index 0000000000..3a1538c1f8 --- /dev/null +++ b/internal/sql/repository/pipelineConfig/CIPipelineRepository_ent.go @@ -0,0 +1,8 @@ +package pipelineConfig + +import "github.com/devtron-labs/devtron/pkg/bean/common" + +func (p *CiPipeline) GetWorkflowCacheConfig() common.WorkflowCacheConfigType { + //in oss, there is no pipeline level workflow cache config, so we pass inherit to get the app level config + return common.WorkflowCacheConfigInherit +} diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index a737a7076a..9569be90c6 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -20,7 +20,6 @@ import ( "context" "errors" apiBean "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" @@ -50,6 +49,7 @@ type CdWorkflowRepository interface { FindWorkflowRunnerByCdWorkflowId(wfIds []int) ([]*CdWorkflowRunner, error) FindPreviousCdWfRunnerByStatus(pipelineId int, currentWFRunnerId int, status []string) ([]*CdWorkflowRunner, error) FindWorkflowRunnerById(wfrId int) (*CdWorkflowRunner, error) + FindPreOrPostCdWorkflowRunnerById(wfrId int) (*CdWorkflowRunner, error) FindBasicWorkflowRunnerById(wfrId int) (*CdWorkflowRunner, error) FindRetriedWorkflowCountByReferenceId(wfrId int) (int, error) FindLatestWfrByAppIdAndEnvironmentId(appId int, environmentId int) (*CdWorkflowRunner, error) @@ -149,24 +149,6 @@ func (c *CdWorkflowRunner) IsExternalRun() bool { return isExtCluster } -type CiPipelineMaterialResponse struct { - Id int `json:"id"` - GitMaterialId int `json:"gitMaterialId"` - GitMaterialUrl string `json:"gitMaterialUrl"` - GitMaterialName string `json:"gitMaterialName"` - Type string `json:"type"` - Value string `json:"value"` - Active bool `json:"active"` - History []*gitSensor.GitCommit `json:"history,omitempty"` - LastFetchTime time.Time `json:"lastFetchTime"` - IsRepoError bool `json:"isRepoError"` - RepoErrorMsg string `json:"repoErrorMsg"` - IsBranchError bool `json:"isBranchError"` - BranchErrorMsg string `json:"branchErrorMsg"` - Url string `json:"url"` - Regex string `json:"regex"` -} - type TriggerWorkflowStatus struct { CdWorkflowStatus []*CdWorkflowStatus `json:"cdWorkflowStatus"` CiWorkflowStatus []*CiWorkflowStatus `json:"ciWorkflowStatus"` @@ -457,7 +439,6 @@ func (impl *CdWorkflowRepositoryImpl) SaveWorkFlowRunnerWithTx(wfr *CdWorkflowRu } func (impl *CdWorkflowRepositoryImpl) UpdateWorkFlowRunnerWithTx(wfr *CdWorkflowRunner, tx *pg.Tx) error { - wfr.Message = util.GetTruncatedMessage(wfr.Message, 1000) err := tx.Update(wfr) return err } @@ -528,6 +509,16 @@ func (impl *CdWorkflowRepositoryImpl) FindWorkflowRunnerById(wfrId int) (*CdWork return wfr, err } +func (impl *CdWorkflowRepositoryImpl) FindPreOrPostCdWorkflowRunnerById(wfrId int) (*CdWorkflowRunner, error) { + wfr := &CdWorkflowRunner{} + err := impl.dbConnection.Model(wfr). + Column("cd_workflow_runner.*", "CdWorkflow", "CdWorkflow.Pipeline", "CdWorkflow.CiArtifact", "CdWorkflow.Pipeline.Environment"). + Where("cd_workflow_runner.id = ?", wfrId). + Where("cd_workflow_runner.workflow_type != ?", apiBean.CD_WORKFLOW_TYPE_DEPLOY). + Select() + return wfr, err +} + func (impl *CdWorkflowRepositoryImpl) FindBasicWorkflowRunnerById(wfrId int) (*CdWorkflowRunner, error) { wfr := &CdWorkflowRunner{} err := impl.dbConnection.Model(wfr). diff --git a/internal/sql/repository/pipelineConfig/CiPipelineRepository.go b/internal/sql/repository/pipelineConfig/CiPipelineRepository.go index fb90f43255..c6adb8b7a8 100644 --- a/internal/sql/repository/pipelineConfig/CiPipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/CiPipelineRepository.go @@ -21,8 +21,8 @@ import ( "fmt" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/ciPipeline" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" - "github.com/devtron-labs/devtron/pkg/pipeline/constants" "github.com/devtron-labs/devtron/pkg/sql" "github.com/devtron-labs/devtron/util/response/pagination" "github.com/go-pg/pg" @@ -641,7 +641,7 @@ func (impl *CiPipelineRepositoryImpl) FindLinkedCiCount(ciPipelineId int) (int, pipeline := &CiPipeline{} cnt, err := impl.dbConnection.Model(pipeline). Where("parent_ci_pipeline = ?", ciPipelineId). - Where("ci_pipeline_type != ?", constants.LINKED_CD). + Where("ci_pipeline_type != ?", buildCommonBean.LINKED_CD). Where("deleted = ?", false). Count() if err == pg.ErrNoRows { @@ -656,7 +656,7 @@ func (impl *CiPipelineRepositoryImpl) GetLinkedCiPipelines(ctx context.Context, var linkedCIPipelines []*CiPipeline err := impl.dbConnection.Model(&linkedCIPipelines). Where("parent_ci_pipeline = ?", ciPipelineId). - Where("ci_pipeline_type != ?", constants.LINKED_CD). + Where("ci_pipeline_type != ?", buildCommonBean.LINKED_CD). Where("deleted = ?", false). Select() if err != nil { @@ -694,7 +694,7 @@ func (impl *CiPipelineRepositoryImpl) GetDownStreamInfo(ctx context.Context, sou JoinOn("e.active = ?", true). // constrains Where("ci_pipeline.parent_ci_pipeline = ?", sourceCiPipelineId). - Where("ci_pipeline.ci_pipeline_type != ?", constants.LINKED_CD). + Where("ci_pipeline.ci_pipeline_type != ?", buildCommonBean.LINKED_CD). Where("ci_pipeline.deleted = ?", false) // app name filtering with lower case if len(appNameMatch) != 0 { diff --git a/internal/sql/repository/pipelineConfig/bean/cdWorkflow/bean.go b/internal/sql/repository/pipelineConfig/bean/cdWorkflow/bean.go deleted file mode 100644 index 645ef52eb2..0000000000 --- a/internal/sql/repository/pipelineConfig/bean/cdWorkflow/bean.go +++ /dev/null @@ -1,54 +0,0 @@ -package cdWorkflow - -import ( - "errors" - "github.com/devtron-labs/common-lib/utils/k8s/health" - "github.com/devtron-labs/devtron/client/argocdServer/bean" -) - -var WfrTerminalStatusList = []string{WorkflowAborted, WorkflowFailed, WorkflowSucceeded, bean.HIBERNATING, string(health.HealthStatusHealthy), string(health.HealthStatusDegraded)} - -type WorkflowStatus int - -const ( - WF_UNKNOWN WorkflowStatus = iota - REQUEST_ACCEPTED - ENQUEUED - QUE_ERROR - WF_STARTED - DROPPED_STALE - DEQUE_ERROR - TRIGGER_ERROR -) - -const ( - WorkflowStarting = "Starting" - WorkflowInQueue = "Queued" - WorkflowInitiated = "Initiating" - WorkflowInProgress = "Progressing" - WorkflowAborted = "Aborted" - WorkflowFailed = "Failed" - WorkflowSucceeded = "Succeeded" - WorkflowTimedOut = "TimedOut" - WorkflowUnableToFetchState = "UnableToFetch" - WorkflowTypeDeploy = "DEPLOY" - WorkflowTypePre = "PRE" - WorkflowTypePost = "POST" -) - -func (a WorkflowStatus) String() string { - return [...]string{"WF_UNKNOWN", "REQUEST_ACCEPTED", "ENQUEUED", "QUE_ERROR", "WF_STARTED", "DROPPED_STALE", "DEQUE_ERROR", "TRIGGER_ERROR"}[a] -} - -var ErrorDeploymentSuperseded = errors.New(NEW_DEPLOYMENT_INITIATED) - -const ( - WORKFLOW_EXECUTOR_TYPE_AWF = "AWF" - WORKFLOW_EXECUTOR_TYPE_SYSTEM = "SYSTEM" - NEW_DEPLOYMENT_INITIATED = "A new deployment was initiated before this deployment completed!" - PIPELINE_DELETED = "The pipeline has been deleted!" - FOUND_VULNERABILITY = "Found vulnerability on image" - GITOPS_REPO_NOT_CONFIGURED = "GitOps repository is not configured for the app" -) - -type WorkflowExecutorType string diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 48c5728c91..a5d5654ff2 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -26,13 +26,13 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" read4 "github.com/devtron-labs/devtron/pkg/app/appDetails/read" userrepository "github.com/devtron-labs/devtron/pkg/auth/user/repository" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" ciConfig "github.com/devtron-labs/devtron/pkg/build/pipeline/read" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/read" "github.com/devtron-labs/devtron/pkg/dockerRegistry" - "github.com/devtron-labs/devtron/pkg/pipeline/constants" errors2 "github.com/juju/errors" "go.opentelemetry.io/otel" "golang.org/x/exp/slices" @@ -652,7 +652,7 @@ func (impl AppListingServiceImpl) setIpAccessProvidedData(ctx context.Context, a } if ciPipeline != nil && ciPipeline.CiTemplate != nil && len(*ciPipeline.CiTemplate.DockerRegistryId) > 0 { - if !ciPipeline.IsExternal || ciPipeline.ParentCiPipeline != 0 && ciPipeline.PipelineType != string(constants.LINKED_CD) { + if !ciPipeline.IsExternal || ciPipeline.ParentCiPipeline != 0 && ciPipeline.PipelineType != string(buildCommonBean.LINKED_CD) { appDetailContainer.IsExternalCi = false } // get dockerRegistryId starts diff --git a/pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppArgoCdService.go b/pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppArgoCdService.go index 9b637360f0..05c6b07ef2 100644 --- a/pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppArgoCdService.go +++ b/pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppArgoCdService.go @@ -25,7 +25,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" "github.com/devtron-labs/devtron/internal/util" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" - cluster2 "github.com/devtron-labs/devtron/pkg/cluster" + bean2 "github.com/devtron-labs/devtron/pkg/cluster/bean" "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" util2 "github.com/devtron-labs/devtron/util" @@ -167,7 +167,7 @@ func (impl *FullModeDeploymentServiceImpl) DeleteACD(acdAppName string, ctx cont func (impl *FullModeDeploymentServiceImpl) createInArgo(ctx context.Context, chartGitAttribute *commonBean.ChartGitAttribute, envModel bean.EnvironmentBean, argocdAppName string) error { appNamespace := envModel.Namespace if appNamespace == "" { - appNamespace = cluster2.DEFAULT_NAMESPACE + appNamespace = bean2.DefaultNamespace } appReq := &argocdServer.AppTemplate{ ApplicationName: argocdAppName, diff --git a/pkg/bean/app.go b/pkg/bean/app.go index 96c581b5f6..83c6717093 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -29,6 +29,7 @@ import ( "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/bean/common" CiPipeline2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + common2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" "github.com/devtron-labs/devtron/pkg/chartRepo/repository" bean3 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/pipeline/bean" @@ -117,39 +118,39 @@ type CiMaterial struct { } type CiPipeline struct { - IsManual bool `json:"isManual"` - DockerArgs map[string]string `json:"dockerArgs"` - IsExternal bool `json:"isExternal"` - ParentCiPipeline int `json:"parentCiPipeline"` - ParentAppId int `json:"parentAppId"` - AppId int `json:"appId"` - AppName string `json:"appName,omitempty"` - AppType helper.AppType `json:"appType,omitempty"` - ExternalCiConfig ExternalCiConfig `json:"externalCiConfig"` - CiMaterial []*CiMaterial `json:"ciMaterial,omitempty" validate:"dive,min=1"` - Name string `json:"name,omitempty" validate:"name-component,max=100"` // name suffix of corresponding pipeline. required, unique, validation corresponding to gocd pipelineName will be applicable - Id int `json:"id,omitempty" ` - Version string `json:"version,omitempty"` // matchIf token version in gocd . used for update request - Active bool `json:"active,omitempty"` // pipeline is active or not - Deleted bool `json:"deleted,omitempty"` - BeforeDockerBuild []*Task `json:"beforeDockerBuild,omitempty" validate:"dive"` - AfterDockerBuild []*Task `json:"afterDockerBuild,omitempty" validate:"dive"` - BeforeDockerBuildScripts []*CiScript `json:"beforeDockerBuildScripts,omitempty" validate:"dive"` - AfterDockerBuildScripts []*CiScript `json:"afterDockerBuildScripts,omitempty" validate:"dive"` - LinkedCount int `json:"linkedCount"` - PipelineType CiPipeline2.PipelineType `json:"pipelineType,omitempty"` - ScanEnabled bool `json:"scanEnabled,notnull"` - AppWorkflowId int `json:"appWorkflowId,omitempty"` - PreBuildStage *bean.PipelineStageDto `json:"preBuildStage,omitempty" validate:"omitempty,dive"` - PostBuildStage *bean.PipelineStageDto `json:"postBuildStage,omitempty" validate:"omitempty,dive"` - TargetPlatform string `json:"targetPlatform,omitempty"` - IsDockerConfigOverridden bool `json:"isDockerConfigOverridden"` - DockerConfigOverride DockerConfigOverride `json:"dockerConfigOverride,omitempty"` - EnvironmentId int `json:"environmentId,omitempty"` - LastTriggeredEnvId int `json:"lastTriggeredEnvId"` - CustomTagObject *CustomTagData `json:"customTag,omitempty"` - DefaultTag []string `json:"defaultTag,omitempty"` - EnableCustomTag bool `json:"enableCustomTag"` + IsManual bool `json:"isManual"` + DockerArgs map[string]string `json:"dockerArgs"` + IsExternal bool `json:"isExternal"` + ParentCiPipeline int `json:"parentCiPipeline"` + ParentAppId int `json:"parentAppId"` + AppId int `json:"appId"` + AppName string `json:"appName,omitempty"` + AppType helper.AppType `json:"appType,omitempty"` + ExternalCiConfig ExternalCiConfig `json:"externalCiConfig"` + CiMaterial []*CiMaterial `json:"ciMaterial,omitempty" validate:"dive,min=1"` + Name string `json:"name,omitempty" validate:"name-component,max=100"` // name suffix of corresponding pipeline. required, unique, validation corresponding to gocd pipelineName will be applicable + Id int `json:"id,omitempty" ` + Version string `json:"version,omitempty"` // matchIf token version in gocd . used for update request + Active bool `json:"active,omitempty"` // pipeline is active or not + Deleted bool `json:"deleted,omitempty"` + BeforeDockerBuild []*Task `json:"beforeDockerBuild,omitempty" validate:"dive"` + AfterDockerBuild []*Task `json:"afterDockerBuild,omitempty" validate:"dive"` + BeforeDockerBuildScripts []*CiScript `json:"beforeDockerBuildScripts,omitempty" validate:"dive"` + AfterDockerBuildScripts []*CiScript `json:"afterDockerBuildScripts,omitempty" validate:"dive"` + LinkedCount int `json:"linkedCount"` + PipelineType common2.PipelineType `json:"pipelineType,omitempty"` + ScanEnabled bool `json:"scanEnabled,notnull"` + AppWorkflowId int `json:"appWorkflowId,omitempty"` + PreBuildStage *bean.PipelineStageDto `json:"preBuildStage,omitempty" validate:"omitempty,dive"` + PostBuildStage *bean.PipelineStageDto `json:"postBuildStage,omitempty" validate:"omitempty,dive"` + TargetPlatform string `json:"targetPlatform,omitempty"` + IsDockerConfigOverridden bool `json:"isDockerConfigOverridden"` + DockerConfigOverride DockerConfigOverride `json:"dockerConfigOverride,omitempty"` + EnvironmentId int `json:"environmentId,omitempty"` + LastTriggeredEnvId int `json:"lastTriggeredEnvId"` + CustomTagObject *CustomTagData `json:"customTag,omitempty"` + DefaultTag []string `json:"defaultTag,omitempty"` + EnableCustomTag bool `json:"enableCustomTag"` } func (ciPipeline *CiPipeline) IsLinkedCi() bool { @@ -164,14 +165,14 @@ type DockerConfigOverride struct { } type CiPipelineMin struct { - Name string `json:"name,omitempty" validate:"name-component,max=100"` //name suffix of corresponding pipeline. required, unique, validation corresponding to gocd pipelineName will be applicable - Id int `json:"id,omitempty" ` - Version string `json:"version,omitempty"` //matchIf token version in gocd . used for update request - IsExternal bool `json:"isExternal,omitempty"` - ParentCiPipeline int `json:"parentCiPipeline"` - ParentAppId int `json:"parentAppId"` - PipelineType CiPipeline2.PipelineType `json:"pipelineType,omitempty"` - ScanEnabled bool `json:"scanEnabled,notnull"` + Name string `json:"name,omitempty" validate:"name-component,max=100"` //name suffix of corresponding pipeline. required, unique, validation corresponding to gocd pipelineName will be applicable + Id int `json:"id,omitempty" ` + Version string `json:"version,omitempty"` //matchIf token version in gocd . used for update request + IsExternal bool `json:"isExternal,omitempty"` + ParentCiPipeline int `json:"parentCiPipeline"` + ParentAppId int `json:"parentAppId"` + PipelineType common2.PipelineType `json:"pipelineType,omitempty"` + ScanEnabled bool `json:"scanEnabled,notnull"` } type CiScript struct { @@ -303,20 +304,20 @@ type CiPatchRequest struct { IsJob bool `json:"-"` IsCloneJob bool `json:"isCloneJob,omitempty"` - ParentCDPipeline int `json:"parentCDPipeline"` - DeployEnvId int `json:"deployEnvId"` - SwitchFromCiPipelineId int `json:"switchFromCiPipelineId"` - SwitchFromExternalCiPipelineId int `json:"switchFromExternalCiPipelineId"` - SwitchFromCiPipelineType CiPipeline2.PipelineType `json:"-"` - SwitchToCiPipelineType CiPipeline2.PipelineType `json:"-"` + ParentCDPipeline int `json:"parentCDPipeline"` + DeployEnvId int `json:"deployEnvId"` + SwitchFromCiPipelineId int `json:"switchFromCiPipelineId"` + SwitchFromExternalCiPipelineId int `json:"switchFromExternalCiPipelineId"` + SwitchFromCiPipelineType common2.PipelineType `json:"-"` + SwitchToCiPipelineType common2.PipelineType `json:"-"` } -func (ciPatchRequest CiPatchRequest) SwitchSourceInfo() (int, CiPipeline2.PipelineType) { +func (ciPatchRequest CiPatchRequest) SwitchSourceInfo() (int, common2.PipelineType) { // get the ciPipeline - var switchFromType CiPipeline2.PipelineType + var switchFromType common2.PipelineType var switchFromPipelineId int if ciPatchRequest.SwitchFromExternalCiPipelineId != 0 { - switchFromType = CiPipeline2.EXTERNAL + switchFromType = common2.EXTERNAL switchFromPipelineId = ciPatchRequest.SwitchFromExternalCiPipelineId } else { switchFromPipelineId = ciPatchRequest.SwitchFromCiPipelineId diff --git a/pkg/build/git/gitWebhook/GitWebhookService.go b/pkg/build/git/gitWebhook/GitWebhookService.go index 3f301db311..7ef9e0edc6 100644 --- a/pkg/build/git/gitWebhook/GitWebhookService.go +++ b/pkg/build/git/gitWebhook/GitWebhookService.go @@ -23,7 +23,7 @@ import ( bean2 "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/build/git/gitWebhook/repository" - "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/build/trigger" "go.uber.org/zap" ) @@ -33,15 +33,16 @@ type GitWebhookService interface { type GitWebhookServiceImpl struct { logger *zap.SugaredLogger - ciHandler pipeline.CiHandler gitWebhookRepository repository.GitWebhookRepository + ciHandlerService trigger.HandlerService } -func NewGitWebhookServiceImpl(Logger *zap.SugaredLogger, ciHandler pipeline.CiHandler, gitWebhookRepository repository.GitWebhookRepository) *GitWebhookServiceImpl { +func NewGitWebhookServiceImpl(Logger *zap.SugaredLogger, gitWebhookRepository repository.GitWebhookRepository, + ciHandlerService trigger.HandlerService) *GitWebhookServiceImpl { return &GitWebhookServiceImpl{ logger: Logger, - ciHandler: ciHandler, gitWebhookRepository: gitWebhookRepository, + ciHandlerService: ciHandlerService, } } @@ -70,7 +71,7 @@ func (impl *GitWebhookServiceImpl) HandleGitWebhook(gitWebhookRequest gitSensor. } } - resp, err := impl.ciHandler.HandleCIWebhook(bean.GitCiTriggerRequest{ + resp, err := impl.ciHandlerService.HandleCIWebhook(bean.GitCiTriggerRequest{ CiPipelineMaterial: ciPipelineMaterial, TriggeredBy: bean2.SYSTEM_USER_ID, // Automatic trigger, system user ExtraEnvironmentVariables: gitWebhookRequest.ExtraEnvironmentVariables, diff --git a/pkg/build/pipeline/bean/CIPipelineMaterialBean.go b/pkg/build/pipeline/bean/CIPipelineMaterialBean.go new file mode 100644 index 0000000000..8f09c47ebe --- /dev/null +++ b/pkg/build/pipeline/bean/CIPipelineMaterialBean.go @@ -0,0 +1,30 @@ +package bean + +import ( + "github.com/devtron-labs/devtron/client/gitSensor" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "time" +) + +type CiPipelineMaterialResponse struct { + Id int `json:"id"` + GitMaterialId int `json:"gitMaterialId"` + GitMaterialUrl string `json:"gitMaterialUrl"` + GitMaterialName string `json:"gitMaterialName"` + Type string `json:"type"` + Value string `json:"value"` + Active bool `json:"active"` + History []*gitSensor.GitCommit `json:"history,omitempty"` + LastFetchTime time.Time `json:"lastFetchTime"` + IsRepoError bool `json:"isRepoError"` + RepoErrorMsg string `json:"repoErrorMsg"` + IsBranchError bool `json:"isBranchError"` + BranchErrorMsg string `json:"branchErrorMsg"` + Url string `json:"url"` + Regex string `json:"regex"` +} + +type MaterialTriggerInfo struct { + GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` + CiMaterials []CiPipelineMaterialResponse `json:"ciMaterials"` +} diff --git a/pkg/build/pipeline/bean/CiBuildConfig.go b/pkg/build/pipeline/bean/CiBuildConfig.go index 34357f5e68..39d8671a35 100644 --- a/pkg/build/pipeline/bean/CiBuildConfig.go +++ b/pkg/build/pipeline/bean/CiBuildConfig.go @@ -30,23 +30,6 @@ const UniquePlaceHolderForAppName = "$etron" const PIPELINE_NAME_ALREADY_EXISTS_ERROR = "pipeline name already exist" const PIPELINE_TYPE_IS_NOT_VALID = "PipelineType is not valid for pipeline %s" -type PipelineType string - -// default PipelineType -const DefaultPipelineType = CI_BUILD - -const ( - CI_BUILD PipelineType = "CI_BUILD" - LINKED PipelineType = "LINKED" - EXTERNAL PipelineType = "EXTERNAL" - CI_JOB PipelineType = "CI_JOB" - LINKED_CD PipelineType = "LINKED_CD" -) - -func (pType PipelineType) ToString() string { - return string(pType) -} - type CiBuildConfigBean struct { Id int `json:"id"` GitMaterialId int `json:"gitMaterialId,omitempty" validate:"required"` @@ -81,15 +64,6 @@ type BuildPackConfig struct { ProjectPath string `json:"projectPath,omitempty"` } -func (pType PipelineType) IsValidPipelineType() bool { - switch pType { - case CI_BUILD, LINKED, EXTERNAL, CI_JOB, LINKED_CD: - return true - default: - return false - } -} - const ( ExtraEnvVarExternalCiArtifactKey = "externalCiArtifact" ExtraEnvVarImageDigestKey = "imageDigest" diff --git a/pkg/build/pipeline/bean/common/CommonBuildBean.go b/pkg/build/pipeline/bean/common/CommonBuildBean.go new file mode 100644 index 0000000000..81692c377a --- /dev/null +++ b/pkg/build/pipeline/bean/common/CommonBuildBean.go @@ -0,0 +1,40 @@ +package common + +type PipelineType string + +// default PipelineType +const DefaultPipelineType = CI_BUILD + +const ( + CI_BUILD PipelineType = "CI_BUILD" + LINKED PipelineType = "LINKED" + EXTERNAL PipelineType = "EXTERNAL" + CI_JOB PipelineType = "CI_JOB" + LINKED_CD PipelineType = "LINKED_CD" +) + +func (pType PipelineType) ToString() string { + return string(pType) +} + +func (pType PipelineType) IsValidPipelineType() bool { + switch pType { + case CI_BUILD, LINKED, EXTERNAL, CI_JOB, LINKED_CD: + return true + default: + return false + } +} + +const ( + BEFORE_DOCKER_BUILD = "BEFORE_DOCKER_BUILD" + AFTER_DOCKER_BUILD = "AFTER_DOCKER_BUILD" +) + +type RefPluginName = string + +const ( + COPY_CONTAINER_IMAGE RefPluginName = "Copy container image" + COPY_CONTAINER_IMAGE_VERSION_V1 = "1.0.0" + COPY_CONTAINER_IMAGE_VERSION_V2 = "2.0.0" +) diff --git a/pkg/build/trigger/HandlerService.go b/pkg/build/trigger/HandlerService.go new file mode 100644 index 0000000000..7b5832ffa4 --- /dev/null +++ b/pkg/build/trigger/HandlerService.go @@ -0,0 +1,1957 @@ +package trigger + +import ( + "bufio" + "context" + "errors" + "fmt" + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + "github.com/caarlos0/env" + blob_storage "github.com/devtron-labs/common-lib/blob-storage" + "github.com/devtron-labs/common-lib/utils" + bean4 "github.com/devtron-labs/common-lib/utils/bean" + "github.com/devtron-labs/common-lib/utils/k8s" + commonBean "github.com/devtron-labs/common-lib/workflow" + client "github.com/devtron-labs/devtron/client/events" + "github.com/devtron-labs/devtron/client/gitSensor" + "github.com/devtron-labs/devtron/internal/middleware" + "github.com/devtron-labs/devtron/internal/sql/constants" + repository5 "github.com/devtron-labs/devtron/internal/sql/repository" + appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" + repository3 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" + "github.com/devtron-labs/devtron/internal/sql/repository/helper" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" + "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/app" + "github.com/devtron-labs/devtron/pkg/attributes" + bean3 "github.com/devtron-labs/devtron/pkg/attributes/bean" + "github.com/devtron-labs/devtron/pkg/auth/user" + bean6 "github.com/devtron-labs/devtron/pkg/auth/user/bean" + "github.com/devtron-labs/devtron/pkg/bean" + "github.com/devtron-labs/devtron/pkg/bean/common" + "github.com/devtron-labs/devtron/pkg/build/pipeline" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" + "github.com/devtron-labs/devtron/pkg/cluster" + adapter2 "github.com/devtron-labs/devtron/pkg/cluster/adapter" + clusterBean "github.com/devtron-labs/devtron/pkg/cluster/bean" + "github.com/devtron-labs/devtron/pkg/cluster/environment" + repository6 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" + eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" + "github.com/devtron-labs/devtron/pkg/executor" + pipeline2 "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/pipeline/adapter" + pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" + constants2 "github.com/devtron-labs/devtron/pkg/pipeline/constants" + "github.com/devtron-labs/devtron/pkg/pipeline/executors" + "github.com/devtron-labs/devtron/pkg/pipeline/repository" + "github.com/devtron-labs/devtron/pkg/pipeline/types" + util2 "github.com/devtron-labs/devtron/pkg/pipeline/util" + "github.com/devtron-labs/devtron/pkg/plugin" + bean2 "github.com/devtron-labs/devtron/pkg/plugin/bean" + repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository" + "github.com/devtron-labs/devtron/pkg/resourceQualifiers" + "github.com/devtron-labs/devtron/pkg/variables" + repository4 "github.com/devtron-labs/devtron/pkg/variables/repository" + "github.com/devtron-labs/devtron/util/sliceUtil" + "github.com/go-pg/pg" + "go.uber.org/zap" + "io/ioutil" + "k8s.io/client-go/rest" + "net/http" + "os" + "path" + "path/filepath" + "slices" + "strconv" + "strings" + "time" +) + +type HandlerService interface { + HandlePodDeleted(ciWorkflow *pipelineConfig.CiWorkflow) + CheckAndReTriggerCI(workflowStatus eventProcessorBean.CiCdStatus) error + HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) + HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) + + StartCiWorkflowAndPrepareWfRequest(trigger types.Trigger) (*pipelineConfig.CiPipeline, map[string]string, *pipelineConfig.CiWorkflow, *types.WorkflowRequest, error) + + CancelBuild(workflowId int, forceAbort bool) (int, error) + GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, func() error, error) + GetHistoricBuildLogs(workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) + DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) +} + +type BuildxCacheFlags struct { + BuildxCacheModeMin bool `env:"BUILDX_CACHE_MODE_MIN" envDefault:"false" description:"To set build cache mode to minimum in buildx" ` + AsyncBuildxCacheExport bool `env:"ASYNC_BUILDX_CACHE_EXPORT" envDefault:"false" description:"To enable async container image cache export"` +} + +type HandlerServiceImpl struct { + Logger *zap.SugaredLogger + workflowService executor.WorkflowService + ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository + ciPipelineRepository pipelineConfig.CiPipelineRepository + ciArtifactRepository repository5.CiArtifactRepository + pipelineStageService pipeline2.PipelineStageService + userService user.UserService + ciTemplateService pipeline.CiTemplateReadService + appCrudOperationService app.AppCrudOperationService + envRepository repository6.EnvironmentRepository + appRepository appRepository.AppRepository + customTagService pipeline2.CustomTagService + config *types.CiConfig + scopedVariableManager variables.ScopedVariableManager + ciCdPipelineOrchestrator pipeline2.CiCdPipelineOrchestrator + buildxCacheFlags *BuildxCacheFlags + attributeService attributes.AttributesService + pluginInputVariableParser pipeline2.PluginInputVariableParser + globalPluginService plugin.GlobalPluginService + ciService pipeline2.CiService + ciWorkflowRepository pipelineConfig.CiWorkflowRepository + gitSensorClient gitSensor.Client + ciLogService pipeline2.CiLogService + blobConfigStorageService pipeline2.BlobStorageConfigService + clusterService cluster.ClusterService + envService environment.EnvironmentService + K8sUtil *k8s.K8sServiceImpl +} + +func NewHandlerServiceImpl(Logger *zap.SugaredLogger, workflowService executor.WorkflowService, + ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, + ciPipelineRepository pipelineConfig.CiPipelineRepository, + ciArtifactRepository repository5.CiArtifactRepository, + pipelineStageService pipeline2.PipelineStageService, + userService user.UserService, + ciTemplateService pipeline.CiTemplateReadService, + appCrudOperationService app.AppCrudOperationService, + envRepository repository6.EnvironmentRepository, + appRepository appRepository.AppRepository, + scopedVariableManager variables.ScopedVariableManager, + customTagService pipeline2.CustomTagService, + ciCdPipelineOrchestrator pipeline2.CiCdPipelineOrchestrator, attributeService attributes.AttributesService, + pluginInputVariableParser pipeline2.PluginInputVariableParser, + globalPluginService plugin.GlobalPluginService, + ciService pipeline2.CiService, + ciWorkflowRepository pipelineConfig.CiWorkflowRepository, + gitSensorClient gitSensor.Client, + ciLogService pipeline2.CiLogService, + blobConfigStorageService pipeline2.BlobStorageConfigService, + clusterService cluster.ClusterService, + envService environment.EnvironmentService, + K8sUtil *k8s.K8sServiceImpl, +) *HandlerServiceImpl { + buildxCacheFlags := &BuildxCacheFlags{} + err := env.Parse(buildxCacheFlags) + if err != nil { + Logger.Infow("error occurred while parsing BuildxCacheFlags env,so setting BuildxCacheModeMin and AsyncBuildxCacheExport to default value", "err", err) + } + cis := &HandlerServiceImpl{ + Logger: Logger, + workflowService: workflowService, + ciPipelineMaterialRepository: ciPipelineMaterialRepository, + ciPipelineRepository: ciPipelineRepository, + ciArtifactRepository: ciArtifactRepository, + pipelineStageService: pipelineStageService, + userService: userService, + ciTemplateService: ciTemplateService, + appCrudOperationService: appCrudOperationService, + envRepository: envRepository, + appRepository: appRepository, + scopedVariableManager: scopedVariableManager, + customTagService: customTagService, + ciCdPipelineOrchestrator: ciCdPipelineOrchestrator, + buildxCacheFlags: buildxCacheFlags, + attributeService: attributeService, + pluginInputVariableParser: pluginInputVariableParser, + globalPluginService: globalPluginService, + ciService: ciService, + ciWorkflowRepository: ciWorkflowRepository, + gitSensorClient: gitSensorClient, + ciLogService: ciLogService, + blobConfigStorageService: blobConfigStorageService, + clusterService: clusterService, + envService: envService, + K8sUtil: K8sUtil, + } + config, err := types.GetCiConfig() + if err != nil { + return nil + } + cis.config = config + return cis +} + +func (impl *HandlerServiceImpl) HandlePodDeleted(ciWorkflow *pipelineConfig.CiWorkflow) { + if !impl.config.WorkflowRetriesEnabled() { + impl.Logger.Debug("ci workflow retry feature disabled") + return + } + retryCount, refCiWorkflow, err := impl.getRefWorkflowAndCiRetryCount(ciWorkflow) + if err != nil { + impl.Logger.Errorw("error in getRefWorkflowAndCiRetryCount", "ciWorkflowId", ciWorkflow.Id, "err", err) + } + impl.Logger.Infow("re-triggering ci by UpdateCiWorkflowStatusFailedCron", "refCiWorkflowId", refCiWorkflow.Id, "ciWorkflow.Status", ciWorkflow.Status, "ciWorkflow.Message", ciWorkflow.Message, "retryCount", retryCount) + err = impl.reTriggerCi(retryCount, refCiWorkflow) + if err != nil { + impl.Logger.Errorw("error in reTriggerCi", "ciWorkflowId", refCiWorkflow.Id, "workflowStatus", ciWorkflow.Status, "ciWorkflowMessage", "ciWorkflow.Message", "retryCount", retryCount, "err", err) + } +} + +func (impl *HandlerServiceImpl) CheckAndReTriggerCI(workflowStatus eventProcessorBean.CiCdStatus) error { + + // return if re-trigger feature is disabled + if !impl.config.WorkflowRetriesEnabled() { + impl.Logger.Debug("CI re-trigger is disabled") + return nil + } + + status, message, ciWorkFlow, err := impl.extractPodStatusAndWorkflow(workflowStatus) + if err != nil { + impl.Logger.Errorw("error in extractPodStatusAndWorkflow", "err", err) + return err + } + + if !executors.CheckIfReTriggerRequired(status, message, ciWorkFlow.Status) { + impl.Logger.Debugw("not re-triggering ci", "status", status, "message", message, "ciWorkflowStatus", ciWorkFlow.Status) + return nil + } + + impl.Logger.Debugw("re-triggering ci", "status", status, "message", message, "ciWorkflowStatus", ciWorkFlow.Status, "ciWorkFlowId", ciWorkFlow.Id) + + retryCount, refCiWorkflow, err := impl.getRefWorkflowAndCiRetryCount(ciWorkFlow) + if err != nil { + impl.Logger.Errorw("error while getting retry count value for a ciWorkflow", "ciWorkFlowId", ciWorkFlow.Id) + return err + } + + err = impl.reTriggerCi(retryCount, refCiWorkflow) + if err != nil { + impl.Logger.Errorw("error in reTriggerCi", "err", err, "status", status, "message", message, "retryCount", retryCount, "ciWorkFlowId", ciWorkFlow.Id) + } + return err +} + +func (impl *HandlerServiceImpl) reTriggerCi(retryCount int, refCiWorkflow *pipelineConfig.CiWorkflow) error { + if retryCount >= impl.config.MaxCiWorkflowRetries { + impl.Logger.Infow("maximum retries exhausted for this ciWorkflow", "ciWorkflowId", refCiWorkflow.Id, "retries", retryCount, "configuredRetries", impl.config.MaxCiWorkflowRetries) + return nil + } + impl.Logger.Infow("re-triggering ci for a ci workflow", "ReferenceCiWorkflowId", refCiWorkflow.Id) + ciPipelineMaterialIds := make([]int, 0, len(refCiWorkflow.GitTriggers)) + for id, _ := range refCiWorkflow.GitTriggers { + ciPipelineMaterialIds = append(ciPipelineMaterialIds, id) + } + ciMaterials, err := impl.ciPipelineMaterialRepository.GetByIdsIncludeDeleted(ciPipelineMaterialIds) + if err != nil { + impl.Logger.Errorw("error in getting ci Pipeline Materials using ciPipeline Material Ids", "ciPipelineMaterialIds", ciPipelineMaterialIds, "err", err) + return err + } + + trigger := types.Trigger{} + trigger.BuildTriggerObject(refCiWorkflow, ciMaterials, bean6.SYSTEM_USER_ID, true, nil, "") + + // updating runtime params + trigger.RuntimeParameters, err = impl.updateRuntimeParamsForAutoCI(trigger.PipelineId, trigger.RuntimeParameters) + if err != nil { + impl.Logger.Errorw("err, updateRuntimeParamsForAutoCI", "ciPipelineId", trigger.PipelineId, + "runtimeParameters", trigger.RuntimeParameters, "err", err) + return err + } + _, err = impl.triggerCiPipeline(trigger) + + if err != nil { + impl.Logger.Errorw("error occurred in re-triggering ciWorkflow", "triggerDetails", trigger, "err", err) + return err + } + return nil +} + +func (impl *HandlerServiceImpl) HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) { + impl.Logger.Debugw("HandleCIManual for pipeline ", "PipelineId", ciTriggerRequest.PipelineId) + commitHashes, runtimeParams, err := impl.buildManualTriggerCommitHashes(ciTriggerRequest) + if err != nil { + return 0, err + } + + ciArtifact, err := impl.ciArtifactRepository.GetLatestArtifactTimeByCiPipelineId(ciTriggerRequest.PipelineId) + if err != nil && err != pg.ErrNoRows { + impl.Logger.Errorw("Error in GetLatestArtifactTimeByCiPipelineId", "err", err, "pipelineId", ciTriggerRequest.PipelineId) + return 0, err + } + + createdOn := time.Time{} + if err != pg.ErrNoRows { + createdOn = ciArtifact.CreatedOn + } + + trigger := types.Trigger{ + PipelineId: ciTriggerRequest.PipelineId, + CommitHashes: commitHashes, + CiMaterials: nil, + TriggeredBy: ciTriggerRequest.TriggeredBy, + InvalidateCache: ciTriggerRequest.InvalidateCache, + RuntimeParameters: runtimeParams, + EnvironmentId: ciTriggerRequest.EnvironmentId, + PipelineType: ciTriggerRequest.PipelineType, + CiArtifactLastFetch: createdOn, + } + id, err := impl.triggerCiPipeline(trigger) + + if err != nil { + return 0, err + } + return id, nil +} + +func (impl *HandlerServiceImpl) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) { + impl.Logger.Debugw("HandleCIWebhook for material ", "material", gitCiTriggerRequest.CiPipelineMaterial) + ciPipeline, err := impl.GetCiPipeline(gitCiTriggerRequest.CiPipelineMaterial.Id) + if err != nil { + impl.Logger.Errorw("err in getting ci_pipeline by ciPipelineMaterialId", "ciPipelineMaterialId", gitCiTriggerRequest.CiPipelineMaterial.Id, "err", err) + return 0, err + } + if ciPipeline.IsManual || ciPipeline.PipelineType == buildCommonBean.LINKED_CD.ToString() { + impl.Logger.Debugw("not handling for manual pipeline or in case of linked cd", "pipelineId", ciPipeline.Id) + return 0, err + } + + ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(ciPipeline.Id) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return 0, err + } + isValidBuildSequence, err := impl.validateBuildSequence(gitCiTriggerRequest, ciPipeline.Id) + if !isValidBuildSequence { + return 0, errors.New("ignoring older build for ciMaterial " + strconv.Itoa(gitCiTriggerRequest.CiPipelineMaterial.Id) + + " commit " + gitCiTriggerRequest.CiPipelineMaterial.GitCommit.Commit) + } + // updating runtime params + runtimeParams := common.NewRuntimeParameters() + for k, v := range gitCiTriggerRequest.ExtraEnvironmentVariables { + runtimeParams = runtimeParams.AddSystemVariable(k, v) + } + runtimeParams, err = impl.updateRuntimeParamsForAutoCI(ciPipeline.Id, runtimeParams) + if err != nil { + impl.Logger.Errorw("err, updateRuntimeParamsForAutoCI", "ciPipelineId", ciPipeline.Id, + "runtimeParameters", runtimeParams, "err", err) + return 0, err + } + commitHashes, err := impl.buildAutomaticTriggerCommitHashes(ciMaterials, gitCiTriggerRequest) + if err != nil { + return 0, err + } + + trigger := types.Trigger{ + PipelineId: ciPipeline.Id, + CommitHashes: commitHashes, + CiMaterials: ciMaterials, + TriggeredBy: gitCiTriggerRequest.TriggeredBy, + RuntimeParameters: runtimeParams, + } + id, err := impl.triggerCiPipeline(trigger) + if err != nil { + return 0, err + } + return id, nil +} + +func (impl *HandlerServiceImpl) extractPodStatusAndWorkflow(workflowStatus eventProcessorBean.CiCdStatus) (string, string, *pipelineConfig.CiWorkflow, error) { + workflowName, status, _, message, _, _ := pipeline2.ExtractWorkflowStatus(workflowStatus) + if workflowName == "" { + impl.Logger.Errorw("extract workflow status, invalid wf name", "workflowName", workflowName, "status", status, "message", message) + return status, message, nil, errors.New("invalid wf name") + } + workflowId, err := strconv.Atoi(workflowName[:strings.Index(workflowName, "-")]) + if err != nil { + impl.Logger.Errorw("extract workflowId, invalid wf name", "workflowName", workflowName, "err", err) + return status, message, nil, err + } + + savedWorkflow, err := impl.ciWorkflowRepository.FindById(workflowId) + if err != nil { + impl.Logger.Errorw("cannot get saved wf", "workflowId", workflowId, "err", err) + return status, message, nil, err + } + + return status, message, savedWorkflow, err + +} + +func (impl *HandlerServiceImpl) getRefWorkflowAndCiRetryCount(savedWorkflow *pipelineConfig.CiWorkflow) (int, *pipelineConfig.CiWorkflow, error) { + var err error + + if savedWorkflow.ReferenceCiWorkflowId != 0 { + savedWorkflow, err = impl.ciWorkflowRepository.FindById(savedWorkflow.ReferenceCiWorkflowId) + if err != nil { + impl.Logger.Errorw("cannot get saved wf", "err", err) + return 0, savedWorkflow, err + } + } + retryCount, err := impl.ciWorkflowRepository.FindRetriedWorkflowCountByReferenceId(savedWorkflow.Id) + return retryCount, savedWorkflow, err +} + +func (impl *HandlerServiceImpl) validateBuildSequence(gitCiTriggerRequest bean.GitCiTriggerRequest, pipelineId int) (bool, error) { + isValid := true + lastTriggeredBuild, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflow(pipelineId) + if !(lastTriggeredBuild.Status == string(v1alpha1.NodePending) || lastTriggeredBuild.Status == string(v1alpha1.NodeRunning)) { + return true, nil + } + if err != nil && !util.IsErrNoRows(err) { + impl.Logger.Errorw("cannot get last build for pipeline", "pipelineId", pipelineId) + return false, err + } + + ciPipelineMaterial := gitCiTriggerRequest.CiPipelineMaterial + + if ciPipelineMaterial.Type == string(constants.SOURCE_TYPE_BRANCH_FIXED) { + if ciPipelineMaterial.GitCommit.Date.Before(lastTriggeredBuild.GitTriggers[ciPipelineMaterial.Id].Date) { + impl.Logger.Warnw("older commit cannot be built for pipeline", "pipelineId", pipelineId, "ciMaterial", gitCiTriggerRequest.CiPipelineMaterial.Id) + isValid = false + } + } + + return isValid, nil +} + +func (impl *HandlerServiceImpl) buildAutomaticTriggerCommitHashes(ciMaterials []*pipelineConfig.CiPipelineMaterial, request bean.GitCiTriggerRequest) (map[int]pipelineConfig.GitCommit, error) { + commitHashes := map[int]pipelineConfig.GitCommit{} + for _, ciMaterial := range ciMaterials { + if ciMaterial.Id == request.CiPipelineMaterial.Id || len(ciMaterials) == 1 { + request.CiPipelineMaterial.GitCommit = SetGitCommitValuesForBuildingCommitHash(ciMaterial, request.CiPipelineMaterial.GitCommit) + commitHashes[ciMaterial.Id] = request.CiPipelineMaterial.GitCommit + } else { + // this is possible in case of non Webhook, as there would be only one pipeline material per git material in case of PR + lastCommit, err := impl.getLastSeenCommit(ciMaterial.Id) + if err != nil { + return map[int]pipelineConfig.GitCommit{}, err + } + lastCommit = SetGitCommitValuesForBuildingCommitHash(ciMaterial, lastCommit) + commitHashes[ciMaterial.Id] = lastCommit + } + } + return commitHashes, nil +} + +func (impl *HandlerServiceImpl) GetCiPipeline(ciMaterialId int) (*pipelineConfig.CiPipeline, error) { + ciMaterial, err := impl.ciPipelineMaterialRepository.GetById(ciMaterialId) + if err != nil { + return nil, err + } + ciPipeline := ciMaterial.CiPipeline + return ciPipeline, nil +} + +func (impl *HandlerServiceImpl) buildManualTriggerCommitHashes(ciTriggerRequest bean.CiTriggerRequest) (map[int]pipelineConfig.GitCommit, *common.RuntimeParameters, error) { + commitHashes := map[int]pipelineConfig.GitCommit{} + runtimeParams := impl.getRuntimeParamsForBuildingManualTriggerHashes(ciTriggerRequest) + for _, ciPipelineMaterial := range ciTriggerRequest.CiPipelineMaterial { + + pipeLineMaterialFromDb, err := impl.ciPipelineMaterialRepository.GetById(ciPipelineMaterial.Id) + if err != nil { + impl.Logger.Errorw("err in fetching pipeline material by id", "err", err) + return map[int]pipelineConfig.GitCommit{}, nil, err + } + + pipelineType := pipeLineMaterialFromDb.Type + if pipelineType == constants.SOURCE_TYPE_BRANCH_FIXED { + gitCommit, err := impl.BuildManualTriggerCommitHashesForSourceTypeBranchFix(ciPipelineMaterial, pipeLineMaterialFromDb) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return map[int]pipelineConfig.GitCommit{}, nil, err + } + commitHashes[ciPipelineMaterial.Id] = gitCommit + + } else if pipelineType == constants.SOURCE_TYPE_WEBHOOK { + gitCommit, extraEnvVariables, err := impl.BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial, pipeLineMaterialFromDb) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return map[int]pipelineConfig.GitCommit{}, nil, err + } + commitHashes[ciPipelineMaterial.Id] = gitCommit + for key, value := range extraEnvVariables { + runtimeParams = runtimeParams.AddSystemVariable(key, value) + } + } + } + return commitHashes, runtimeParams, nil +} + +func (impl *HandlerServiceImpl) BuildManualTriggerCommitHashesForSourceTypeBranchFix(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (pipelineConfig.GitCommit, error) { + commitMetadataRequest := &gitSensor.CommitMetadataRequest{ + PipelineMaterialId: ciPipelineMaterial.Id, + GitHash: ciPipelineMaterial.GitCommit.Commit, + GitTag: ciPipelineMaterial.GitTag, + } + gitCommitResponse, err := impl.gitSensorClient.GetCommitMetadataForPipelineMaterial(context.Background(), commitMetadataRequest) + if err != nil { + impl.Logger.Errorw("err in fetching commit metadata", "commitMetadataRequest", commitMetadataRequest, "err", err) + return pipelineConfig.GitCommit{}, err + } + if gitCommitResponse == nil { + return pipelineConfig.GitCommit{}, errors.New("commit not found") + } + + gitCommit := pipelineConfig.GitCommit{ + Commit: gitCommitResponse.Commit, + Author: gitCommitResponse.Author, + Date: gitCommitResponse.Date, + Message: gitCommitResponse.Message, + Changes: gitCommitResponse.Changes, + GitRepoName: pipeLineMaterialFromDb.GitMaterial.Name[strings.Index(pipeLineMaterialFromDb.GitMaterial.Name, "-")+1:], + GitRepoUrl: pipeLineMaterialFromDb.GitMaterial.Url, + CiConfigureSourceValue: pipeLineMaterialFromDb.Value, + CiConfigureSourceType: pipeLineMaterialFromDb.Type, + } + + return gitCommit, nil +} + +func (impl *HandlerServiceImpl) BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (pipelineConfig.GitCommit, map[string]string, error) { + webhookDataInput := ciPipelineMaterial.GitCommit.WebhookData + + // fetch webhook data on the basis of Id + webhookDataRequest := &gitSensor.WebhookDataRequest{ + Id: webhookDataInput.Id, + CiPipelineMaterialId: ciPipelineMaterial.Id, + } + + webhookAndCiData, err := impl.gitSensorClient.GetWebhookData(context.Background(), webhookDataRequest) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return pipelineConfig.GitCommit{}, nil, err + } + webhookData := webhookAndCiData.WebhookData + + // if webhook event is of merged type, then fetch latest commit for target branch + if webhookData.EventActionType == bean.WEBHOOK_EVENT_MERGED_ACTION_TYPE { + + // get target branch name from webhook + targetBranchName := webhookData.Data[bean.WEBHOOK_SELECTOR_TARGET_BRANCH_NAME_NAME] + if targetBranchName == "" { + impl.Logger.Error("target branch not found from webhook data") + return pipelineConfig.GitCommit{}, nil, err + } + + // get latest commit hash for target branch + latestCommitMetadataRequest := &gitSensor.CommitMetadataRequest{ + PipelineMaterialId: ciPipelineMaterial.Id, + BranchName: targetBranchName, + } + + latestCommit, err := impl.gitSensorClient.GetCommitMetadata(context.Background(), latestCommitMetadataRequest) + + if err != nil { + impl.Logger.Errorw("err", "err", err) + return pipelineConfig.GitCommit{}, nil, err + } + + // update webhookData (local) with target latest hash + webhookData.Data[bean.WEBHOOK_SELECTOR_TARGET_CHECKOUT_NAME] = latestCommit.Commit + + } + + // build git commit + gitCommit := pipelineConfig.GitCommit{ + GitRepoName: pipeLineMaterialFromDb.GitMaterial.Name[strings.Index(pipeLineMaterialFromDb.GitMaterial.Name, "-")+1:], + GitRepoUrl: pipeLineMaterialFromDb.GitMaterial.Url, + CiConfigureSourceValue: pipeLineMaterialFromDb.Value, + CiConfigureSourceType: pipeLineMaterialFromDb.Type, + WebhookData: pipelineConfig.WebhookData{ + Id: int(webhookData.Id), + EventActionType: webhookData.EventActionType, + Data: webhookData.Data, + }, + } + + return gitCommit, webhookAndCiData.ExtraEnvironmentVariables, nil +} + +func (impl *HandlerServiceImpl) getLastSeenCommit(ciMaterialId int) (pipelineConfig.GitCommit, error) { + var materialIds []int + materialIds = append(materialIds, ciMaterialId) + headReq := &gitSensor.HeadRequest{ + MaterialIds: materialIds, + } + res, err := impl.gitSensorClient.GetHeadForPipelineMaterials(context.Background(), headReq) + if err != nil { + return pipelineConfig.GitCommit{}, err + } + if len(res) == 0 { + return pipelineConfig.GitCommit{}, errors.New("received empty response") + } + gitCommit := pipelineConfig.GitCommit{ + Commit: res[0].GitCommit.Commit, + Author: res[0].GitCommit.Author, + Date: res[0].GitCommit.Date, + Message: res[0].GitCommit.Message, + Changes: res[0].GitCommit.Changes, + } + return gitCommit, nil +} + +func SetGitCommitValuesForBuildingCommitHash(ciMaterial *pipelineConfig.CiPipelineMaterial, oldGitCommit pipelineConfig.GitCommit) pipelineConfig.GitCommit { + newGitCommit := oldGitCommit + newGitCommit.CiConfigureSourceType = ciMaterial.Type + newGitCommit.CiConfigureSourceValue = ciMaterial.Value + newGitCommit.GitRepoUrl = ciMaterial.GitMaterial.Url + newGitCommit.GitRepoName = ciMaterial.GitMaterial.Name[strings.Index(ciMaterial.GitMaterial.Name, "-")+1:] + return newGitCommit +} + +func (impl *HandlerServiceImpl) triggerCiPipeline(trigger types.Trigger) (int, error) { + pipeline, variableSnapshot, savedCiWf, workflowRequest, err := impl.StartCiWorkflowAndPrepareWfRequest(trigger) + if err != nil { + return 0, err + } + workflowRequest.CiPipelineType = trigger.PipelineType + err = impl.executeCiPipeline(workflowRequest) + if err != nil { + impl.Logger.Errorw("error in executing ci pipeline", "err", err) + dbErr := impl.markCurrentCiWorkflowFailed(savedCiWf, err) + if dbErr != nil { + impl.Logger.Errorw("update ci workflow error", "err", dbErr) + } + return 0, err + } + impl.Logger.Debugw("ci triggered", " pipeline ", trigger.PipelineId) + + var variableSnapshotHistories = sliceUtil.GetBeansPtr( + repository4.GetSnapshotBean(savedCiWf.Id, repository4.HistoryReferenceTypeCIWORKFLOW, variableSnapshot)) + if len(variableSnapshotHistories) > 0 { + err = impl.scopedVariableManager.SaveVariableHistoriesForTrigger(variableSnapshotHistories, trigger.TriggeredBy) + if err != nil { + impl.Logger.Errorf("Not able to save variable snapshot for CI trigger %s", err) + } + } + + middleware.CiTriggerCounter.WithLabelValues(pipeline.App.AppName, pipeline.Name).Inc() + go impl.ciService.WriteCITriggerEvent(trigger, pipeline, workflowRequest) + return savedCiWf.Id, err +} + +func (impl *HandlerServiceImpl) GetCiMaterials(pipelineId int, ciMaterials []*pipelineConfig.CiPipelineMaterial) ([]*pipelineConfig.CiPipelineMaterial, error) { + if !(len(ciMaterials) == 0) { + return ciMaterials, nil + } else { + ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(pipelineId) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return nil, err + } + impl.Logger.Debug("ciMaterials for pipeline trigger ", ciMaterials) + return ciMaterials, nil + } +} + +func (impl *HandlerServiceImpl) StartCiWorkflowAndPrepareWfRequest(trigger types.Trigger) (*pipelineConfig.CiPipeline, map[string]string, *pipelineConfig.CiWorkflow, *types.WorkflowRequest, error) { + impl.Logger.Debugw("ci pipeline manual trigger", "request", trigger) + ciMaterials, err := impl.GetCiMaterials(trigger.PipelineId, trigger.CiMaterials) + if err != nil { + return nil, nil, nil, nil, err + } + + ciPipelineScripts, err := impl.ciPipelineRepository.FindCiScriptsByCiPipelineId(trigger.PipelineId) + if err != nil && !util.IsErrNoRows(err) { + return nil, nil, nil, nil, err + } + + var pipeline *pipelineConfig.CiPipeline + for _, m := range ciMaterials { + pipeline = m.CiPipeline + break + } + + scope := resourceQualifiers.Scope{ + AppId: pipeline.App.Id, + } + ciWorkflowConfigNamespace := impl.config.GetDefaultNamespace() + envModal, isJob, err := impl.getEnvironmentForJob(pipeline, trigger) + if err != nil { + return nil, nil, nil, nil, err + } + if isJob && envModal != nil { + + err = impl.checkArgoSetupRequirement(envModal) + if err != nil { + return nil, nil, nil, nil, err + } + + ciWorkflowConfigNamespace = envModal.Namespace + + // This will be populated for jobs running in selected environment + scope.EnvId = envModal.Id + scope.ClusterId = envModal.ClusterId + + scope.SystemMetadata = &resourceQualifiers.SystemMetadata{ + EnvironmentName: envModal.Name, + ClusterName: envModal.Cluster.ClusterName, + Namespace: envModal.Namespace, + } + } + if scope.SystemMetadata == nil { + scope.SystemMetadata = &resourceQualifiers.SystemMetadata{ + Namespace: ciWorkflowConfigNamespace, + AppName: pipeline.App.AppName, + } + } + savedCiWf, err := impl.saveNewWorkflowForCITrigger(pipeline, ciWorkflowConfigNamespace, trigger.CommitHashes, trigger.TriggeredBy, ciMaterials, trigger.EnvironmentId, isJob, trigger.ReferenceCiWorkflowId) + if err != nil { + impl.Logger.Errorw("could not save new workflow", "err", err) + return nil, nil, nil, nil, err + } + // preCiSteps, postCiSteps, refPluginsData, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(pipeline.Id, ciEvent) + request := pipelineConfigBean.NewBuildPrePostStepDataReq(pipeline.Id, pipelineConfigBean.CiStage, scope) + request = updateBuildPrePostStepDataReq(request, trigger) + prePostAndRefPluginResponse, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(request) + if err != nil { + impl.Logger.Errorw("error in getting pre steps data for wf request", "err", err, "ciPipelineId", pipeline.Id) + dbErr := impl.markCurrentCiWorkflowFailed(savedCiWf, err) + if dbErr != nil { + impl.Logger.Errorw("saving workflow error", "err", dbErr) + } + return nil, nil, nil, nil, err + } + preCiSteps := prePostAndRefPluginResponse.PreStageSteps + postCiSteps := prePostAndRefPluginResponse.PostStageSteps + refPluginsData := prePostAndRefPluginResponse.RefPluginData + variableSnapshot := prePostAndRefPluginResponse.VariableSnapshot + + if len(preCiSteps) == 0 && isJob { + errMsg := fmt.Sprintf("No tasks are configured in this job pipeline") + validationErr := util.NewApiError(http.StatusNotFound, errMsg, errMsg) + return nil, nil, nil, nil, validationErr + } + + // get env variables of git trigger data and add it in the extraEnvVariables + gitTriggerEnvVariables, _, err := impl.ciCdPipelineOrchestrator.GetGitCommitEnvVarDataForCICDStage(savedCiWf.GitTriggers) + if err != nil { + impl.Logger.Errorw("error in getting gitTrigger env data for stage", "gitTriggers", savedCiWf.GitTriggers, "err", err) + return nil, nil, nil, nil, err + } + + for k, v := range gitTriggerEnvVariables { + trigger.RuntimeParameters = trigger.RuntimeParameters.AddSystemVariable(k, v) + } + + workflowRequest, err := impl.buildWfRequestForCiPipeline(pipeline, trigger, ciMaterials, savedCiWf, ciWorkflowConfigNamespace, ciPipelineScripts, preCiSteps, postCiSteps, refPluginsData, isJob) + if err != nil { + impl.Logger.Errorw("make workflow req", "err", err) + return nil, nil, nil, nil, err + } + err = impl.handleRuntimeParamsValidations(trigger, ciMaterials, workflowRequest) + if err != nil { + savedCiWf.Status = cdWorkflow.WorkflowAborted + savedCiWf.Message = err.Error() + err1 := impl.ciService.UpdateCiWorkflowWithStage(savedCiWf) + if err1 != nil { + impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag") + } + return nil, nil, nil, nil, err + } + + workflowRequest.Scope = scope + workflowRequest.AppId = pipeline.AppId + workflowRequest.Env = envModal + if isJob { + workflowRequest.Type = pipelineConfigBean.JOB_WORKFLOW_PIPELINE_TYPE + } else { + workflowRequest.Type = pipelineConfigBean.CI_WORKFLOW_PIPELINE_TYPE + } + workflowRequest, err = impl.updateWorkflowRequestWithBuildCacheData(workflowRequest, scope) + if err != nil { + impl.Logger.Errorw("error, updateWorkflowRequestWithBuildCacheData", "workflowRequest", workflowRequest, "err", err) + return nil, nil, nil, nil, err + } + if impl.canSetK8sDriverData(workflowRequest) { + err = impl.setBuildxK8sDriverData(workflowRequest) + if err != nil { + impl.Logger.Errorw("error in setBuildxK8sDriverData", "BUILDX_K8S_DRIVER_OPTIONS", impl.config.BuildxK8sDriverOptions, "err", err) + return nil, nil, nil, nil, err + } + } + savedCiWf.LogLocation = fmt.Sprintf("%s/%s/main.log", impl.config.GetDefaultBuildLogsKeyPrefix(), workflowRequest.WorkflowNamePrefix) + err = impl.updateCiWorkflow(workflowRequest, savedCiWf) + appLabels, err := impl.appCrudOperationService.GetLabelsByAppId(pipeline.AppId) + if err != nil { + return nil, nil, nil, nil, err + } + workflowRequest.AppLabels = appLabels + workflowRequest = impl.updateWorkflowRequestWithEntSupportData(workflowRequest) + return pipeline, variableSnapshot, savedCiWf, workflowRequest, nil +} + +func (impl *HandlerServiceImpl) setBuildxK8sDriverData(workflowRequest *types.WorkflowRequest) error { + dockerBuildConfig := workflowRequest.CiBuildConfig.DockerBuildConfig + k8sDriverOptions, err := impl.getK8sDriverOptions(workflowRequest, dockerBuildConfig.TargetPlatform) + if err != nil { + impl.Logger.Errorw("error in parsing BUILDX_K8S_DRIVER_OPTIONS from the devtron-cm", "err", err) + } + dockerBuildConfig.BuildxK8sDriverOptions = k8sDriverOptions + return nil +} + +func (impl *HandlerServiceImpl) getEnvironmentForJob(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger) (*repository6.Environment, bool, error) { + app, err := impl.appRepository.FindById(pipeline.AppId) + if err != nil { + impl.Logger.Errorw("could not find app", "err", err) + return nil, false, err + } + + var env *repository6.Environment + isJob := false + if app.AppType == helper.Job { + isJob = true + if trigger.EnvironmentId != 0 { + env, err = impl.envRepository.FindById(trigger.EnvironmentId) + if err != nil { + impl.Logger.Errorw("could not find environment", "err", err) + return nil, isJob, err + } + return env, isJob, nil + } + } + return nil, isJob, nil +} + +// TODO: Send all trigger data +func (impl *HandlerServiceImpl) BuildPayload(trigger types.Trigger, pipeline *pipelineConfig.CiPipeline) *client.Payload { + payload := &client.Payload{} + payload.AppName = pipeline.App.AppName + payload.PipelineName = pipeline.Name + return payload +} + +func (impl *HandlerServiceImpl) saveNewWorkflowForCITrigger(pipeline *pipelineConfig.CiPipeline, ciWorkflowConfigNamespace string, + commitHashes map[int]pipelineConfig.GitCommit, userId int32, ciMaterials []*pipelineConfig.CiPipelineMaterial, EnvironmentId int, isJob bool, refCiWorkflowId int) (*pipelineConfig.CiWorkflow, error) { + + isCiTriggerBlocked, err := impl.checkIfCITriggerIsBlocked(pipeline, ciMaterials, isJob) + if err != nil { + impl.Logger.Errorw("error, checkIfCITriggerIsBlocked", "pipelineId", pipeline.Id, "err", err) + return &pipelineConfig.CiWorkflow{}, err + } + ciWorkflow := &pipelineConfig.CiWorkflow{ + Name: pipeline.Name + "-" + strconv.Itoa(pipeline.Id), + Status: cdWorkflow.WorkflowStarting, // starting CIStage + Message: "", + StartedOn: time.Now(), + CiPipelineId: pipeline.Id, + Namespace: impl.config.GetDefaultNamespace(), + BlobStorageEnabled: impl.config.BlobStorageEnabled, + GitTriggers: commitHashes, + LogLocation: "", + TriggeredBy: userId, + ReferenceCiWorkflowId: refCiWorkflowId, + ExecutorType: impl.config.GetWorkflowExecutorType(), + } + if isJob { + ciWorkflow.Namespace = ciWorkflowConfigNamespace + ciWorkflow.EnvironmentId = EnvironmentId + } + if isCiTriggerBlocked { + return impl.handleWFIfCITriggerIsBlocked(ciWorkflow) + } + err = impl.ciService.SaveCiWorkflowWithStage(ciWorkflow) + if err != nil { + impl.Logger.Errorw("saving workflow error", "err", err) + return &pipelineConfig.CiWorkflow{}, err + } + impl.Logger.Debugw("workflow saved ", "id", ciWorkflow.Id) + return ciWorkflow, nil +} + +func (impl *HandlerServiceImpl) executeCiPipeline(workflowRequest *types.WorkflowRequest) error { + _, _, err := impl.workflowService.SubmitWorkflow(workflowRequest) + if err != nil { + impl.Logger.Errorw("workflow error", "err", err) + return err + } + return nil +} + +func (impl *HandlerServiceImpl) buildS3ArtifactLocation(ciWorkflowConfigLogsBucket string, savedWf *pipelineConfig.CiWorkflow) (string, string, string) { + ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() + ArtifactLocation := fmt.Sprintf("s3://"+path.Join(ciWorkflowConfigLogsBucket, ciArtifactLocationFormat), savedWf.Id, savedWf.Id) + artifactFileName := fmt.Sprintf(ciArtifactLocationFormat, savedWf.Id, savedWf.Id) + return ArtifactLocation, ciWorkflowConfigLogsBucket, artifactFileName +} + +func (impl *HandlerServiceImpl) buildDefaultArtifactLocation(savedWf *pipelineConfig.CiWorkflow) string { + ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() + ArtifactLocation := fmt.Sprintf(ciArtifactLocationFormat, savedWf.Id, savedWf.Id) + return ArtifactLocation +} + +func (impl *HandlerServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, savedWf *pipelineConfig.CiWorkflow, ciWorkflowConfigNamespace string, ciPipelineScripts []*pipelineConfig.CiPipelineScript, preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, refPluginsData []*pipelineConfigBean.RefPluginObject, isJob bool) (*types.WorkflowRequest, error) { + var ciProjectDetails []pipelineConfigBean.CiProjectDetails + commitHashes := trigger.CommitHashes + for _, ciMaterial := range ciMaterials { + // ignore those materials which have inactive git material + if ciMaterial == nil || ciMaterial.GitMaterial == nil || !ciMaterial.GitMaterial.Active { + continue + } + commitHashForPipelineId := commitHashes[ciMaterial.Id] + ciProjectDetail := pipelineConfigBean.CiProjectDetails{ + GitRepository: ciMaterial.GitMaterial.Url, + MaterialName: ciMaterial.GitMaterial.Name, + CheckoutPath: ciMaterial.GitMaterial.CheckoutPath, + FetchSubmodules: ciMaterial.GitMaterial.FetchSubmodules, + CommitHash: commitHashForPipelineId.Commit, + Author: commitHashForPipelineId.Author, + SourceType: ciMaterial.Type, + SourceValue: ciMaterial.Value, + GitTag: ciMaterial.GitTag, + Message: commitHashForPipelineId.Message, + Type: string(ciMaterial.Type), + CommitTime: commitHashForPipelineId.Date.Format(bean.LayoutRFC3339), + GitOptions: pipelineConfigBean.GitOptions{ + UserName: ciMaterial.GitMaterial.GitProvider.UserName, + Password: ciMaterial.GitMaterial.GitProvider.Password, + SshPrivateKey: ciMaterial.GitMaterial.GitProvider.SshPrivateKey, + AccessToken: ciMaterial.GitMaterial.GitProvider.AccessToken, + AuthMode: ciMaterial.GitMaterial.GitProvider.AuthMode, + EnableTLSVerification: ciMaterial.GitMaterial.GitProvider.EnableTLSVerification, + TlsKey: ciMaterial.GitMaterial.GitProvider.TlsKey, + TlsCert: ciMaterial.GitMaterial.GitProvider.TlsCert, + CaCert: ciMaterial.GitMaterial.GitProvider.CaCert, + }, + } + var err error + ciProjectDetail, err = impl.updateCIProjectDetailWithCloningMode(pipeline.AppId, ciMaterial, ciProjectDetail) + if err != nil { + impl.Logger.Errorw("error, updateCIProjectDetailWithCloningMode", "pipelineId", pipeline.Id, "err", err) + return nil, err + } + if ciMaterial.Type == constants.SOURCE_TYPE_WEBHOOK { + webhookData := commitHashForPipelineId.WebhookData + ciProjectDetail.WebhookData = pipelineConfig.WebhookData{ + Id: webhookData.Id, + EventActionType: webhookData.EventActionType, + Data: webhookData.Data, + } + } + + ciProjectDetails = append(ciProjectDetails, ciProjectDetail) + } + + var beforeDockerBuildScripts []*bean.CiScript + var afterDockerBuildScripts []*bean.CiScript + for _, ciPipelineScript := range ciPipelineScripts { + ciTask := &bean.CiScript{ + Id: ciPipelineScript.Id, + Index: ciPipelineScript.Index, + Name: ciPipelineScript.Name, + Script: ciPipelineScript.Script, + OutputLocation: ciPipelineScript.OutputLocation, + } + + if ciPipelineScript.Stage == buildCommonBean.BEFORE_DOCKER_BUILD { + beforeDockerBuildScripts = append(beforeDockerBuildScripts, ciTask) + } else if ciPipelineScript.Stage == buildCommonBean.AFTER_DOCKER_BUILD { + afterDockerBuildScripts = append(afterDockerBuildScripts, ciTask) + } + } + + if !(len(beforeDockerBuildScripts) == 0 && len(afterDockerBuildScripts) == 0) { + // found beforeDockerBuildScripts/afterDockerBuildScripts + // building preCiSteps & postCiSteps from them, refPluginsData not needed + preCiSteps = buildCiStepsDataFromDockerBuildScripts(beforeDockerBuildScripts) + postCiSteps = buildCiStepsDataFromDockerBuildScripts(afterDockerBuildScripts) + refPluginsData = []*pipelineConfigBean.RefPluginObject{} + } + + host, err := impl.attributeService.GetByKey(bean3.HostUrlKey) + if err != nil { + impl.Logger.Errorw("error in getting host url", "err", err, "hostUrl", host.Value) + return nil, err + } + ciWorkflowConfigCiCacheBucket := impl.config.DefaultCacheBucket + + ciWorkflowConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion + + ciWorkflowConfigCiImage := impl.config.GetDefaultImage() + + ciTemplate := pipeline.CiTemplate + ciLevelArgs := pipeline.DockerArgs + + if ciLevelArgs == "" { + ciLevelArgs = "{}" + } + + if pipeline.CiTemplate.DockerBuildOptions == "" { + pipeline.CiTemplate.DockerBuildOptions = "{}" + } + userEmailId, err := impl.userService.GetActiveEmailById(trigger.TriggeredBy) + if err != nil { + impl.Logger.Errorw("unable to find user email by id", "err", err, "id", trigger.TriggeredBy) + return nil, err + } + var dockerfilePath string + var dockerRepository string + var checkoutPath string + var ciBuildConfigBean *buildBean.CiBuildConfigBean + dockerRegistry := &repository3.DockerArtifactStore{} + ciBaseBuildConfigEntity := ciTemplate.CiBuildConfig + ciBaseBuildConfigBean, err := adapter.ConvertDbBuildConfigToBean(ciBaseBuildConfigEntity) + if err != nil { + impl.Logger.Errorw("error occurred while converting buildconfig dbEntity to configBean", "ciBuildConfigEntity", ciBaseBuildConfigEntity, "err", err) + return nil, errors.New("error while parsing ci build config") + } + if !pipeline.IsExternal && pipeline.IsDockerConfigOverridden { + templateOverrideBean, err := impl.ciTemplateService.FindTemplateOverrideByCiPipelineId(pipeline.Id) + if err != nil { + return nil, err + } + ciBuildConfigBean = templateOverrideBean.CiBuildConfig + // updating args coming from ciBaseBuildConfigEntity because it is not part of Ci override + if ciBuildConfigBean != nil && ciBuildConfigBean.DockerBuildConfig != nil && ciBaseBuildConfigBean != nil && ciBaseBuildConfigBean.DockerBuildConfig != nil { + ciBuildConfigBean.DockerBuildConfig.Args = ciBaseBuildConfigBean.DockerBuildConfig.Args + } + templateOverride := templateOverrideBean.CiTemplateOverride + checkoutPath = templateOverride.GitMaterial.CheckoutPath + dockerfilePath = templateOverride.DockerfilePath + dockerRepository = templateOverride.DockerRepository + dockerRegistry = templateOverride.DockerRegistry + } else { + checkoutPath = ciTemplate.GitMaterial.CheckoutPath + dockerfilePath = ciTemplate.DockerfilePath + dockerRegistry = ciTemplate.DockerRegistry + dockerRepository = ciTemplate.DockerRepository + ciBuildConfigBean = ciBaseBuildConfigBean + if ciBuildConfigBean != nil { + ciBuildConfigBean.BuildContextGitMaterialId = ciTemplate.BuildContextGitMaterialId + } + + } + if checkoutPath == "" { + checkoutPath = "./" + } + var dockerImageTag string + customTag, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id)) + if err != nil && err != pg.ErrNoRows { + return nil, err + } + if customTag.Id != 0 && customTag.Enabled == true { + imagePathReservation, err := impl.customTagService.GenerateImagePath(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id), dockerRegistry.RegistryURL, dockerRepository) + if err != nil { + if errors.Is(err, pipelineConfigBean.ErrImagePathInUse) { + errMsg := pipelineConfigBean.ImageTagUnavailableMessage + validationErr := util.NewApiError(http.StatusConflict, errMsg, errMsg) + dbErr := impl.markCurrentCiWorkflowFailed(savedWf, validationErr) + if dbErr != nil { + impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag", "err", dbErr, "savedWf", savedWf.Id) + } + return nil, err + } + return nil, err + } + savedWf.ImagePathReservationIds = []int{imagePathReservation.Id} + // imagePath = docker.io/avd0/dashboard:fd23414b + imagePathSplit := strings.Split(imagePathReservation.ImagePath, ":") + if len(imagePathSplit) >= 1 { + dockerImageTag = imagePathSplit[len(imagePathSplit)-1] + } + } else { + dockerImageTag = impl.buildImageTag(commitHashes, pipeline.Id, savedWf.Id) + } + + // copyContainerImage plugin specific logic + var registryCredentialMap map[string]bean2.RegistryCredentials + var pluginArtifactStage string + var imageReservationIds []int + var registryDestinationImageMap map[string][]string + if !isJob { + registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imageReservationIds, err = impl.GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps, postCiSteps, dockerImageTag, customTag.Id, + fmt.Sprintf(pipelineConfigBean.ImagePathPattern, + dockerRegistry.RegistryURL, + dockerRepository, + dockerImageTag), + dockerRegistry.Id) + if err != nil { + impl.Logger.Errorw("error in getting env variables for copyContainerImage plugin") + dbErr := impl.markCurrentCiWorkflowFailed(savedWf, err) + if dbErr != nil { + impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag", "err", dbErr, "savedWf", savedWf.Id) + } + return nil, err + } + + savedWf.ImagePathReservationIds = append(savedWf.ImagePathReservationIds, imageReservationIds...) + } + // mergedArgs := string(merged) + oldArgs := ciTemplate.Args + ciBuildConfigBean, err = adapter.OverrideCiBuildConfig(dockerfilePath, oldArgs, ciLevelArgs, ciTemplate.DockerBuildOptions, ciTemplate.TargetPlatform, ciBuildConfigBean) + if err != nil { + impl.Logger.Errorw("error occurred while overriding ci build config", "oldArgs", oldArgs, "ciLevelArgs", ciLevelArgs, "error", err) + return nil, errors.New("error while parsing ci build config") + } + buildContextCheckoutPath, err := impl.ciPipelineMaterialRepository.GetCheckoutPath(ciBuildConfigBean.BuildContextGitMaterialId) + if err != nil && err != pg.ErrNoRows { + impl.Logger.Errorw("error occurred while getting checkout path from git material", "gitMaterialId", ciBuildConfigBean.BuildContextGitMaterialId, "error", err) + return nil, err + } + if buildContextCheckoutPath == "" { + buildContextCheckoutPath = checkoutPath + } + if ciBuildConfigBean.UseRootBuildContext { + // use root build context i.e '.' + buildContextCheckoutPath = "." + } + + ciBuildConfigBean.PipelineType = trigger.PipelineType + + if ciBuildConfigBean.CiBuildType == buildBean.SELF_DOCKERFILE_BUILD_TYPE || ciBuildConfigBean.CiBuildType == buildBean.MANAGED_DOCKERFILE_BUILD_TYPE { + ciBuildConfigBean.DockerBuildConfig.BuildContext = filepath.Join(buildContextCheckoutPath, ciBuildConfigBean.DockerBuildConfig.BuildContext) + dockerBuildConfig := ciBuildConfigBean.DockerBuildConfig + dockerfilePath = filepath.Join(checkoutPath, dockerBuildConfig.DockerfilePath) + dockerBuildConfig.DockerfilePath = dockerfilePath + checkoutPath = dockerfilePath[:strings.LastIndex(dockerfilePath, "/")+1] + } else if ciBuildConfigBean.CiBuildType == buildBean.BUILDPACK_BUILD_TYPE { + buildPackConfig := ciBuildConfigBean.BuildPackConfig + checkoutPath = filepath.Join(checkoutPath, buildPackConfig.ProjectPath) + } + + if ciBuildConfigBean.DockerBuildConfig != nil { + ciBuildConfigBean = impl.updateCIBuildConfig(ciBuildConfigBean) + } + + workflowRequest := &types.WorkflowRequest{ + WorkflowNamePrefix: strconv.Itoa(savedWf.Id) + "-" + savedWf.Name, + PipelineName: pipeline.Name, + PipelineId: pipeline.Id, + CiCacheFileName: pipeline.Name + "-" + strconv.Itoa(pipeline.Id) + ".tar.gz", + CiProjectDetails: ciProjectDetails, + Namespace: ciWorkflowConfigNamespace, + BlobStorageConfigured: savedWf.BlobStorageEnabled, + CiImage: ciWorkflowConfigCiImage, + WorkflowId: savedWf.Id, + TriggeredBy: savedWf.TriggeredBy, + CacheLimit: impl.config.CacheLimit, + ScanEnabled: pipeline.ScanEnabled, + CloudProvider: impl.config.CloudProvider, + DefaultAddressPoolBaseCidr: impl.config.GetDefaultAddressPoolBaseCidr(), + DefaultAddressPoolSize: impl.config.GetDefaultAddressPoolSize(), + PreCiSteps: preCiSteps, + PostCiSteps: postCiSteps, + RefPlugins: refPluginsData, + AppName: pipeline.App.AppName, + TriggerByAuthor: userEmailId, + CiBuildConfig: ciBuildConfigBean, + CiBuildDockerMtuValue: impl.config.CiRunnerDockerMTUValue, + CacheInvalidate: trigger.InvalidateCache, + SystemEnvironmentVariables: trigger.RuntimeParameters.GetSystemVariables(), + EnableBuildContext: impl.config.EnableBuildContext, + OrchestratorHost: impl.config.OrchestratorHost, + HostUrl: host.Value, + OrchestratorToken: impl.config.OrchestratorToken, + ImageRetryCount: impl.config.ImageRetryCount, + ImageRetryInterval: impl.config.ImageRetryInterval, + WorkflowExecutor: impl.config.GetWorkflowExecutorType(), + Type: pipelineConfigBean.CI_WORKFLOW_PIPELINE_TYPE, + CiArtifactLastFetch: trigger.CiArtifactLastFetch, + RegistryCredentialMap: registryCredentialMap, + PluginArtifactStage: pluginArtifactStage, + ImageScanMaxRetries: impl.config.ImageScanMaxRetries, + ImageScanRetryDelay: impl.config.ImageScanRetryDelay, + UseDockerApiToGetDigest: impl.config.UseDockerApiToGetDigest, + RegistryDestinationImageMap: registryDestinationImageMap, + } + workflowRequest.SetEntOnlyFields(trigger, impl.config) + workflowCacheConfig := impl.ciCdPipelineOrchestrator.GetWorkflowCacheConfig(pipeline.App.AppType, trigger.PipelineType, pipeline.GetWorkflowCacheConfig()) + workflowRequest.IgnoreDockerCachePush = !workflowCacheConfig.Value + workflowRequest.IgnoreDockerCachePull = !workflowCacheConfig.Value + impl.Logger.Debugw("Ignore Cache values", "IgnoreDockerCachePush", workflowRequest.IgnoreDockerCachePush, "IgnoreDockerCachePull", workflowRequest.IgnoreDockerCachePull) + if pipeline.App.AppType == helper.Job { + workflowRequest.AppName = pipeline.App.DisplayName + } + if pipeline.ScanEnabled { + scanToolMetadata, scanVia, err := impl.fetchImageScanExecutionMedium() + if err != nil { + impl.Logger.Errorw("error occurred getting scanned via", "err", err) + return nil, err + } + workflowRequest.SetExecuteImageScanningVia(scanVia) + if scanVia.IsScanMediumExternal() { + imageScanExecutionSteps, refPlugins, err := impl.fetchImageScanExecutionStepsForWfRequest(scanToolMetadata) + if err != nil { + impl.Logger.Errorw("error occurred, fetchImageScanExecutionStepsForWfRequest", "scanToolMetadata", scanToolMetadata, "err", err) + return nil, err + } + workflowRequest.SetImageScanningSteps(imageScanExecutionSteps) + workflowRequest.RefPlugins = append(workflowRequest.RefPlugins, refPlugins...) + } + } + if dockerRegistry != nil { + workflowRequest, err = impl.updateWorkflowRequestWithRemoteConnConf(dockerRegistry, workflowRequest) + if err != nil { + impl.Logger.Errorw("error occurred updating workflow request", "dockerRegistryId", dockerRegistry.Id, "err", err) + return nil, err + } + workflowRequest.DockerRegistryId = dockerRegistry.Id + workflowRequest.DockerRegistryType = string(dockerRegistry.RegistryType) + workflowRequest.DockerImageTag = dockerImageTag + workflowRequest.DockerRegistryURL = dockerRegistry.RegistryURL + workflowRequest.DockerRepository = dockerRepository + workflowRequest.CheckoutPath = checkoutPath + workflowRequest.DockerUsername = dockerRegistry.Username + workflowRequest.DockerPassword = dockerRegistry.Password + workflowRequest.AwsRegion = dockerRegistry.AWSRegion + workflowRequest.AccessKey = dockerRegistry.AWSAccessKeyId + workflowRequest.SecretKey = dockerRegistry.AWSSecretAccessKey + workflowRequest.DockerConnection = dockerRegistry.Connection + workflowRequest.DockerCert = dockerRegistry.Cert + + } + ciWorkflowConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() + + switch workflowRequest.CloudProvider { + case types.BLOB_STORAGE_S3: + // No AccessKey is used for uploading artifacts, instead IAM based auth is used + workflowRequest.CiCacheRegion = ciWorkflowConfigCiCacheRegion + workflowRequest.CiCacheLocation = ciWorkflowConfigCiCacheBucket + workflowRequest.CiArtifactLocation, workflowRequest.CiArtifactBucket, workflowRequest.CiArtifactFileName = impl.buildS3ArtifactLocation(ciWorkflowConfigLogsBucket, savedWf) + workflowRequest.BlobStorageS3Config = &blob_storage.BlobStorageS3Config{ + AccessKey: impl.config.BlobStorageS3AccessKey, + Passkey: impl.config.BlobStorageS3SecretKey, + EndpointUrl: impl.config.BlobStorageS3Endpoint, + IsInSecure: impl.config.BlobStorageS3EndpointInsecure, + CiCacheBucketName: ciWorkflowConfigCiCacheBucket, + CiCacheRegion: ciWorkflowConfigCiCacheRegion, + CiCacheBucketVersioning: impl.config.BlobStorageS3BucketVersioned, + CiArtifactBucketName: workflowRequest.CiArtifactBucket, + CiArtifactRegion: impl.config.GetDefaultCdLogsBucketRegion(), + CiArtifactBucketVersioning: impl.config.BlobStorageS3BucketVersioned, + CiLogBucketName: impl.config.GetDefaultBuildLogsBucket(), + CiLogRegion: impl.config.GetDefaultCdLogsBucketRegion(), + CiLogBucketVersioning: impl.config.BlobStorageS3BucketVersioned, + } + case types.BLOB_STORAGE_GCP: + workflowRequest.GcpBlobConfig = &blob_storage.GcpBlobConfig{ + CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, + CacheBucketName: ciWorkflowConfigCiCacheBucket, + LogBucketName: ciWorkflowConfigLogsBucket, + ArtifactBucketName: ciWorkflowConfigLogsBucket, + } + workflowRequest.CiArtifactLocation = impl.buildDefaultArtifactLocation(savedWf) + workflowRequest.CiArtifactFileName = workflowRequest.CiArtifactLocation + case types.BLOB_STORAGE_AZURE: + workflowRequest.AzureBlobConfig = &blob_storage.AzureBlobConfig{ + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, + AccountName: impl.config.AzureAccountName, + BlobContainerCiCache: impl.config.AzureBlobContainerCiCache, + AccountKey: impl.config.AzureAccountKey, + BlobContainerCiLog: impl.config.AzureBlobContainerCiLog, + BlobContainerArtifact: impl.config.AzureBlobContainerCiLog, + } + workflowRequest.BlobStorageS3Config = &blob_storage.BlobStorageS3Config{ + EndpointUrl: impl.config.AzureGatewayUrl, + IsInSecure: impl.config.AzureGatewayConnectionInsecure, + CiLogBucketName: impl.config.AzureBlobContainerCiLog, + CiLogRegion: impl.config.DefaultCacheBucketRegion, + CiLogBucketVersioning: impl.config.BlobStorageS3BucketVersioned, + AccessKey: impl.config.AzureAccountName, + } + workflowRequest.CiArtifactLocation = impl.buildDefaultArtifactLocation(savedWf) + workflowRequest.CiArtifactFileName = workflowRequest.CiArtifactLocation + default: + if impl.config.BlobStorageEnabled { + return nil, fmt.Errorf("blob storage %s not supported", workflowRequest.CloudProvider) + } + } + return workflowRequest, nil +} + +func (impl *HandlerServiceImpl) GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, customTag string, customTagId int, buildImagePath string, buildImagedockerRegistryId string) (map[string][]string, map[string]bean2.RegistryCredentials, string, []int, error) { + + copyContainerImagePluginDetail, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(buildCommonBean.COPY_CONTAINER_IMAGE) + if err != nil && err != pg.ErrNoRows { + impl.Logger.Errorw("error in getting copyContainerImage plugin id", "err", err) + return nil, nil, "", nil, err + } + + pluginIdToVersionMap := make(map[int]string) + for _, p := range copyContainerImagePluginDetail { + pluginIdToVersionMap[p.Id] = p.Version + } + + for _, step := range preCiSteps { + if _, ok := pluginIdToVersionMap[step.RefPluginId]; ok { + // for copyContainerImage plugin parse destination images and save its data in image path reservation table + return nil, nil, "", nil, errors.New("copyContainerImage plugin not allowed in pre-ci step, please remove it and try again") + } + } + + registryCredentialMap := make(map[string]bean2.RegistryCredentials) + registryDestinationImageMap := make(map[string][]string) + var allDestinationImages []string //saving all images to be reserved in this array + + for _, step := range postCiSteps { + if version, ok := pluginIdToVersionMap[step.RefPluginId]; ok { + destinationImageMap, credentialMap, err := impl.pluginInputVariableParser.HandleCopyContainerImagePluginInputVariables(step.InputVars, customTag, buildImagePath, buildImagedockerRegistryId) + if err != nil { + impl.Logger.Errorw("error in parsing copyContainerImage input variable", "err", err) + return nil, nil, "", nil, err + } + if version == buildCommonBean.COPY_CONTAINER_IMAGE_VERSION_V1 { + // this is needed in ci runner only for v1 + registryDestinationImageMap = destinationImageMap + } + for _, images := range destinationImageMap { + allDestinationImages = append(allDestinationImages, images...) + } + for k, v := range credentialMap { + registryCredentialMap[k] = v + } + } + } + + pluginArtifactStage := repository5.POST_CI + for _, image := range allDestinationImages { + if image == buildImagePath { + return nil, registryCredentialMap, pluginArtifactStage, nil, + pipelineConfigBean.ErrImagePathInUse + } + } + + var imagePathReservationIds []int + if len(allDestinationImages) > 0 { + savedCIArtifacts, err := impl.ciArtifactRepository.FindCiArtifactByImagePaths(allDestinationImages) + if err != nil { + impl.Logger.Errorw("error in fetching artifacts by image path", "err", err) + return nil, nil, pluginArtifactStage, nil, err + } + if len(savedCIArtifacts) > 0 { + // if already present in ci artifact, return "image path already in use error" + return nil, nil, pluginArtifactStage, nil, pipelineConfigBean.ErrImagePathInUse + } + imagePathReservationIds, err = impl.ReserveImagesGeneratedAtPlugin(customTagId, allDestinationImages) + if err != nil { + return nil, nil, pluginArtifactStage, imagePathReservationIds, err + } + } + return registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imagePathReservationIds, nil +} + +func (impl *HandlerServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, destinationImages []string) ([]int, error) { + var imagePathReservationIds []int + for _, image := range destinationImages { + imagePathReservationData, err := impl.customTagService.ReserveImagePath(image, customTagId) + if err != nil { + impl.Logger.Errorw("Error in marking custom tag reserved", "err", err) + return imagePathReservationIds, err + } + imagePathReservationIds = append(imagePathReservationIds, imagePathReservationData.Id) + } + return imagePathReservationIds, nil +} + +func buildCiStepsDataFromDockerBuildScripts(dockerBuildScripts []*bean.CiScript) []*pipelineConfigBean.StepObject { + // before plugin support, few variables were set as env vars in ci-runner + // these variables are now moved to global vars in plugin steps, but to avoid error in old scripts adding those variables in payload + inputVars := []*commonBean.VariableObject{ + { + Name: "DOCKER_IMAGE_TAG", + Format: "STRING", + VariableType: commonBean.VariableTypeRefGlobal, + ReferenceVariableName: "DOCKER_IMAGE_TAG", + }, + { + Name: "DOCKER_REPOSITORY", + Format: "STRING", + VariableType: commonBean.VariableTypeRefGlobal, + ReferenceVariableName: "DOCKER_REPOSITORY", + }, + { + Name: "DOCKER_REGISTRY_URL", + Format: "STRING", + VariableType: commonBean.VariableTypeRefGlobal, + ReferenceVariableName: "DOCKER_REGISTRY_URL", + }, + { + Name: "DOCKER_IMAGE", + Format: "STRING", + VariableType: commonBean.VariableTypeRefGlobal, + ReferenceVariableName: "DOCKER_IMAGE", + }, + } + var ciSteps []*pipelineConfigBean.StepObject + for _, dockerBuildScript := range dockerBuildScripts { + ciStep := &pipelineConfigBean.StepObject{ + Name: dockerBuildScript.Name, + Index: dockerBuildScript.Index, + Script: dockerBuildScript.Script, + ArtifactPaths: []string{dockerBuildScript.OutputLocation}, + StepType: string(repository.PIPELINE_STEP_TYPE_INLINE), + ExecutorType: string(repository2.SCRIPT_TYPE_SHELL), + InputVars: inputVars, + } + ciSteps = append(ciSteps, ciStep) + } + return ciSteps +} + +func (impl *HandlerServiceImpl) buildImageTag(commitHashes map[int]pipelineConfig.GitCommit, id int, wfId int) string { + dockerImageTag := "" + toAppendDevtronParamInTag := true + for _, v := range commitHashes { + if v.WebhookData.Id == 0 { + if v.Commit == "" { + continue + } + dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _getTruncatedImageTag(v.Commit)) + } else { + _targetCheckout := v.WebhookData.Data[bean.WEBHOOK_SELECTOR_TARGET_CHECKOUT_NAME] + if _targetCheckout == "" { + continue + } + // if not PR based then meaning tag based + isPRBasedEvent := v.WebhookData.EventActionType == bean.WEBHOOK_EVENT_MERGED_ACTION_TYPE + if !isPRBasedEvent && impl.config.CiCdConfig.UseImageTagFromGitProviderForTagBasedBuild { + dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _targetCheckout) + } else { + dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _getTruncatedImageTag(_targetCheckout)) + } + if isPRBasedEvent { + _sourceCheckout := v.WebhookData.Data[bean.WEBHOOK_SELECTOR_SOURCE_CHECKOUT_NAME] + dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _getTruncatedImageTag(_sourceCheckout)) + } else { + toAppendDevtronParamInTag = !impl.config.CiCdConfig.UseImageTagFromGitProviderForTagBasedBuild + } + } + } + toAppendDevtronParamInTag = toAppendDevtronParamInTag && dockerImageTag != "" + if toAppendDevtronParamInTag { + dockerImageTag = fmt.Sprintf("%s-%d-%d", dockerImageTag, id, wfId) + } + // replace / with underscore, as docker image tag doesn't support slash. it gives error + dockerImageTag = strings.ReplaceAll(dockerImageTag, "/", "_") + return dockerImageTag +} + +func getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, commitOrCheckOutData string) string { + if dockerImageTag == "" { + dockerImageTag = commitOrCheckOutData + } else { + if commitOrCheckOutData != "" { + dockerImageTag = fmt.Sprintf("%s-%s", dockerImageTag, commitOrCheckOutData) + } + } + return dockerImageTag +} + +func (impl *HandlerServiceImpl) updateCiWorkflow(request *types.WorkflowRequest, savedWf *pipelineConfig.CiWorkflow) error { + ciBuildConfig := request.CiBuildConfig + ciBuildType := string(ciBuildConfig.CiBuildType) + savedWf.CiBuildType = ciBuildType + return impl.ciService.UpdateCiWorkflowWithStage(savedWf) +} + +func (impl *HandlerServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, workflowRequest *types.WorkflowRequest) error { + // externalCi artifact is meant only for CI_JOB + if trigger.PipelineType != string(buildCommonBean.CI_JOB) { + return nil + } + + // checking if user has given run time parameters for externalCiArtifact, if given then sending git material to Ci-Runner + externalCiArtifact, exists := trigger.RuntimeParameters.GetGlobalRuntimeVariables()[buildBean.ExtraEnvVarExternalCiArtifactKey] + // validate externalCiArtifact as docker image + if exists { + externalCiArtifact = strings.TrimSpace(externalCiArtifact) + if !strings.Contains(externalCiArtifact, ":") { + if utils.IsValidDockerTagName(externalCiArtifact) { + fullImageUrl, err := utils.BuildDockerImagePath(bean4.DockerRegistryInfo{ + DockerImageTag: externalCiArtifact, + DockerRegistryId: workflowRequest.DockerRegistryId, + DockerRegistryType: workflowRequest.DockerRegistryType, + DockerRegistryURL: workflowRequest.DockerRegistryURL, + DockerRepository: workflowRequest.DockerRepository, + }) + if err != nil { + impl.Logger.Errorw("Error in building docker image", "err", err) + return err + } + externalCiArtifact = fullImageUrl + } else { + impl.Logger.Errorw("validation error", "externalCiArtifact", externalCiArtifact) + return fmt.Errorf("invalid image name or url given in externalCiArtifact") + } + + } + // This will overwrite the existing runtime parameters value for constants.externalCiArtifact + trigger.RuntimeParameters = trigger.RuntimeParameters.AddRuntimeGlobalVariable(buildBean.ExtraEnvVarExternalCiArtifactKey, externalCiArtifact) + var artifactExists bool + var err error + + imageDigest, ok := trigger.RuntimeParameters.GetGlobalRuntimeVariables()[buildBean.ExtraEnvVarImageDigestKey] + if !ok || len(imageDigest) == 0 { + artifactExists, err = impl.ciArtifactRepository.IfArtifactExistByImage(externalCiArtifact, trigger.PipelineId) + if err != nil { + impl.Logger.Errorw("error in fetching ci artifact", "err", err) + return err + } + if artifactExists { + impl.Logger.Errorw("ci artifact already exists with same image name", "artifact", externalCiArtifact) + return fmt.Errorf("ci artifact already exists with same image name") + } + workflowRequest, err = impl.updateWorkflowRequestForDigestPull(trigger.PipelineId, workflowRequest) + if err != nil { + impl.Logger.Errorw("error in updating workflow request", "err", err) + return err + } + } else { + artifactExists, err = impl.ciArtifactRepository.IfArtifactExistByImageDigest(imageDigest, externalCiArtifact, trigger.PipelineId) + if err != nil { + impl.Logger.Errorw("error in fetching ci artifact", "err", err, "imageDigest", imageDigest) + return err + } + if artifactExists { + impl.Logger.Errorw("ci artifact already exists with same digest", "artifact", externalCiArtifact) + return fmt.Errorf("ci artifact already exists with same digest") + } + } + } + if trigger.PipelineType == string(buildCommonBean.CI_JOB) && len(ciMaterials) != 0 && !exists && externalCiArtifact == "" { + ciMaterials[0].GitMaterial = nil + ciMaterials[0].GitMaterialId = 0 + } + return nil +} + +func _getTruncatedImageTag(imageTag string) string { + _length := len(imageTag) + if _length == 0 { + return imageTag + } + + _truncatedLength := 8 + + if _length < _truncatedLength { + return imageTag + } else { + return imageTag[:_truncatedLength] + } +} + +func (impl *HandlerServiceImpl) markCurrentCiWorkflowFailed(savedCiWf *pipelineConfig.CiWorkflow, validationErr error) error { + // currently such requirement is not there + if savedCiWf == nil { + return nil + } + if savedCiWf.Id != 0 && slices.Contains(cdWorkflow.WfrTerminalStatusList, savedCiWf.Status) { + impl.Logger.Debug("workflow is already in terminal state", "status", savedCiWf.Status, "workflowId", savedCiWf.Id, "message", savedCiWf.Message) + return nil + } + + savedCiWf.Status = cdWorkflow.WorkflowFailed + savedCiWf.Message = validationErr.Error() + savedCiWf.FinishedOn = time.Now() + + var dbErr error + if savedCiWf.Id == 0 { + dbErr = impl.ciService.SaveCiWorkflowWithStage(savedCiWf) + } else { + dbErr = impl.ciService.UpdateCiWorkflowWithStage(savedCiWf) + } + + if dbErr != nil { + impl.Logger.Errorw("save/update workflow error", "err", dbErr) + return dbErr + } + return nil +} + +func (impl *HandlerServiceImpl) CancelBuild(workflowId int, forceAbort bool) (int, error) { + workflow, err := impl.ciWorkflowRepository.FindById(workflowId) + if err != nil { + impl.Logger.Errorw("error in finding ci-workflow by workflow id", "ciWorkflowId", workflowId, "err", err) + return 0, err + } + isExt := workflow.Namespace != constants2.DefaultCiWorkflowNamespace + var env *repository6.Environment + var restConfig *rest.Config + if isExt { + restConfig, err = impl.getRestConfig(workflow) + if err != nil { + return 0, err + } + } + // Terminate workflow + cancelWfDtoRequest := &types.CancelWfRequestDto{ + ExecutorType: workflow.ExecutorType, + WorkflowName: workflow.Name, + Namespace: workflow.Namespace, + RestConfig: restConfig, + IsExt: isExt, + Environment: env, + } + // Terminate workflow + err = impl.workflowService.TerminateWorkflow(cancelWfDtoRequest) + if err != nil && forceAbort { + impl.Logger.Errorw("error in terminating workflow, with force abort flag flag as true", "workflowName", workflow.Name, "err", err) + + cancelWfDtoRequest.WorkflowGenerateName = fmt.Sprintf("%d-%s", workflowId, workflow.Name) + err1 := impl.workflowService.TerminateDanglingWorkflows(cancelWfDtoRequest) + if err1 != nil { + impl.Logger.Errorw("error in terminating dangling workflows", "cancelWfDtoRequest", cancelWfDtoRequest, "err", err) + // ignoring error here in case of force abort, confirmed from product + } + } else if err != nil && strings.Contains(err.Error(), "cannot find workflow") { + return 0, &util.ApiError{Code: "200", HttpStatusCode: http.StatusBadRequest, UserMessage: err.Error()} + } else if err != nil { + impl.Logger.Errorw("cannot terminate wf", "err", err) + return 0, err + } + if forceAbort { + err = impl.handleForceAbortCaseForCi(workflow, forceAbort) + if err != nil { + impl.Logger.Errorw("error in handleForceAbortCaseForCi", "forceAbortFlag", forceAbort, "workflow", workflow, "err", err) + return 0, err + } + return workflow.Id, nil + } + + workflow.Status = cdWorkflow.WorkflowCancel + if workflow.ExecutorType == cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM { + workflow.PodStatus = "Failed" + workflow.Message = constants2.TERMINATE_MESSAGE + } + err = impl.ciService.UpdateCiWorkflowWithStage(workflow) + if err != nil { + impl.Logger.Errorw("cannot update deleted workflow status, but wf deleted", "err", err) + return 0, err + } + imagePathReservationId := workflow.ImagePathReservationId + err = impl.customTagService.DeactivateImagePathReservation(imagePathReservationId) + if err != nil { + impl.Logger.Errorw("error in marking image tag unreserved", "err", err) + return 0, err + } + imagePathReservationIds := workflow.ImagePathReservationIds + if len(imagePathReservationIds) > 0 { + err = impl.customTagService.DeactivateImagePathReservationByImageIds(imagePathReservationIds) + if err != nil { + impl.Logger.Errorw("error in marking image tag unreserved", "err", err) + return 0, err + } + } + return workflow.Id, nil +} + +func (impl *HandlerServiceImpl) handleForceAbortCaseForCi(workflow *pipelineConfig.CiWorkflow, forceAbort bool) error { + isWorkflowInNonTerminalStage := workflow.Status == string(v1alpha1.NodePending) || workflow.Status == string(v1alpha1.NodeRunning) + if !isWorkflowInNonTerminalStage { + if forceAbort { + return impl.updateWorkflowForForceAbort(workflow) + } else { + return &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "cannot cancel build, build not in progress"} + } + } + //this arises when someone deletes the workflow in resource browser and wants to force abort a ci + if workflow.Status == string(v1alpha1.NodeRunning) && forceAbort { + return impl.updateWorkflowForForceAbort(workflow) + } + return nil +} + +func (impl *HandlerServiceImpl) updateWorkflowForForceAbort(workflow *pipelineConfig.CiWorkflow) error { + workflow.Status = cdWorkflow.WorkflowCancel + workflow.PodStatus = string(bean.Failed) + workflow.Message = constants2.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE + err := impl.ciService.UpdateCiWorkflowWithStage(workflow) + if err != nil { + impl.Logger.Errorw("error in updating workflow status", "err", err) + return err + } + return nil +} + +func (impl *HandlerServiceImpl) getRestConfig(workflow *pipelineConfig.CiWorkflow) (*rest.Config, error) { + env, err := impl.envRepository.FindById(workflow.EnvironmentId) + if err != nil { + impl.Logger.Errorw("could not fetch stage env", "err", err) + return nil, err + } + + clusterBean := adapter2.GetClusterBean(*env.Cluster) + + clusterConfig := clusterBean.GetClusterConfig() + restConfig, err := impl.K8sUtil.GetRestConfigByCluster(clusterConfig) + if err != nil { + impl.Logger.Errorw("error in getting rest config by cluster id", "err", err) + return nil, err + } + return restConfig, nil +} + +func (impl *HandlerServiceImpl) GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, func() error, error) { + ciWorkflow, err := impl.ciWorkflowRepository.FindById(workflowId) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return nil, nil, err + } + return impl.getWorkflowLogs(ciWorkflow) +} + +func (impl *HandlerServiceImpl) getWorkflowLogs(ciWorkflow *pipelineConfig.CiWorkflow) (*bufio.Reader, func() error, error) { + if string(v1alpha1.NodePending) == ciWorkflow.PodStatus { + return bufio.NewReader(strings.NewReader("")), func() error { return nil }, nil + } + ciLogRequest := types.BuildLogRequest{ + PodName: ciWorkflow.PodName, + Namespace: ciWorkflow.Namespace, + } + isExt := false + clusterConfig := &k8s.ClusterConfig{} + if ciWorkflow.EnvironmentId != 0 { + env, err := impl.envRepository.FindById(ciWorkflow.EnvironmentId) + if err != nil { + return nil, nil, err + } + var clusterBean clusterBean.ClusterBean + if env != nil && env.Cluster != nil { + clusterBean = adapter2.GetClusterBean(*env.Cluster) + } + clusterConfig = clusterBean.GetClusterConfig() + isExt = true + } + + logStream, cleanUp, err := impl.ciLogService.FetchRunningWorkflowLogs(ciLogRequest, clusterConfig, isExt) + if logStream == nil || err != nil { + if !ciWorkflow.BlobStorageEnabled { + return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "logs-not-stored-in-repository"} + } else if string(v1alpha1.NodeSucceeded) == ciWorkflow.Status || string(v1alpha1.NodeError) == ciWorkflow.Status || string(v1alpha1.NodeFailed) == ciWorkflow.Status || ciWorkflow.Status == cdWorkflow.WorkflowCancel { + impl.Logger.Debugw("pod is not live", "podName", ciWorkflow.PodName, "err", err) + return impl.getLogsFromRepository(ciWorkflow, clusterConfig, isExt) + } + if err != nil { + impl.Logger.Errorw("err on fetch workflow logs", "err", err) + return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: err.Error()} + } else if logStream == nil { + return nil, cleanUp, fmt.Errorf("no logs found for pod %s", ciWorkflow.PodName) + } + } + logReader := bufio.NewReader(logStream) + return logReader, cleanUp, err +} + +func (impl *HandlerServiceImpl) getLogsFromRepository(ciWorkflow *pipelineConfig.CiWorkflow, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { + impl.Logger.Debug("getting historic logs", "ciWorkflowId", ciWorkflow.Id) + ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() + ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion + logsFilePath := impl.config.GetDefaultBuildLogsKeyPrefix() + "/" + ciWorkflow.Name + "/main.log" // this is for backward compatibilty + if strings.Contains(ciWorkflow.LogLocation, "main.log") { + logsFilePath = ciWorkflow.LogLocation + } + ciLogRequest := types.BuildLogRequest{ + PipelineId: ciWorkflow.CiPipelineId, + WorkflowId: ciWorkflow.Id, + PodName: ciWorkflow.PodName, + LogsFilePath: logsFilePath, + CloudProvider: impl.config.CloudProvider, + AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, + AccountName: impl.config.AzureAccountName, + BlobContainerName: impl.config.AzureBlobContainerCiLog, + AccountKey: impl.config.AzureAccountKey, + }, + AwsS3BaseConfig: &blob_storage.AwsS3BaseConfig{ + AccessKey: impl.config.BlobStorageS3AccessKey, + Passkey: impl.config.BlobStorageS3SecretKey, + EndpointUrl: impl.config.BlobStorageS3Endpoint, + IsInSecure: impl.config.BlobStorageS3EndpointInsecure, + BucketName: ciConfigLogsBucket, + Region: ciConfigCiCacheRegion, + VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, + }, + GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ + BucketName: ciConfigLogsBucket, + CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, + }, + } + useExternalBlobStorage := pipeline2.IsExternalBlobStorageEnabled(isExt, impl.config.UseBlobStorageConfigInCiWorkflow) + if useExternalBlobStorage { + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) + if err != nil { + impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, nil, err + } + rq := &ciLogRequest + rq.SetBuildLogRequest(cmConfig, secretConfig) + } + oldLogsStream, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, ciLogRequest) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return nil, nil, err + } + logReader := bufio.NewReader(oldLogsStream) + return logReader, cleanUp, err +} + +func (impl *HandlerServiceImpl) GetHistoricBuildLogs(workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) { + var err error + if ciWorkflow == nil { + ciWorkflow, err = impl.ciWorkflowRepository.FindById(workflowId) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return nil, err + } + } + ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() + ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion + ciLogRequest := types.BuildLogRequest{ + PipelineId: ciWorkflow.CiPipelineId, + WorkflowId: ciWorkflow.Id, + PodName: ciWorkflow.PodName, + LogsFilePath: ciWorkflow.LogLocation, + CloudProvider: impl.config.CloudProvider, + AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, + AccountName: impl.config.AzureAccountName, + BlobContainerName: impl.config.AzureBlobContainerCiLog, + AccountKey: impl.config.AzureAccountKey, + }, + AwsS3BaseConfig: &blob_storage.AwsS3BaseConfig{ + AccessKey: impl.config.BlobStorageS3AccessKey, + Passkey: impl.config.BlobStorageS3SecretKey, + EndpointUrl: impl.config.BlobStorageS3Endpoint, + IsInSecure: impl.config.BlobStorageS3EndpointInsecure, + BucketName: ciConfigLogsBucket, + Region: ciConfigCiCacheRegion, + VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, + }, + GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ + BucketName: ciConfigLogsBucket, + CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, + }, + } + useExternalBlobStorage := pipeline2.IsExternalBlobStorageEnabled(ciWorkflow.IsExternalRunInJobType(), impl.config.UseBlobStorageConfigInCiWorkflow) + if useExternalBlobStorage { + envBean, err := impl.envService.FindById(ciWorkflow.EnvironmentId) + if err != nil { + impl.Logger.Errorw("error in getting envBean by envId", "err", err, "envId", ciWorkflow.EnvironmentId) + return nil, err + } + clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(envBean.ClusterId) + if err != nil { + impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig by clusterId", "err", err, "clusterId", envBean.ClusterId) + return nil, err + } + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) + if err != nil { + impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, err + } + rq := &ciLogRequest + rq.SetBuildLogRequest(cmConfig, secretConfig) + } + logsFile, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, ciLogRequest) + logs, err := ioutil.ReadFile(logsFile.Name()) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return map[string]string{}, err + } + logStr := string(logs) + resp := make(map[string]string) + resp["logs"] = logStr + defer cleanUp() + return resp, err +} + +func (impl *HandlerServiceImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) { + ciWorkflow, err := impl.ciWorkflowRepository.FindById(buildId) + if err != nil { + impl.Logger.Errorw("unable to fetch ciWorkflow", "err", err) + return nil, err + } + useExternalBlobStorage := pipeline2.IsExternalBlobStorageEnabled(ciWorkflow.IsExternalRunInJobType(), impl.config.UseBlobStorageConfigInCiWorkflow) + if !ciWorkflow.BlobStorageEnabled { + return nil, errors.New("logs-not-stored-in-repository") + } + + if ciWorkflow.CiPipelineId != pipelineId { + impl.Logger.Error("invalid request, wf not in pipeline") + return nil, errors.New("invalid request, wf not in pipeline") + } + + ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() + item := strconv.Itoa(ciWorkflow.Id) + ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion + azureBlobConfig := &blob_storage.AzureBlobBaseConfig{ + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, + AccountName: impl.config.AzureAccountName, + BlobContainerName: impl.config.AzureBlobContainerCiLog, + AccountKey: impl.config.AzureAccountKey, + } + awsS3BaseConfig := &blob_storage.AwsS3BaseConfig{ + AccessKey: impl.config.BlobStorageS3AccessKey, + Passkey: impl.config.BlobStorageS3SecretKey, + EndpointUrl: impl.config.BlobStorageS3Endpoint, + IsInSecure: impl.config.BlobStorageS3EndpointInsecure, + BucketName: ciConfigLogsBucket, + Region: ciConfigCiCacheRegion, + VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, + } + gcpBlobBaseConfig := &blob_storage.GcpBlobBaseConfig{ + BucketName: ciConfigLogsBucket, + CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, + } + + ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() + key := fmt.Sprintf(ciArtifactLocationFormat, ciWorkflow.Id, ciWorkflow.Id) + if len(ciWorkflow.CiArtifactLocation) != 0 && util2.IsValidUrlSubPath(ciWorkflow.CiArtifactLocation) { + key = ciWorkflow.CiArtifactLocation + } else if util2.IsValidUrlSubPath(key) { + impl.ciWorkflowRepository.MigrateCiArtifactLocation(ciWorkflow.Id, key) + } + baseLogLocationPathConfig := impl.config.BaseLogLocationPath + blobStorageService := blob_storage.NewBlobStorageServiceImpl(nil) + destinationKey := filepath.Clean(filepath.Join(baseLogLocationPathConfig, item)) + request := &blob_storage.BlobStorageRequest{ + StorageType: impl.config.CloudProvider, + SourceKey: key, + DestinationKey: destinationKey, + AzureBlobBaseConfig: azureBlobConfig, + AwsS3BaseConfig: awsS3BaseConfig, + GcpBlobBaseConfig: gcpBlobBaseConfig, + } + if useExternalBlobStorage { + envBean, err := impl.envService.FindById(ciWorkflow.EnvironmentId) + if err != nil { + impl.Logger.Errorw("error in getting envBean by envId", "err", err, "envId", ciWorkflow.EnvironmentId) + return nil, err + } + clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(envBean.ClusterId) + if err != nil { + impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig by clusterId", "err", err, "clusterId", envBean.ClusterId) + return nil, err + } + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) + if err != nil { + impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, err + } + request = pipeline2.UpdateRequestWithExtClusterCmAndSecret(request, cmConfig, secretConfig) + } + _, numBytes, err := blobStorageService.Get(request) + if err != nil { + impl.Logger.Errorw("error occurred while downloading file", "request", request, "error", err) + return nil, errors.New("failed to download resource") + } + + file, err := os.Open(destinationKey) + if err != nil { + impl.Logger.Errorw("unable to open file", "file", item, "err", err) + return nil, errors.New("unable to open file") + } + + impl.Logger.Infow("Downloaded ", "filename", file.Name(), "bytes", numBytes) + return file, nil +} diff --git a/pkg/build/trigger/HandlerService_ent.go b/pkg/build/trigger/HandlerService_ent.go new file mode 100644 index 0000000000..3938288091 --- /dev/null +++ b/pkg/build/trigger/HandlerService_ent.go @@ -0,0 +1,104 @@ +package trigger + +import ( + "encoding/json" + "github.com/devtron-labs/common-lib/imageScan/bean" + repository3 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/util" + bean3 "github.com/devtron-labs/devtron/pkg/bean" + "github.com/devtron-labs/devtron/pkg/bean/common" + bean2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" + pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/types" + "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" + "github.com/devtron-labs/devtron/pkg/resourceQualifiers" + "net/http" +) + +func (impl *HandlerServiceImpl) updateRuntimeParamsForAutoCI(ciPipelineId int, runtimeParameters *common.RuntimeParameters) (*common.RuntimeParameters, error) { + return runtimeParameters, nil +} + +func (impl *HandlerServiceImpl) getRuntimeParamsForBuildingManualTriggerHashes(ciTriggerRequest bean3.CiTriggerRequest) *common.RuntimeParameters { + return common.NewRuntimeParameters() +} + +func (impl *HandlerServiceImpl) fetchImageScanExecutionMedium() (*repository.ScanToolMetadata, bean.ScanExecutionMedium, error) { + return &repository.ScanToolMetadata{}, "", nil +} + +func (impl *HandlerServiceImpl) fetchImageScanExecutionStepsForWfRequest(scanToolMetadata *repository.ScanToolMetadata) ([]*types.ImageScanningSteps, []*pipelineConfigBean.RefPluginObject, error) { + return nil, nil, nil +} + +func (impl *HandlerServiceImpl) checkIfCITriggerIsBlocked(pipeline *pipelineConfig.CiPipeline, + ciMaterials []*pipelineConfig.CiPipelineMaterial, isJob bool) (bool, error) { + return false, nil +} + +func (impl *HandlerServiceImpl) handleWFIfCITriggerIsBlocked(ciWorkflow *pipelineConfig.CiWorkflow) (*pipelineConfig.CiWorkflow, error) { + impl.Logger.Errorw("cannot trigger pipeline, blocked by mandatory plugin policy", "ciPipelineId", ciWorkflow.CiPipelineId) + return &pipelineConfig.CiWorkflow{}, util.GetApiErrorAdapter(http.StatusInternalServerError, "500", "Invalid flow access, corrupt data possibility", "Invalid flow access, corrupt data possibility") +} + +func (impl *HandlerServiceImpl) checkArgoSetupRequirement(envModal *repository2.Environment) error { + return nil +} + +func (impl *HandlerServiceImpl) updateWorkflowRequestForDigestPull(pipelineId int, workflowRequest *types.WorkflowRequest) (*types.WorkflowRequest, error) { + return workflowRequest, nil +} + +func (impl *HandlerServiceImpl) updateCIProjectDetailWithCloningMode(appId int, ciMaterial *pipelineConfig.CiPipelineMaterial, + ciProjectDetail pipelineConfigBean.CiProjectDetails) (pipelineConfigBean.CiProjectDetails, error) { + return ciProjectDetail, nil +} + +func (impl *HandlerServiceImpl) updateWorkflowRequestWithRemoteConnConf(dockerRegistry *repository3.DockerArtifactStore, + workflowRequest *types.WorkflowRequest) (*types.WorkflowRequest, error) { + return workflowRequest, nil +} + +func (impl *HandlerServiceImpl) updateWorkflowRequestWithEntSupportData(workflowRequest *types.WorkflowRequest) *types.WorkflowRequest { + return workflowRequest +} + +func (impl *HandlerServiceImpl) updateWorkflowRequestWithBuildCacheData(workflowRequest *types.WorkflowRequest, + scope resourceQualifiers.Scope) (*types.WorkflowRequest, error) { + workflowRequest.BuildxCacheModeMin = impl.buildxCacheFlags.BuildxCacheModeMin + workflowRequest.AsyncBuildxCacheExport = impl.buildxCacheFlags.AsyncBuildxCacheExport + return workflowRequest, nil +} + +func (impl *HandlerServiceImpl) canSetK8sDriverData(workflowRequest *types.WorkflowRequest) bool { + return impl.config != nil && impl.config.BuildxK8sDriverOptions != "" && workflowRequest.CiBuildConfig != nil && + workflowRequest.CiBuildConfig.DockerBuildConfig != nil +} + +func (impl *HandlerServiceImpl) getK8sDriverOptions(workflowRequest *types.WorkflowRequest, targetPlatforms string) ([]map[string]string, error) { + buildxK8sDriverOptions := make([]map[string]string, 0) + err := json.Unmarshal([]byte(impl.config.BuildxK8sDriverOptions), &buildxK8sDriverOptions) + if err != nil { + return nil, err + } + return buildxK8sDriverOptions, nil +} + +func (impl *HandlerServiceImpl) updateCIBuildConfig(ciBuildConfigBean *bean2.CiBuildConfigBean) *bean2.CiBuildConfigBean { + defaultTargetPlatform := impl.config.DefaultTargetPlatform + useBuildx := impl.config.UseBuildx + if ciBuildConfigBean.DockerBuildConfig != nil { + if ciBuildConfigBean.DockerBuildConfig.TargetPlatform == "" && useBuildx { + ciBuildConfigBean.DockerBuildConfig.TargetPlatform = defaultTargetPlatform + ciBuildConfigBean.DockerBuildConfig.UseBuildx = useBuildx + } + ciBuildConfigBean.DockerBuildConfig.BuildxProvenanceMode = impl.config.BuildxProvenanceMode + } + return ciBuildConfigBean +} + +func updateBuildPrePostStepDataReq(req *pipelineConfigBean.BuildPrePostStepDataRequest, trigger types.Trigger) *pipelineConfigBean.BuildPrePostStepDataRequest { + return req +} diff --git a/pkg/build/trigger/wire_trigger.go b/pkg/build/trigger/wire_trigger.go new file mode 100644 index 0000000000..aca718e3cd --- /dev/null +++ b/pkg/build/trigger/wire_trigger.go @@ -0,0 +1,10 @@ +package trigger + +import ( + "github.com/google/wire" +) + +var WireSet = wire.NewSet( + NewHandlerServiceImpl, + wire.Bind(new(HandlerService), new(*HandlerServiceImpl)), +) diff --git a/pkg/build/wire_build.go b/pkg/build/wire_build.go index 6610dd24cc..32c2fc0b0b 100644 --- a/pkg/build/wire_build.go +++ b/pkg/build/wire_build.go @@ -20,6 +20,7 @@ import ( "github.com/devtron-labs/devtron/pkg/build/artifacts" "github.com/devtron-labs/devtron/pkg/build/git" "github.com/devtron-labs/devtron/pkg/build/pipeline" + "github.com/devtron-labs/devtron/pkg/build/trigger" "github.com/google/wire" ) @@ -27,4 +28,5 @@ var WireSet = wire.NewSet( artifacts.WireSet, pipeline.WireSet, git.GitWireSet, + trigger.WireSet, ) diff --git a/pkg/bulkAction/service/BulkUpdateService.go b/pkg/bulkAction/service/BulkUpdateService.go index 3b94648b69..3e6c0f7f78 100644 --- a/pkg/bulkAction/service/BulkUpdateService.go +++ b/pkg/bulkAction/service/BulkUpdateService.go @@ -35,6 +35,7 @@ import ( "github.com/devtron-labs/devtron/internal/util" appWorkflow2 "github.com/devtron-labs/devtron/pkg/appWorkflow" bean2 "github.com/devtron-labs/devtron/pkg/bean" + "github.com/devtron-labs/devtron/pkg/build/trigger" bean4 "github.com/devtron-labs/devtron/pkg/bulkAction/bean" "github.com/devtron-labs/devtron/pkg/bulkAction/utils" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" @@ -102,6 +103,7 @@ type BulkUpdateServiceImpl struct { chartRefService chartRef.ChartRefService deployedAppService deployedApp.DeployedAppService cdPipelineEventPublishService out.CDPipelineEventPublishService + ciHandlerService trigger.HandlerService } func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateRepository, @@ -121,7 +123,8 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateReposito deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, chartRefService chartRef.ChartRefService, deployedAppService deployedApp.DeployedAppService, - cdPipelineEventPublishService out.CDPipelineEventPublishService) *BulkUpdateServiceImpl { + cdPipelineEventPublishService out.CDPipelineEventPublishService, + ciHandlerService trigger.HandlerService) *BulkUpdateServiceImpl { return &BulkUpdateServiceImpl{ bulkUpdateRepository: bulkUpdateRepository, logger: logger, @@ -141,6 +144,7 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateReposito chartRefService: chartRefService, deployedAppService: deployedAppService, cdPipelineEventPublishService: cdPipelineEventPublishService, + ciHandlerService: ciHandlerService, } } @@ -1467,7 +1471,7 @@ func (impl BulkUpdateServiceImpl) BulkBuildTrigger(request *bean4.BulkApplicatio } ciTriggerRequest := latestCommitsMap[pipeline.CiPipelineId] - _, err = impl.ciHandler.HandleCIManual(ciTriggerRequest) + _, err = impl.ciHandlerService.HandleCIManual(ciTriggerRequest) if err != nil { impl.logger.Errorw("service err, HandleCIManual", "err", err, "ciTriggerRequest", ciTriggerRequest) //return nil, err diff --git a/pkg/chartRepo/ChartRepositoryService.go b/pkg/chartRepo/ChartRepositoryService.go index 1df2c48e43..8cdef0a9fd 100644 --- a/pkg/chartRepo/ChartRepositoryService.go +++ b/pkg/chartRepo/ChartRepositoryService.go @@ -501,7 +501,7 @@ func (impl *ChartRepositoryServiceImpl) ValidateAndUpdateChartRepo(request *Char } func (impl *ChartRepositoryServiceImpl) TriggerChartSyncManual(chartProviderConfig *ChartProviderConfig) error { - defaultClusterBean, err := impl.clusterReadService.FindOne(bean2.DEFAULT_CLUSTER) + defaultClusterBean, err := impl.clusterReadService.FindOne(bean2.DefaultCluster) if err != nil { impl.logger.Errorw("defaultClusterBean err, TriggerChartSyncManual", "err", err) return err diff --git a/pkg/cluster/ClusterService.go b/pkg/cluster/ClusterService.go index 9bab50066a..1b00c2f84f 100644 --- a/pkg/cluster/ClusterService.go +++ b/pkg/cluster/ClusterService.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "fmt" + informerBean "github.com/devtron-labs/common-lib/informer" "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/pkg/cluster/adapter" "github.com/devtron-labs/devtron/pkg/cluster/bean" @@ -28,7 +29,6 @@ import ( cronUtil "github.com/devtron-labs/devtron/util/cron" "github.com/robfig/cron/v3" "log" - "net/http" "net/url" "sync" "time" @@ -38,8 +38,8 @@ import ( casbin2 "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" repository3 "github.com/devtron-labs/devtron/pkg/auth/user/repository" "github.com/devtron-labs/devtron/pkg/k8s/informer" - errors1 "github.com/juju/errors" - "k8s.io/apimachinery/pkg/api/errors" + customErr "github.com/juju/errors" + k8sError "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes" @@ -56,18 +56,6 @@ import ( "go.uber.org/zap" ) -const ( - DEFAULT_CLUSTER = "default_cluster" - DEFAULT_NAMESPACE = "default" - CLUSTER_MODIFY_EVENT_SECRET_TYPE = "cluster.request/modify" - CLUSTER_ACTION_ADD = "add" - CLUSTER_ACTION_UPDATE = "update" - SECRET_FIELD_CLUSTER_ID = "cluster_id" - SECRET_FIELD_UPDATED_ON = "updated_on" - SECRET_FIELD_ACTION = "action" - TokenFilePath = "/var/run/secrets/kubernetes.io/serviceaccount/token" -) - type ClusterService interface { Save(parent context.Context, bean *bean.ClusterBean, userId int32) (*bean.ClusterBean, error) UpdateClusterDescription(bean *bean.ClusterBean, userId int32) error @@ -195,7 +183,7 @@ func (impl *ClusterServiceImpl) Save(parent context.Context, bean *bean.ClusterB if err != nil { if len(err.Error()) > 2000 { - err = errors.NewBadRequest("unable to connect to cluster") + err = k8sError.NewBadRequest("unable to connect to cluster") } return nil, err } @@ -248,16 +236,16 @@ func (impl *ClusterServiceImpl) Save(parent context.Context, bean *bean.ClusterB secretName := ParseSecretNameForKubelinkInformer(bean.Id) data := make(map[string][]byte) - data[SECRET_FIELD_CLUSTER_ID] = []byte(fmt.Sprintf("%v", bean.Id)) - data[SECRET_FIELD_ACTION] = []byte(CLUSTER_ACTION_ADD) - data[SECRET_FIELD_UPDATED_ON] = []byte(time.Now().String()) // this field will ensure that informer detects change as other fields can be constant even if cluster config changes - _, err = impl.K8sUtil.CreateSecret(DEFAULT_NAMESPACE, data, secretName, CLUSTER_MODIFY_EVENT_SECRET_TYPE, k8sClient, nil, nil) + data[informerBean.SecretFieldClusterId] = []byte(fmt.Sprintf("%v", bean.Id)) + data[informerBean.SecretFieldAction] = []byte(informerBean.ClusterActionAdd) + data[clusterBean.SecretFieldUpdatedOn] = []byte(time.Now().String()) // this field will ensure that informer detects change as other fields can be constant even if cluster config changes + // TODO Asutosh: Why not UPSERT ?? + _, err = impl.K8sUtil.CreateSecret(clusterBean.DefaultNamespace, data, secretName, informerBean.ClusterModifyEventSecretType, k8sClient, nil, nil) if err != nil { - impl.logger.Errorw("error in updating secret for informers") + impl.logger.Errorw("error in creating secret for informers", "secretName", secretName, "err", err) return bean, nil } - - return bean, err + return bean, nil } // UpdateClusterDescription is new api service logic to only update description, this should be done in cluster update operation only @@ -415,7 +403,7 @@ func (impl *ClusterServiceImpl) Update(ctx context.Context, bean *bean.ClusterBe } if bean.ServerUrl != model.ServerUrl || bean.InsecureSkipTLSVerify != model.InsecureSkipTlsVerify || dbConfigBearerToken != requestConfigBearerToken || dbConfigTlsKey != requestConfigTlsKey || dbConfigCertData != requestConfigCertData || dbConfigCAData != requestConfigCAData { - if bean.ClusterName == clusterBean.DEFAULT_CLUSTER { + if bean.ClusterName == clusterBean.DefaultCluster { impl.logger.Errorw("default_cluster is reserved by the system and cannot be updated, default_cluster", "name", bean.ClusterName) return nil, fmt.Errorf("default_cluster is reserved by the system and cannot be updated") } @@ -480,37 +468,47 @@ func (impl *ClusterServiceImpl) Update(ctx context.Context, bean *bean.ClusterBe impl.SyncNsInformer(bean) } impl.logger.Infow("saving secret for cluster informer") + if bean.HasConfigOrUrlChanged { + data := make(map[string][]byte) + data[informerBean.SecretFieldClusterId] = []byte(fmt.Sprintf("%v", bean.Id)) + data[informerBean.SecretFieldAction] = []byte(informerBean.ClusterActionUpdate) + data[clusterBean.SecretFieldUpdatedOn] = []byte(time.Now().String()) // this field will ensure that informer detects change as other fields can be constant even if cluster config changes + if err = impl.upsertClusterSecret(bean, data); err != nil { + impl.logger.Errorw("error upserting cluster secret", "data", data, "err", err) + // TODO Asutosh: why error is not propagated ?? + return bean, nil + } + } + return bean, nil +} + +func (impl *ClusterServiceImpl) upsertClusterSecret(bean *bean.ClusterBean, data map[string][]byte) error { k8sClient, err := impl.K8sUtil.GetCoreV1ClientInCluster() if err != nil { - return bean, nil + impl.logger.Errorw("error in getting k8s client", "err", err) + return err } - // below secret will act as an event for informer running on secret object in kubelink - if bean.HasConfigOrUrlChanged { - secretName := ParseSecretNameForKubelinkInformer(bean.Id) - secret, err := impl.K8sUtil.GetSecret(DEFAULT_NAMESPACE, secretName, k8sClient) - statusError, _ := err.(*errors.StatusError) - if err != nil && statusError.Status().Code != http.StatusNotFound { - impl.logger.Errorw("secret not found", "err", err) - return bean, nil + // below secret will act as an event for informer running on a secret object in kubelink and kubewatch + secretName := ParseSecretNameForKubelinkInformer(bean.Id) + secret, err := impl.K8sUtil.GetSecret(clusterBean.DefaultNamespace, secretName, k8sClient) + if err != nil && !k8sError.IsNotFound(err) { + impl.logger.Errorw("error in getting cluster secret", "secretName", secretName, "err", err) + return err + } else if k8sError.IsNotFound(err) { + _, err = impl.K8sUtil.CreateSecret(clusterBean.DefaultNamespace, data, secretName, informerBean.ClusterModifyEventSecretType, k8sClient, nil, nil) + if err != nil { + impl.logger.Errorw("error in creating secret for informers", "secretName", secretName, "err", err) + return err } - data := make(map[string][]byte) - data[SECRET_FIELD_CLUSTER_ID] = []byte(fmt.Sprintf("%v", bean.Id)) - data[SECRET_FIELD_ACTION] = []byte(CLUSTER_ACTION_UPDATE) - data[SECRET_FIELD_UPDATED_ON] = []byte(time.Now().String()) // this field will ensure that informer detects change as other fields can be constant even if cluster config changes - if secret == nil { - _, err = impl.K8sUtil.CreateSecret(DEFAULT_NAMESPACE, data, secretName, CLUSTER_MODIFY_EVENT_SECRET_TYPE, k8sClient, nil, nil) - if err != nil { - impl.logger.Errorw("error in creating secret for informers") - } - } else { - secret.Data = data - secret, err = impl.K8sUtil.UpdateSecret(DEFAULT_NAMESPACE, secret, k8sClient) - if err != nil { - impl.logger.Errorw("error in updating secret for informers") - } + } else { + secret.Data = data + secret, err = impl.K8sUtil.UpdateSecret(clusterBean.DefaultNamespace, secret, k8sClient) + if err != nil { + impl.logger.Errorw("error in updating secret for informers", "secretName", secretName, "err", err) + return err } } - return bean, nil + return nil } func (impl *ClusterServiceImpl) SyncNsInformer(bean *bean.ClusterBean) { @@ -609,8 +607,9 @@ func (impl *ClusterServiceImpl) DeleteFromDb(bean *bean.ClusterBean, userId int3 impl.logger.Errorw("error in getting in cluster k8s client", "err", err, "clusterName", bean.ClusterName) return "", nil } + // TODO Asutosh: why we maintain this duplicate code ?? secretName := ParseSecretNameForKubelinkInformer(bean.Id) - err = impl.K8sUtil.DeleteSecret(DEFAULT_NAMESPACE, secretName, k8sClient) + err = impl.K8sUtil.DeleteSecret(clusterBean.DefaultNamespace, secretName, k8sClient) impl.logger.Errorw("error in deleting secret", "error", err) return existingCluster.ClusterName, nil } @@ -621,7 +620,7 @@ func (impl *ClusterServiceImpl) CheckIfConfigIsValid(cluster *bean.ClusterBean) if err != nil { if _, ok := err.(*url.Error); ok { return fmt.Errorf("Incorrect server url : %v", err) - } else if statusError, ok := err.(*errors.StatusError); ok { + } else if statusError, ok := err.(*k8sError.StatusError); ok { if statusError != nil { errReason := statusError.ErrStatus.Reason var errMsg string @@ -850,16 +849,16 @@ func (impl *ClusterServiceImpl) ValidateKubeconfig(kubeConfig string) (map[strin err := json.Unmarshal([]byte(kubeConfig), &kubeConfigDataMap) if err != nil { impl.logger.Errorw("error in unmarshalling kubeConfig") - return nil, errors1.New("invalid kubeConfig found , " + err.Error()) + return nil, customErr.New("invalid kubeConfig found , " + err.Error()) } if kubeConfigDataMap["apiVersion"] == nil { impl.logger.Errorw("api version missing from kubeConfig") - return nil, errors1.New("api version missing from kubeConfig") + return nil, customErr.New("api version missing from kubeConfig") } if kubeConfigDataMap["kind"] == nil { impl.logger.Errorw("kind missing from kubeConfig") - return nil, errors1.New("kind missing from kubeConfig") + return nil, customErr.New("kind missing from kubeConfig") } gvk.Version = kubeConfigDataMap["apiVersion"].(string) @@ -909,7 +908,7 @@ func (impl *ClusterServiceImpl) ValidateKubeconfig(kubeConfig string) (map[strin clusterBeanObject.ErrorInConnecting = "cluster name missing from kubeconfig" } - if clusterBeanObject.ClusterName == clusterBean.DEFAULT_CLUSTER { + if clusterBeanObject.ClusterName == clusterBean.DefaultCluster { clusterBeanObject.ErrorInConnecting = "default_cluster is reserved by the system and cannot be updated" } @@ -1000,7 +999,7 @@ func (impl *ClusterServiceImpl) ValidateKubeconfig(kubeConfig string) (map[strin if len(ValidateObjects) == 0 { impl.logger.Errorw("No valid cluster object provided in kubeconfig for context", "context", kubeConfig) - return nil, errors1.New("No valid cluster object provided in kubeconfig for context") + return nil, customErr.New("No valid cluster object provided in kubeconfig for context") } else { return ValidateObjects, nil } @@ -1014,7 +1013,7 @@ func (impl *ClusterServiceImpl) GetAndUpdateConnectionStatusForOneCluster(k8sCli if err != nil { if _, ok := err.(*url.Error); ok { err = fmt.Errorf("Incorrect server url : %v", err) - } else if statusError, ok := err.(*errors.StatusError); ok { + } else if statusError, ok := err.(*k8sError.StatusError); ok { if statusError != nil { errReason := statusError.ErrStatus.Reason var errMsg string diff --git a/pkg/cluster/bean/bean.go b/pkg/cluster/bean/bean.go index 6548613adb..a2438a2b44 100644 --- a/pkg/cluster/bean/bean.go +++ b/pkg/cluster/bean/bean.go @@ -7,7 +7,7 @@ import ( const ( DefaultClusterId = 1 - DEFAULT_CLUSTER = "default_cluster" + DefaultCluster = "default_cluster" ) type PrometheusAuth struct { @@ -81,3 +81,8 @@ type DefaultClusterComponent struct { EnvName string `json:"envName"` Status string `json:"status"` } + +const ( + DefaultNamespace = "default" + SecretFieldUpdatedOn = "updated_on" +) diff --git a/pkg/cluster/clusterUtil.go b/pkg/cluster/clusterUtil.go index 102519ff19..48ed681400 100644 --- a/pkg/cluster/clusterUtil.go +++ b/pkg/cluster/clusterUtil.go @@ -3,9 +3,9 @@ package cluster import "fmt" const ( - SECRET_NAME = "cluster-event" + SecretName = "cluster-event" ) func ParseSecretNameForKubelinkInformer(clusterId int) string { - return fmt.Sprintf("%s-%d", SECRET_NAME, clusterId) + return fmt.Sprintf("%s-%d", SecretName, clusterId) } diff --git a/pkg/delete/DeleteService.go b/pkg/delete/DeleteService.go index e9a53b3922..278d4182d0 100644 --- a/pkg/delete/DeleteService.go +++ b/pkg/delete/DeleteService.go @@ -108,7 +108,7 @@ func (impl DeleteServiceImpl) DeleteClusterSecret(deleteRequest *bean2.ClusterBe return nil } secretName := cluster.ParseSecretNameForKubelinkInformer(deleteRequest.Id) - err = impl.K8sUtil.DeleteSecret(cluster.DEFAULT_NAMESPACE, secretName, k8sClient) + err = impl.K8sUtil.DeleteSecret(bean2.DefaultNamespace, secretName, k8sClient) return err } diff --git a/pkg/deployment/deployedApp/DeployedAppService.go b/pkg/deployment/deployedApp/DeployedAppService.go index f8c65d2373..83107b22fa 100644 --- a/pkg/deployment/deployedApp/DeployedAppService.go +++ b/pkg/deployment/deployedApp/DeployedAppService.go @@ -46,7 +46,7 @@ type DeployedAppService interface { type DeployedAppServiceImpl struct { logger *zap.SugaredLogger k8sCommonService k8s.K8sCommonService - cdTriggerService devtronApps.TriggerService + cdHandlerService devtronApps.HandlerService envRepository repository.EnvironmentRepository pipelineRepository pipelineConfig.PipelineRepository cdWorkflowRepository pipelineConfig.CdWorkflowRepository @@ -54,14 +54,14 @@ type DeployedAppServiceImpl struct { func NewDeployedAppServiceImpl(logger *zap.SugaredLogger, k8sCommonService k8s.K8sCommonService, - cdTriggerService devtronApps.TriggerService, + cdHandlerService devtronApps.HandlerService, envRepository repository.EnvironmentRepository, pipelineRepository pipelineConfig.PipelineRepository, cdWorkflowRepository pipelineConfig.CdWorkflowRepository) *DeployedAppServiceImpl { return &DeployedAppServiceImpl{ logger: logger, k8sCommonService: k8sCommonService, - cdTriggerService: cdTriggerService, + cdHandlerService: cdHandlerService, envRepository: envRepository, pipelineRepository: pipelineRepository, cdWorkflowRepository: cdWorkflowRepository, @@ -127,7 +127,7 @@ func (impl *DeployedAppServiceImpl) stopStartApp(ctx context.Context, stopReques Context: ctx, ReferenceId: stopRequest.ReferenceId, } - id, _, _, err := impl.cdTriggerService.ManualCdTrigger(triggerContext, overrideRequest) + id, _, _, err := impl.cdHandlerService.ManualCdTrigger(triggerContext, overrideRequest) if err != nil { impl.logger.Errorw("error in stopping app", "err", err, "appId", stopRequest.AppId, "envId", stopRequest.EnvironmentId) return 0, err diff --git a/pkg/deployment/trigger/devtronApps/HandlerService.go b/pkg/deployment/trigger/devtronApps/HandlerService.go new file mode 100644 index 0000000000..c92a009134 --- /dev/null +++ b/pkg/deployment/trigger/devtronApps/HandlerService.go @@ -0,0 +1,345 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package devtronApps + +import ( + "bufio" + "context" + pubsub "github.com/devtron-labs/common-lib/pubsub-lib" + util5 "github.com/devtron-labs/common-lib/utils/k8s" + bean3 "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/api/helm-app/gRPC" + client2 "github.com/devtron-labs/devtron/api/helm-app/service" + "github.com/devtron-labs/devtron/client/argocdServer" + client "github.com/devtron-labs/devtron/client/events" + repository3 "github.com/devtron-labs/devtron/internal/sql/repository" + appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" + "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" + repository4 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/app" + bean4 "github.com/devtron-labs/devtron/pkg/app/bean" + "github.com/devtron-labs/devtron/pkg/app/status" + "github.com/devtron-labs/devtron/pkg/attributes" + "github.com/devtron-labs/devtron/pkg/auth/user" + "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/read" + pipeline2 "github.com/devtron-labs/devtron/pkg/build/pipeline" + chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" + "github.com/devtron-labs/devtron/pkg/cluster" + repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" + repository5 "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/deployment/common" + bean9 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" + "github.com/devtron-labs/devtron/pkg/deployment/manifest" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/publish" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/userDeploymentRequest/service" + "github.com/devtron-labs/devtron/pkg/eventProcessor/out" + "github.com/devtron-labs/devtron/pkg/executor" + "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" + "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/pipeline/history" + "github.com/devtron-labs/devtron/pkg/pipeline/repository" + "github.com/devtron-labs/devtron/pkg/pipeline/types" + "github.com/devtron-labs/devtron/pkg/plugin" + security2 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning" + read2 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/read" + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/devtron-labs/devtron/pkg/variables" + "github.com/devtron-labs/devtron/pkg/workflow/cd" + globalUtil "github.com/devtron-labs/devtron/util" + util2 "github.com/devtron-labs/devtron/util/event" + "github.com/devtron-labs/devtron/util/rbac" + "go.uber.org/zap" + "os" + "time" +) + +/* +files in this package are - +HandlerService.go - containing If and impl with common used code +HandlerService_ent.go - containing ent If and impl with common used code +deployStageHandlerCode.go - code related to deploy stage trigger +deployStageHandlerCode_ent.go - ent code related to deploy stage trigger +preStageHandlerCode.go - code related to pre stage trigger +preStageHandlerCode_ent.go - ent code related to pre stage trigger +postStageHandlerCode.go - code related to post stage trigger +postStageHandlerCode_ent.go - ent code related to post stage trigger +prePostWfAndLogsCode.go - code containing pre/post wf handling(abort) and logs related code +*/ + +type HandlerService interface { + TriggerPostStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) + TriggerPreStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) + + TriggerAutoCDOnPreStageSuccess(triggerContext bean.TriggerContext, cdPipelineId, ciArtifactId, workflowId int) error + + TriggerStageForBulk(triggerRequest bean.TriggerRequest) error + + ManualCdTrigger(triggerContext bean.TriggerContext, overrideRequest *bean3.ValuesOverrideRequest) (int, string, *bean4.ManifestPushTemplate, error) + TriggerAutomaticDeployment(request bean.TriggerRequest) error + + TriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) + + CancelStage(workflowRunnerId int, forceAbort bool, userId int32) (int, error) + DownloadCdWorkflowArtifacts(buildId int) (*os.File, error) + GetRunningWorkflowLogs(environmentId int, pipelineId int, workflowId int) (*bufio.Reader, func() error, error) +} + +type HandlerServiceImpl struct { + logger *zap.SugaredLogger + cdWorkflowCommonService cd.CdWorkflowCommonService + gitOpsManifestPushService publish.GitOpsPushService + gitOpsConfigReadService config.GitOpsConfigReadService + argoK8sClient argocdServer.ArgoK8sClient + ACDConfig *argocdServer.ACDConfig + argoClientWrapperService argocdServer.ArgoClientWrapperService + pipelineStatusTimelineService status.PipelineStatusTimelineService + chartTemplateService util.ChartTemplateService + eventFactory client.EventFactory + eventClient client.EventClient + globalEnvVariables *globalUtil.GlobalEnvVariables + workflowEventPublishService out.WorkflowEventPublishService + manifestCreationService manifest.ManifestCreationService + deployedConfigurationHistoryService history.DeployedConfigurationHistoryService + pipelineStageService pipeline.PipelineStageService + globalPluginService plugin.GlobalPluginService + customTagService pipeline.CustomTagService + pluginInputVariableParser pipeline.PluginInputVariableParser + prePostCdScriptHistoryService history.PrePostCdScriptHistoryService + scopedVariableManager variables.ScopedVariableCMCSManager + imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService + userService user.UserService + config *types.CdConfig + helmAppService client2.HelmAppService + imageScanService security2.ImageScanService + enforcerUtil rbac.EnforcerUtil + userDeploymentRequestService service.UserDeploymentRequestService + helmAppClient gRPC.HelmAppClient //TODO refactoring: use helm app service instead + appRepository appRepository.AppRepository + ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository + imageScanHistoryReadService read2.ImageScanHistoryReadService + imageScanDeployInfoService security2.ImageScanDeployInfoService + imageScanDeployInfoReadService read2.ImageScanDeployInfoReadService + pipelineRepository pipelineConfig.PipelineRepository + pipelineOverrideRepository chartConfig.PipelineOverrideRepository + manifestPushConfigRepository repository.ManifestPushConfigRepository + chartRepository chartRepoRepository.ChartRepository + envRepository repository2.EnvironmentRepository + cdWorkflowRepository pipelineConfig.CdWorkflowRepository + ciWorkflowRepository pipelineConfig.CiWorkflowRepository + ciArtifactRepository repository3.CiArtifactRepository + ciTemplateService pipeline2.CiTemplateReadService + gitMaterialReadService read.GitMaterialReadService + appLabelRepository pipelineConfig.AppLabelRepository + ciPipelineRepository pipelineConfig.CiPipelineRepository + appWorkflowRepository appWorkflow.AppWorkflowRepository + dockerArtifactStoreRepository repository4.DockerArtifactStoreRepository + K8sUtil *util5.K8sServiceImpl + transactionUtilImpl *sql.TransactionUtilImpl + deploymentConfigService common.DeploymentConfigService + ciCdPipelineOrchestrator pipeline.CiCdPipelineOrchestrator + gitOperationService git.GitOperationService + attributeService attributes.AttributesService + clusterRepository repository5.ClusterRepository + cdWorkflowRunnerService cd.CdWorkflowRunnerService + clusterService cluster.ClusterService + ciLogService pipeline.CiLogService + workflowService executor.WorkflowService + blobConfigStorageService pipeline.BlobStorageConfigService +} + +func NewHandlerServiceImpl(logger *zap.SugaredLogger, + cdWorkflowCommonService cd.CdWorkflowCommonService, + gitOpsManifestPushService publish.GitOpsPushService, + gitOpsConfigReadService config.GitOpsConfigReadService, + argoK8sClient argocdServer.ArgoK8sClient, + ACDConfig *argocdServer.ACDConfig, + argoClientWrapperService argocdServer.ArgoClientWrapperService, + pipelineStatusTimelineService status.PipelineStatusTimelineService, + chartTemplateService util.ChartTemplateService, + workflowEventPublishService out.WorkflowEventPublishService, + manifestCreationService manifest.ManifestCreationService, + deployedConfigurationHistoryService history.DeployedConfigurationHistoryService, + pipelineStageService pipeline.PipelineStageService, + globalPluginService plugin.GlobalPluginService, + customTagService pipeline.CustomTagService, + pluginInputVariableParser pipeline.PluginInputVariableParser, + prePostCdScriptHistoryService history.PrePostCdScriptHistoryService, + scopedVariableManager variables.ScopedVariableCMCSManager, + imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService, + userService user.UserService, + helmAppService client2.HelmAppService, + enforcerUtil rbac.EnforcerUtil, + userDeploymentRequestService service.UserDeploymentRequestService, + helmAppClient gRPC.HelmAppClient, + eventFactory client.EventFactory, + eventClient client.EventClient, + envVariables *globalUtil.EnvironmentVariables, + appRepository appRepository.AppRepository, + ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, + imageScanHistoryReadService read2.ImageScanHistoryReadService, + imageScanDeployInfoReadService read2.ImageScanDeployInfoReadService, + imageScanDeployInfoService security2.ImageScanDeployInfoService, + pipelineRepository pipelineConfig.PipelineRepository, + pipelineOverrideRepository chartConfig.PipelineOverrideRepository, + manifestPushConfigRepository repository.ManifestPushConfigRepository, + chartRepository chartRepoRepository.ChartRepository, + envRepository repository2.EnvironmentRepository, + cdWorkflowRepository pipelineConfig.CdWorkflowRepository, + ciWorkflowRepository pipelineConfig.CiWorkflowRepository, + ciArtifactRepository repository3.CiArtifactRepository, + ciTemplateService pipeline2.CiTemplateReadService, + gitMaterialReadService read.GitMaterialReadService, + appLabelRepository pipelineConfig.AppLabelRepository, + ciPipelineRepository pipelineConfig.CiPipelineRepository, + appWorkflowRepository appWorkflow.AppWorkflowRepository, + dockerArtifactStoreRepository repository4.DockerArtifactStoreRepository, + imageScanService security2.ImageScanService, + K8sUtil *util5.K8sServiceImpl, + transactionUtilImpl *sql.TransactionUtilImpl, + deploymentConfigService common.DeploymentConfigService, + ciCdPipelineOrchestrator pipeline.CiCdPipelineOrchestrator, + gitOperationService git.GitOperationService, + attributeService attributes.AttributesService, + clusterRepository repository5.ClusterRepository, + cdWorkflowRunnerService cd.CdWorkflowRunnerService, + clusterService cluster.ClusterService, + ciLogService pipeline.CiLogService, + workflowService executor.WorkflowService, + blobConfigStorageService pipeline.BlobStorageConfigService, +) (*HandlerServiceImpl, error) { + impl := &HandlerServiceImpl{ + logger: logger, + cdWorkflowCommonService: cdWorkflowCommonService, + gitOpsManifestPushService: gitOpsManifestPushService, + gitOpsConfigReadService: gitOpsConfigReadService, + argoK8sClient: argoK8sClient, + ACDConfig: ACDConfig, + argoClientWrapperService: argoClientWrapperService, + pipelineStatusTimelineService: pipelineStatusTimelineService, + chartTemplateService: chartTemplateService, + workflowEventPublishService: workflowEventPublishService, + manifestCreationService: manifestCreationService, + deployedConfigurationHistoryService: deployedConfigurationHistoryService, + pipelineStageService: pipelineStageService, + globalPluginService: globalPluginService, + customTagService: customTagService, + pluginInputVariableParser: pluginInputVariableParser, + prePostCdScriptHistoryService: prePostCdScriptHistoryService, + scopedVariableManager: scopedVariableManager, + imageDigestPolicyService: imageDigestPolicyService, + userService: userService, + helmAppService: helmAppService, + enforcerUtil: enforcerUtil, + eventFactory: eventFactory, + eventClient: eventClient, + + globalEnvVariables: envVariables.GlobalEnvVariables, + userDeploymentRequestService: userDeploymentRequestService, + helmAppClient: helmAppClient, + appRepository: appRepository, + ciPipelineMaterialRepository: ciPipelineMaterialRepository, + imageScanHistoryReadService: imageScanHistoryReadService, + imageScanDeployInfoReadService: imageScanDeployInfoReadService, + imageScanDeployInfoService: imageScanDeployInfoService, + pipelineRepository: pipelineRepository, + pipelineOverrideRepository: pipelineOverrideRepository, + manifestPushConfigRepository: manifestPushConfigRepository, + chartRepository: chartRepository, + envRepository: envRepository, + cdWorkflowRepository: cdWorkflowRepository, + ciWorkflowRepository: ciWorkflowRepository, + ciArtifactRepository: ciArtifactRepository, + ciTemplateService: ciTemplateService, + gitMaterialReadService: gitMaterialReadService, + appLabelRepository: appLabelRepository, + ciPipelineRepository: ciPipelineRepository, + appWorkflowRepository: appWorkflowRepository, + dockerArtifactStoreRepository: dockerArtifactStoreRepository, + + imageScanService: imageScanService, + K8sUtil: K8sUtil, + + transactionUtilImpl: transactionUtilImpl, + + deploymentConfigService: deploymentConfigService, + ciCdPipelineOrchestrator: ciCdPipelineOrchestrator, + gitOperationService: gitOperationService, + attributeService: attributeService, + cdWorkflowRunnerService: cdWorkflowRunnerService, + + clusterRepository: clusterRepository, + clusterService: clusterService, + ciLogService: ciLogService, + workflowService: workflowService, + blobConfigStorageService: blobConfigStorageService, + } + config, err := types.GetCdConfig() + if err != nil { + return nil, err + } + impl.config = config + return impl, nil +} + +func (impl *HandlerServiceImpl) writeCDTriggerEvent(overrideRequest *bean3.ValuesOverrideRequest, artifact *repository3.CiArtifact, releaseId, pipelineOverrideId, wfrId int) { + + event, err := impl.eventFactory.Build(util2.Trigger, &overrideRequest.PipelineId, overrideRequest.AppId, &overrideRequest.EnvId, util2.CD) + if err != nil { + impl.logger.Errorw("error in building cd trigger event", "cdPipelineId", overrideRequest.PipelineId, "err", err) + } + impl.logger.Debugw("event WriteCDTriggerEvent", "event", event) + wfr := impl.getEnrichedWorkflowRunner(overrideRequest, artifact, wfrId) + event = impl.eventFactory.BuildExtraCDData(event, wfr, pipelineOverrideId, bean3.CD_WORKFLOW_TYPE_DEPLOY) + _, evtErr := impl.eventClient.WriteNotificationEvent(event) + if evtErr != nil { + impl.logger.Errorw("CD trigger event not sent", "error", evtErr) + } + deploymentEvent := app.DeploymentEvent{ + ApplicationId: overrideRequest.AppId, + EnvironmentId: overrideRequest.EnvId, // check for production Environment + ReleaseId: releaseId, + PipelineOverrideId: pipelineOverrideId, + TriggerTime: time.Now(), + CiArtifactId: overrideRequest.CiArtifactId, + } + + ciPipelineMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(artifact.PipelineId) + if err != nil { + impl.logger.Errorw("error in ") + } + materialInfoMap, mErr := artifact.ParseMaterialInfo() + if mErr != nil { + impl.logger.Errorw("material info map error", mErr) + return + } + for _, ciPipelineMaterial := range ciPipelineMaterials { + hash := materialInfoMap[ciPipelineMaterial.GitMaterial.Url] + pipelineMaterialInfo := &app.PipelineMaterialInfo{PipelineMaterialId: ciPipelineMaterial.Id, CommitHash: hash} + deploymentEvent.PipelineMaterials = append(deploymentEvent.PipelineMaterials, pipelineMaterialInfo) + } + impl.logger.Infow("triggering deployment event", "event", deploymentEvent) + err = impl.eventClient.WriteNatsEvent(pubsub.CD_SUCCESS, deploymentEvent) + if err != nil { + impl.logger.Errorw("error in writing cd trigger event", "err", err) + } +} diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go similarity index 77% rename from pkg/deployment/trigger/devtronApps/TriggerService.go rename to pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go index 7a5d135045..c0d8a204a1 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService.go +++ b/pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go @@ -20,74 +20,38 @@ import ( "context" "errors" "fmt" - pubsub "github.com/devtron-labs/common-lib/pubsub-lib" - util5 "github.com/devtron-labs/common-lib/utils/k8s" bean3 "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/api/bean/gitOps" bean6 "github.com/devtron-labs/devtron/api/helm-app/bean" "github.com/devtron-labs/devtron/api/helm-app/gRPC" - client2 "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/client/argocdServer" bean7 "github.com/devtron-labs/devtron/client/argocdServer/bean" - client "github.com/devtron-labs/devtron/client/events" - gitSensorClient "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/middleware" "github.com/devtron-labs/devtron/internal/sql/models" repository3 "github.com/devtron-labs/devtron/internal/sql/repository" - appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" - "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" - "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" - repository4 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" bean4 "github.com/devtron-labs/devtron/pkg/app/bean" - "github.com/devtron-labs/devtron/pkg/app/status" statusBean "github.com/devtron-labs/devtron/pkg/app/status/bean" - "github.com/devtron-labs/devtron/pkg/attributes" - "github.com/devtron-labs/devtron/pkg/auth/user" bean2 "github.com/devtron-labs/devtron/pkg/bean" - "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/read" - pipeline2 "github.com/devtron-labs/devtron/pkg/build/pipeline" - chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" - repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" - repository5 "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/deployment/common" bean9 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" - "github.com/devtron-labs/devtron/pkg/deployment/manifest" bean10 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/bean" bean5 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" - "github.com/devtron-labs/devtron/pkg/deployment/manifest/publish" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/adapter" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/helper" - "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/userDeploymentRequest/service" clientErrors "github.com/devtron-labs/devtron/pkg/errors" - "github.com/devtron-labs/devtron/pkg/eventProcessor/out" - "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" k8s2 "github.com/devtron-labs/devtron/pkg/k8s" - "github.com/devtron-labs/devtron/pkg/pipeline" bean8 "github.com/devtron-labs/devtron/pkg/pipeline/bean" - "github.com/devtron-labs/devtron/pkg/pipeline/history" "github.com/devtron-labs/devtron/pkg/pipeline/repository" - "github.com/devtron-labs/devtron/pkg/pipeline/types" - "github.com/devtron-labs/devtron/pkg/plugin" - security2 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning" - read2 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/read" repository6 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/repository" "github.com/devtron-labs/devtron/pkg/sql" - "github.com/devtron-labs/devtron/pkg/variables" - "github.com/devtron-labs/devtron/pkg/workflow/cd" - globalUtil "github.com/devtron-labs/devtron/util" - util2 "github.com/devtron-labs/devtron/util/event" - "github.com/devtron-labs/devtron/util/rbac" "github.com/go-pg/pg" "go.opentelemetry.io/otel" - "go.uber.org/zap" "golang.org/x/exp/slices" "google.golang.org/grpc/codes" status2 "google.golang.org/grpc/status" @@ -100,214 +64,7 @@ import ( "time" ) -type TriggerService interface { - TriggerPostStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) - TriggerPreStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) - - TriggerAutoCDOnPreStageSuccess(triggerContext bean.TriggerContext, cdPipelineId, ciArtifactId, workflowId int) error - - TriggerStageForBulk(triggerRequest bean.TriggerRequest) error - - ManualCdTrigger(triggerContext bean.TriggerContext, overrideRequest *bean3.ValuesOverrideRequest) (int, string, *bean4.ManifestPushTemplate, error) - TriggerAutomaticDeployment(request bean.TriggerRequest) error - - TriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) -} - -type TriggerServiceImpl struct { - logger *zap.SugaredLogger - cdWorkflowCommonService cd.CdWorkflowCommonService - gitOpsManifestPushService publish.GitOpsPushService - gitOpsConfigReadService config.GitOpsConfigReadService - argoK8sClient argocdServer.ArgoK8sClient - ACDConfig *argocdServer.ACDConfig - argoClientWrapperService argocdServer.ArgoClientWrapperService - pipelineStatusTimelineService status.PipelineStatusTimelineService - chartTemplateService util.ChartTemplateService - eventFactory client.EventFactory - eventClient client.EventClient - globalEnvVariables *globalUtil.GlobalEnvVariables - workflowEventPublishService out.WorkflowEventPublishService - manifestCreationService manifest.ManifestCreationService - deployedConfigurationHistoryService history.DeployedConfigurationHistoryService - pipelineStageService pipeline.PipelineStageService - globalPluginService plugin.GlobalPluginService - customTagService pipeline.CustomTagService - pluginInputVariableParser pipeline.PluginInputVariableParser - prePostCdScriptHistoryService history.PrePostCdScriptHistoryService - scopedVariableManager variables.ScopedVariableCMCSManager - cdWorkflowService pipeline.WorkflowService - imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService - userService user.UserService - gitSensorClient gitSensorClient.Client - config *types.CdConfig - helmAppService client2.HelmAppService - imageScanService security2.ImageScanService - enforcerUtil rbac.EnforcerUtil - userDeploymentRequestService service.UserDeploymentRequestService - helmAppClient gRPC.HelmAppClient //TODO refactoring: use helm app service instead - appRepository appRepository.AppRepository - ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository - imageScanHistoryReadService read2.ImageScanHistoryReadService - imageScanDeployInfoService security2.ImageScanDeployInfoService - imageScanDeployInfoReadService read2.ImageScanDeployInfoReadService - pipelineRepository pipelineConfig.PipelineRepository - pipelineOverrideRepository chartConfig.PipelineOverrideRepository - manifestPushConfigRepository repository.ManifestPushConfigRepository - chartRepository chartRepoRepository.ChartRepository - envRepository repository2.EnvironmentRepository - cdWorkflowRepository pipelineConfig.CdWorkflowRepository - ciWorkflowRepository pipelineConfig.CiWorkflowRepository - ciArtifactRepository repository3.CiArtifactRepository - ciTemplateService pipeline2.CiTemplateReadService - gitMaterialReadService read.GitMaterialReadService - appLabelRepository pipelineConfig.AppLabelRepository - ciPipelineRepository pipelineConfig.CiPipelineRepository - appWorkflowRepository appWorkflow.AppWorkflowRepository - dockerArtifactStoreRepository repository4.DockerArtifactStoreRepository - K8sUtil *util5.K8sServiceImpl - transactionUtilImpl *sql.TransactionUtilImpl - deploymentConfigService common.DeploymentConfigService - deploymentServiceTypeConfig *globalUtil.DeploymentServiceTypeConfig - ciCdPipelineOrchestrator pipeline.CiCdPipelineOrchestrator - gitOperationService git.GitOperationService - attributeService attributes.AttributesService - clusterRepository repository5.ClusterRepository - cdWorkflowRunnerService cd.CdWorkflowRunnerService -} - -func NewTriggerServiceImpl(logger *zap.SugaredLogger, - cdWorkflowCommonService cd.CdWorkflowCommonService, - gitOpsManifestPushService publish.GitOpsPushService, - gitOpsConfigReadService config.GitOpsConfigReadService, - argoK8sClient argocdServer.ArgoK8sClient, - ACDConfig *argocdServer.ACDConfig, - argoClientWrapperService argocdServer.ArgoClientWrapperService, - pipelineStatusTimelineService status.PipelineStatusTimelineService, - chartTemplateService util.ChartTemplateService, - workflowEventPublishService out.WorkflowEventPublishService, - manifestCreationService manifest.ManifestCreationService, - deployedConfigurationHistoryService history.DeployedConfigurationHistoryService, - pipelineStageService pipeline.PipelineStageService, - globalPluginService plugin.GlobalPluginService, - customTagService pipeline.CustomTagService, - pluginInputVariableParser pipeline.PluginInputVariableParser, - prePostCdScriptHistoryService history.PrePostCdScriptHistoryService, - scopedVariableManager variables.ScopedVariableCMCSManager, - cdWorkflowService pipeline.WorkflowService, - imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService, - userService user.UserService, - gitSensorClient gitSensorClient.Client, - helmAppService client2.HelmAppService, - enforcerUtil rbac.EnforcerUtil, - userDeploymentRequestService service.UserDeploymentRequestService, - helmAppClient gRPC.HelmAppClient, - eventFactory client.EventFactory, - eventClient client.EventClient, - envVariables *globalUtil.EnvironmentVariables, - appRepository appRepository.AppRepository, - ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, - imageScanHistoryReadService read2.ImageScanHistoryReadService, - imageScanDeployInfoReadService read2.ImageScanDeployInfoReadService, - imageScanDeployInfoService security2.ImageScanDeployInfoService, - pipelineRepository pipelineConfig.PipelineRepository, - pipelineOverrideRepository chartConfig.PipelineOverrideRepository, - manifestPushConfigRepository repository.ManifestPushConfigRepository, - chartRepository chartRepoRepository.ChartRepository, - envRepository repository2.EnvironmentRepository, - cdWorkflowRepository pipelineConfig.CdWorkflowRepository, - ciWorkflowRepository pipelineConfig.CiWorkflowRepository, - ciArtifactRepository repository3.CiArtifactRepository, - ciTemplateService pipeline2.CiTemplateReadService, - gitMaterialReadService read.GitMaterialReadService, - appLabelRepository pipelineConfig.AppLabelRepository, - ciPipelineRepository pipelineConfig.CiPipelineRepository, - appWorkflowRepository appWorkflow.AppWorkflowRepository, - dockerArtifactStoreRepository repository4.DockerArtifactStoreRepository, - imageScanService security2.ImageScanService, - K8sUtil *util5.K8sServiceImpl, - transactionUtilImpl *sql.TransactionUtilImpl, - deploymentConfigService common.DeploymentConfigService, - ciCdPipelineOrchestrator pipeline.CiCdPipelineOrchestrator, - gitOperationService git.GitOperationService, - attributeService attributes.AttributesService, - clusterRepository repository5.ClusterRepository, - cdWorkflowRunnerService cd.CdWorkflowRunnerService, -) (*TriggerServiceImpl, error) { - impl := &TriggerServiceImpl{ - logger: logger, - cdWorkflowCommonService: cdWorkflowCommonService, - gitOpsManifestPushService: gitOpsManifestPushService, - gitOpsConfigReadService: gitOpsConfigReadService, - argoK8sClient: argoK8sClient, - ACDConfig: ACDConfig, - argoClientWrapperService: argoClientWrapperService, - pipelineStatusTimelineService: pipelineStatusTimelineService, - chartTemplateService: chartTemplateService, - workflowEventPublishService: workflowEventPublishService, - manifestCreationService: manifestCreationService, - deployedConfigurationHistoryService: deployedConfigurationHistoryService, - pipelineStageService: pipelineStageService, - globalPluginService: globalPluginService, - customTagService: customTagService, - pluginInputVariableParser: pluginInputVariableParser, - prePostCdScriptHistoryService: prePostCdScriptHistoryService, - scopedVariableManager: scopedVariableManager, - cdWorkflowService: cdWorkflowService, - imageDigestPolicyService: imageDigestPolicyService, - userService: userService, - gitSensorClient: gitSensorClient, - helmAppService: helmAppService, - enforcerUtil: enforcerUtil, - eventFactory: eventFactory, - eventClient: eventClient, - - globalEnvVariables: envVariables.GlobalEnvVariables, - userDeploymentRequestService: userDeploymentRequestService, - helmAppClient: helmAppClient, - appRepository: appRepository, - ciPipelineMaterialRepository: ciPipelineMaterialRepository, - imageScanHistoryReadService: imageScanHistoryReadService, - imageScanDeployInfoReadService: imageScanDeployInfoReadService, - imageScanDeployInfoService: imageScanDeployInfoService, - pipelineRepository: pipelineRepository, - pipelineOverrideRepository: pipelineOverrideRepository, - manifestPushConfigRepository: manifestPushConfigRepository, - chartRepository: chartRepository, - envRepository: envRepository, - cdWorkflowRepository: cdWorkflowRepository, - ciWorkflowRepository: ciWorkflowRepository, - ciArtifactRepository: ciArtifactRepository, - ciTemplateService: ciTemplateService, - gitMaterialReadService: gitMaterialReadService, - appLabelRepository: appLabelRepository, - ciPipelineRepository: ciPipelineRepository, - appWorkflowRepository: appWorkflowRepository, - dockerArtifactStoreRepository: dockerArtifactStoreRepository, - - imageScanService: imageScanService, - K8sUtil: K8sUtil, - - transactionUtilImpl: transactionUtilImpl, - - deploymentConfigService: deploymentConfigService, - deploymentServiceTypeConfig: envVariables.DeploymentServiceTypeConfig, - ciCdPipelineOrchestrator: ciCdPipelineOrchestrator, - gitOperationService: gitOperationService, - attributeService: attributeService, - cdWorkflowRunnerService: cdWorkflowRunnerService, - - clusterRepository: clusterRepository, - } - config, err := types.GetCdConfig() - if err != nil { - return nil, err - } - impl.config = config - return impl, nil -} - -func (impl *TriggerServiceImpl) TriggerStageForBulk(triggerRequest bean.TriggerRequest) error { +func (impl *HandlerServiceImpl) TriggerStageForBulk(triggerRequest bean.TriggerRequest) error { preStage, err := impl.pipelineStageService.GetCdStageByCdPipelineIdAndStageType(triggerRequest.Pipeline.Id, repository.PIPELINE_STAGE_TYPE_PRE_CD, false) if err != nil && err != pg.ErrNoRows { @@ -341,8 +98,8 @@ func (impl *TriggerServiceImpl) TriggerStageForBulk(triggerRequest bean.TriggerR } } -func (impl *TriggerServiceImpl) getCdPipelineForManualCdTrigger(ctx context.Context, pipelineId int) (*pipelineConfig.Pipeline, error) { - _, span := otel.Tracer("TriggerService").Start(ctx, "getCdPipelineForManualCdTrigger") +func (impl *HandlerServiceImpl) getCdPipelineForManualCdTrigger(ctx context.Context, pipelineId int) (*pipelineConfig.Pipeline, error) { + _, span := otel.Tracer("HandlerService").Start(ctx, "getCdPipelineForManualCdTrigger") defer span.End() cdPipeline, err := impl.pipelineRepository.FindById(pipelineId) if err != nil { @@ -362,8 +119,8 @@ func (impl *TriggerServiceImpl) getCdPipelineForManualCdTrigger(ctx context.Cont return cdPipeline, nil } -func (impl *TriggerServiceImpl) validateDeploymentTriggerRequest(ctx context.Context, validateDeploymentTriggerObj *bean.ValidateDeploymentTriggerObj) error { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.validateDeploymentTriggerRequest") +func (impl *HandlerServiceImpl) validateDeploymentTriggerRequest(ctx context.Context, validateDeploymentTriggerObj *bean.ValidateDeploymentTriggerObj) error { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.validateDeploymentTriggerRequest") defer span.End() // custom GitOps repo url validation --> Start err := impl.handleCustomGitOpsRepoValidation(validateDeploymentTriggerObj.Runner, validateDeploymentTriggerObj.CdPipeline, validateDeploymentTriggerObj.DeploymentConfig, validateDeploymentTriggerObj.TriggeredBy) @@ -395,7 +152,7 @@ func (impl *TriggerServiceImpl) validateDeploymentTriggerRequest(ctx context.Con } // TODO: write a wrapper to handle auto and manual trigger -func (impl *TriggerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerContext, overrideRequest *bean3.ValuesOverrideRequest) (int, string, *bean4.ManifestPushTemplate, error) { +func (impl *HandlerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerContext, overrideRequest *bean3.ValuesOverrideRequest) (int, string, *bean4.ManifestPushTemplate, error) { triggerContext.TriggerType = bean.Manual // setting triggeredAt variable to have consistent data for various audit log places in db for deployment time @@ -617,7 +374,7 @@ func isNotHibernateRequest(deploymentType models.DeploymentType) bool { } // TODO: write a wrapper to handle auto and manual trigger -func (impl *TriggerServiceImpl) TriggerAutomaticDeployment(request bean.TriggerRequest) error { +func (impl *HandlerServiceImpl) TriggerAutomaticDeployment(request bean.TriggerRequest) error { // in case of manual trigger auth is already applied and for auto triggers there is no need for auth check here triggeredBy := request.TriggeredBy pipeline := request.Pipeline @@ -694,7 +451,7 @@ func (impl *TriggerServiceImpl) TriggerAutomaticDeployment(request bean.TriggerR return nil } -func (impl *TriggerServiceImpl) TriggerCD(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) error { +func (impl *HandlerServiceImpl) TriggerCD(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) error { impl.logger.Debugw("automatic pipeline trigger attempt async", "artifactId", artifact.Id) err := impl.triggerReleaseAsync(ctx, artifact, cdWorkflowId, wfrId, pipeline, envDeploymentConfig, triggeredAt, triggeredBy) if err != nil { @@ -704,7 +461,7 @@ func (impl *TriggerServiceImpl) TriggerCD(ctx context.Context, artifact *reposit return err } -func (impl *TriggerServiceImpl) triggerReleaseAsync(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) error { +func (impl *HandlerServiceImpl) triggerReleaseAsync(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) error { err := impl.validateAndTrigger(ctx, pipeline, envDeploymentConfig, artifact, cdWorkflowId, wfrId, triggeredAt, triggeredBy) if err != nil { impl.logger.Errorw("error in trigger for pipeline", "pipelineId", strconv.Itoa(pipeline.Id)) @@ -713,7 +470,7 @@ func (impl *TriggerServiceImpl) triggerReleaseAsync(ctx context.Context, artifac return err } -func (impl *TriggerServiceImpl) validateAndTrigger(ctx context.Context, p *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time, triggeredBy int32) error { +func (impl *HandlerServiceImpl) validateAndTrigger(ctx context.Context, p *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time, triggeredBy int32) error { //TODO: verify this logic object := impl.enforcerUtil.GetAppRBACNameByAppId(p.AppId) envApp := strings.Split(object, "/") @@ -725,7 +482,7 @@ func (impl *TriggerServiceImpl) validateAndTrigger(ctx context.Context, p *pipel return err } -func (impl *TriggerServiceImpl) releasePipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time, triggeredBy int32) error { +func (impl *HandlerServiceImpl) releasePipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time, triggeredBy int32) error { startTime := time.Now() defer func() { impl.logger.Debugw("auto trigger release process completed", "timeTaken", time.Since(startTime), "cdPipelineId", pipeline.Id, "artifactId", artifact.Id, "wfrId", wfrId) @@ -759,9 +516,9 @@ func (impl *TriggerServiceImpl) releasePipeline(ctx context.Context, pipeline *p return err } -func (impl *TriggerServiceImpl) triggerAsyncRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, +func (impl *HandlerServiceImpl) triggerAsyncRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, userDeploymentRequestId int, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.triggerAsyncRelease") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.triggerAsyncRelease") defer span.End() // build merged values and save PCO history for the release valuesOverrideResponse, err := impl.manifestCreationService.GetValuesOverrideForTrigger(newCtx, overrideRequest, envDeploymentConfig, triggeredAt) @@ -780,8 +537,8 @@ func (impl *TriggerServiceImpl) triggerAsyncRelease(ctx context.Context, overrid return impl.workflowEventPublishService.TriggerAsyncRelease(userDeploymentRequestId, overrideRequest, valuesOverrideResponse, newCtx, deployedBy) } -func (impl *TriggerServiceImpl) handleCDTriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.handleCDTriggerRelease") +func (impl *HandlerServiceImpl) handleCDTriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.handleCDTriggerRelease") defer span.End() // Handling for auto trigger if overrideRequest.UserId == 0 { @@ -829,7 +586,7 @@ func (impl *TriggerServiceImpl) handleCDTriggerRelease(ctx context.Context, over return impl.TriggerRelease(newCtx, overrideRequest, envDeploymentConfig, triggeredAt, deployedBy) } -func (impl *TriggerServiceImpl) auditDeploymentTriggerHistory(cdWfrId int, valuesOverrideResponse *app.ValuesOverrideResponse, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (err error) { +func (impl *HandlerServiceImpl) auditDeploymentTriggerHistory(cdWfrId int, valuesOverrideResponse *app.ValuesOverrideResponse, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (err error) { if valuesOverrideResponse.Pipeline == nil || valuesOverrideResponse.EnvOverride == nil { impl.logger.Warnw("unable to save histories for deployment trigger, invalid valuesOverrideResponse received", "cdWfrId", cdWfrId) return nil @@ -843,9 +600,9 @@ func (impl *TriggerServiceImpl) auditDeploymentTriggerHistory(cdWfrId int, value } // TriggerRelease will trigger Install/Upgrade request for Devtron App releases synchronously -func (impl *TriggerServiceImpl) TriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, +func (impl *HandlerServiceImpl) TriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.TriggerRelease") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.TriggerRelease") defer span.End() triggerEvent, skipRequest, err := impl.buildTriggerEventForOverrideRequest(overrideRequest, triggeredAt) if err != nil { @@ -897,10 +654,10 @@ func (impl *TriggerServiceImpl) TriggerRelease(ctx context.Context, overrideRequ return releaseNo, valuesOverrideResponse.ManifestPushTemplate, nil } -func (impl *TriggerServiceImpl) performGitOps(ctx context.Context, +func (impl *HandlerServiceImpl) performGitOps(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, triggerEvent bean.TriggerEvent) error { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.performGitOps") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.performGitOps") defer span.End() // update workflow runner status, used in app workflow view err := impl.cdWorkflowCommonService.UpdateNonTerminalStatusInRunner(newCtx, overrideRequest.WfrId, overrideRequest.UserId, cdWorkflow.WorkflowInProgress) @@ -927,7 +684,7 @@ func (impl *TriggerServiceImpl) performGitOps(ctx context.Context, return nil } -func (impl *TriggerServiceImpl) buildTriggerEventForOverrideRequest(overrideRequest *bean3.ValuesOverrideRequest, triggeredAt time.Time) (triggerEvent bean.TriggerEvent, skipRequest bool, err error) { +func (impl *HandlerServiceImpl) buildTriggerEventForOverrideRequest(overrideRequest *bean3.ValuesOverrideRequest, triggeredAt time.Time) (triggerEvent bean.TriggerEvent, skipRequest bool, err error) { triggerEvent = helper.NewTriggerEvent(overrideRequest.DeploymentAppType, triggeredAt, overrideRequest.UserId) request := statusBean.NewTimelineGetRequest(). WithCdWfrId(overrideRequest.WfrId). @@ -962,8 +719,8 @@ func (impl *TriggerServiceImpl) buildTriggerEventForOverrideRequest(overrideRequ return triggerEvent, skipRequest, nil } -func (impl *TriggerServiceImpl) triggerPipeline(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, triggerEvent bean.TriggerEvent, ctx context.Context) (releaseNo int, err error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.triggerPipeline") +func (impl *HandlerServiceImpl) triggerPipeline(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, triggerEvent bean.TriggerEvent, ctx context.Context) (releaseNo int, err error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.triggerPipeline") defer span.End() if triggerEvent.PerformChartPush { impl.logger.Debugw("performing chart push operation in triggerPipeline", "cdWfrId", overrideRequest.WfrId) @@ -999,7 +756,7 @@ func (impl *TriggerServiceImpl) triggerPipeline(overrideRequest *bean3.ValuesOve return valuesOverrideResponse.PipelineOverride.PipelineReleaseCounter, nil } -func (impl *TriggerServiceImpl) buildManifestPushTemplate(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string) (*bean4.ManifestPushTemplate, error) { +func (impl *HandlerServiceImpl) buildManifestPushTemplate(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string) (*bean4.ManifestPushTemplate, error) { manifestPushTemplate := &bean4.ManifestPushTemplate{ WorkflowRunnerId: overrideRequest.WfrId, @@ -1046,8 +803,8 @@ func (impl *TriggerServiceImpl) buildManifestPushTemplate(overrideRequest *bean3 return manifestPushTemplate, nil } -func (impl *TriggerServiceImpl) deployApp(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, triggerEvent bean.TriggerEvent) error { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.deployApp") +func (impl *HandlerServiceImpl) deployApp(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, triggerEvent bean.TriggerEvent) error { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.deployApp") defer span.End() var referenceChartByte []byte var err error @@ -1069,8 +826,8 @@ func (impl *TriggerServiceImpl) deployApp(ctx context.Context, overrideRequest * return nil } -func (impl *TriggerServiceImpl) createHelmAppForCdPipeline(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse) (bool, []byte, error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.createHelmAppForCdPipeline") +func (impl *HandlerServiceImpl) createHelmAppForCdPipeline(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse) (bool, []byte, error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.createHelmAppForCdPipeline") defer span.End() pipelineModel := valuesOverrideResponse.Pipeline envOverride := valuesOverrideResponse.EnvOverride @@ -1175,7 +932,7 @@ func (impl *TriggerServiceImpl) createHelmAppForCdPipeline(ctx context.Context, return true, referenceChartByte, nil } -func (impl *TriggerServiceImpl) getHelmHistoryLimitAndChartMetadataForHelmAppCreation(ctx context.Context, +func (impl *HandlerServiceImpl) getHelmHistoryLimitAndChartMetadataForHelmAppCreation(ctx context.Context, valuesOverrideResponse *app.ValuesOverrideResponse) (*chart.Metadata, int32, *gRPC.ReleaseIdentifier, error) { pipelineModel := valuesOverrideResponse.Pipeline envOverride := valuesOverrideResponse.EnvOverride @@ -1224,9 +981,9 @@ func (impl *TriggerServiceImpl) getHelmHistoryLimitAndChartMetadataForHelmAppCre return chartMetaData, helmRevisionHistory, releaseIdentifier, nil } -func (impl *TriggerServiceImpl) deployArgoCdApp(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, +func (impl *HandlerServiceImpl) deployArgoCdApp(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse) error { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.deployArgoCdApp") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.deployArgoCdApp") defer span.End() name, err := impl.createArgoApplicationIfRequired(newCtx, valuesOverrideResponse.EnvOverride, valuesOverrideResponse.Pipeline, valuesOverrideResponse.DeploymentConfig, overrideRequest.UserId) if err != nil { @@ -1270,7 +1027,7 @@ func (impl *TriggerServiceImpl) deployArgoCdApp(ctx context.Context, overrideReq } // update repoUrl, revision and argo app sync mode (auto/manual) if needed -func (impl *TriggerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envOverride *bean10.EnvConfigOverride, deploymentConfig *bean9.DeploymentConfig) (bool, error) { +func (impl *HandlerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envOverride *bean10.EnvConfigOverride, deploymentConfig *bean9.DeploymentConfig) (bool, error) { if !deploymentConfig.IsArgoAppPatchSupported() { impl.logger.Infow("argo app patch not supported", "pipelineId", pipeline.Id, "pipelineName", pipeline.Name) return false, nil @@ -1279,7 +1036,7 @@ func (impl *TriggerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline impl.logger.Errorw("err in syncing ACD, ctx is NULL", "pipelineId", pipeline.Id, "pipelineName", pipeline.Name) return false, nil } - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.updateArgoPipeline") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.updateArgoPipeline") defer span.End() argoAppName := pipeline.DeploymentAppName impl.logger.Infow("received payload, updateArgoPipeline", "appId", pipeline.AppId, "pipelineName", pipeline.Name, "envId", envOverride.TargetEnvironment, "argoAppName", argoAppName) @@ -1335,9 +1092,9 @@ func (impl *TriggerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline } } -func (impl *TriggerServiceImpl) createArgoApplicationIfRequired(ctx context.Context, envConfigOverride *bean10.EnvConfigOverride, +func (impl *HandlerServiceImpl) createArgoApplicationIfRequired(ctx context.Context, envConfigOverride *bean10.EnvConfigOverride, pipeline *pipelineConfig.Pipeline, deploymentConfig *bean9.DeploymentConfig, userId int32) (string, error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.createArgoApplicationIfRequired") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.createArgoApplicationIfRequired") defer span.End() envModel, err := impl.envRepository.FindById(envConfigOverride.TargetEnvironment) if err != nil { @@ -1384,7 +1141,7 @@ func (impl *TriggerServiceImpl) createArgoApplicationIfRequired(ctx context.Cont } } -func (impl *TriggerServiceImpl) updatePipeline(pipeline *pipelineConfig.Pipeline, userId int32) (bool, error) { +func (impl *HandlerServiceImpl) updatePipeline(pipeline *pipelineConfig.Pipeline, userId int32) (bool, error) { err := impl.pipelineRepository.SetDeploymentAppCreatedInPipeline(true, pipeline.Id, userId) if err != nil { impl.logger.Errorw("error on updating cd pipeline for setting deployment app created", "err", err) @@ -1394,9 +1151,9 @@ func (impl *TriggerServiceImpl) updatePipeline(pipeline *pipelineConfig.Pipeline } // helmInstallReleaseWithCustomChart performs helm install with custom chart -func (impl *TriggerServiceImpl) helmInstallReleaseWithCustomChart(ctx context.Context, releaseIdentifier *gRPC.ReleaseIdentifier, +func (impl *HandlerServiceImpl) helmInstallReleaseWithCustomChart(ctx context.Context, releaseIdentifier *gRPC.ReleaseIdentifier, referenceChartByte []byte, valuesYaml, k8sServerVersion string, forceSync bool) (*gRPC.HelmInstallCustomResponse, error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.helmInstallReleaseWithCustomChart") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.helmInstallReleaseWithCustomChart") defer span.End() helmInstallRequest := gRPC.HelmInstallCustomRequest{ ValuesYaml: valuesYaml, @@ -1413,52 +1170,9 @@ func (impl *TriggerServiceImpl) helmInstallReleaseWithCustomChart(ctx context.Co return impl.helmAppClient.InstallReleaseWithCustomChart(newCtx, &helmInstallRequest) } -func (impl *TriggerServiceImpl) writeCDTriggerEvent(overrideRequest *bean3.ValuesOverrideRequest, artifact *repository3.CiArtifact, releaseId, pipelineOverrideId, wfrId int) { - - event, err := impl.eventFactory.Build(util2.Trigger, &overrideRequest.PipelineId, overrideRequest.AppId, &overrideRequest.EnvId, util2.CD) - if err != nil { - impl.logger.Errorw("error in building cd trigger event", "cdPipelineId", overrideRequest.PipelineId, "err", err) - } - impl.logger.Debugw("event WriteCDTriggerEvent", "event", event) - wfr := impl.getEnrichedWorkflowRunner(overrideRequest, artifact, wfrId) - event = impl.eventFactory.BuildExtraCDData(event, wfr, pipelineOverrideId, bean3.CD_WORKFLOW_TYPE_DEPLOY) - _, evtErr := impl.eventClient.WriteNotificationEvent(event) - if evtErr != nil { - impl.logger.Errorw("CD trigger event not sent", "error", evtErr) - } - deploymentEvent := app.DeploymentEvent{ - ApplicationId: overrideRequest.AppId, - EnvironmentId: overrideRequest.EnvId, // check for production Environment - ReleaseId: releaseId, - PipelineOverrideId: pipelineOverrideId, - TriggerTime: time.Now(), - CiArtifactId: overrideRequest.CiArtifactId, - } - - ciPipelineMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(artifact.PipelineId) - if err != nil { - impl.logger.Errorw("error in ") - } - materialInfoMap, mErr := artifact.ParseMaterialInfo() - if mErr != nil { - impl.logger.Errorw("material info map error", mErr) - return - } - for _, ciPipelineMaterial := range ciPipelineMaterials { - hash := materialInfoMap[ciPipelineMaterial.GitMaterial.Url] - pipelineMaterialInfo := &app.PipelineMaterialInfo{PipelineMaterialId: ciPipelineMaterial.Id, CommitHash: hash} - deploymentEvent.PipelineMaterials = append(deploymentEvent.PipelineMaterials, pipelineMaterialInfo) - } - impl.logger.Infow("triggering deployment event", "event", deploymentEvent) - err = impl.eventClient.WriteNatsEvent(pubsub.CD_SUCCESS, deploymentEvent) - if err != nil { - impl.logger.Errorw("error in writing cd trigger event", "err", err) - } -} - -func (impl *TriggerServiceImpl) markImageScanDeployed(ctx context.Context, appId, envId, clusterId int, +func (impl *HandlerServiceImpl) markImageScanDeployed(ctx context.Context, appId, envId, clusterId int, imageDigest string, isScanEnabled bool, image string) error { - _, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.markImageScanDeployed") + _, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.markImageScanDeployed") defer span.End() // TODO KB: send NATS event for self consumption impl.logger.Debugw("mark image scan deployed for devtron app, from cd auto or manual trigger", "imageDigest", imageDigest) @@ -1526,11 +1240,11 @@ func (impl *TriggerServiceImpl) markImageScanDeployed(ctx context.Context, appId return err } -func (impl *TriggerServiceImpl) isDevtronAsyncHelmInstallModeEnabled(forceSync bool) bool { +func (impl *HandlerServiceImpl) isDevtronAsyncHelmInstallModeEnabled(forceSync bool) bool { return impl.globalEnvVariables.EnableAsyncHelmInstallDevtronChart && !forceSync } -func (impl *TriggerServiceImpl) isDevtronAsyncInstallModeEnabled(overrideRequest *bean3.ValuesOverrideRequest) (bool, error) { +func (impl *HandlerServiceImpl) isDevtronAsyncInstallModeEnabled(overrideRequest *bean3.ValuesOverrideRequest) (bool, error) { if util.IsHelmApp(overrideRequest.DeploymentAppType) { return impl.isDevtronAsyncHelmInstallModeEnabled(overrideRequest.ForceSyncDeployment), nil } else if util.IsAcdApp(overrideRequest.DeploymentAppType) { @@ -1540,7 +1254,7 @@ func (impl *TriggerServiceImpl) isDevtronAsyncInstallModeEnabled(overrideRequest return false, nil } -func (impl *TriggerServiceImpl) deleteCorruptedPipelineStage(pipelineStage *repository.PipelineStage, triggeredBy int32) (error, bool) { +func (impl *HandlerServiceImpl) deleteCorruptedPipelineStage(pipelineStage *repository.PipelineStage, triggeredBy int32) (error, bool) { if pipelineStage != nil { stageReq := &bean8.PipelineStageDto{ Id: pipelineStage.Id, @@ -1556,7 +1270,7 @@ func (impl *TriggerServiceImpl) deleteCorruptedPipelineStage(pipelineStage *repo return nil, false } -func (impl *TriggerServiceImpl) handleCustomGitOpsRepoValidation(runner *pipelineConfig.CdWorkflowRunner, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredBy int32) error { +func (impl *HandlerServiceImpl) handleCustomGitOpsRepoValidation(runner *pipelineConfig.CdWorkflowRunner, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredBy int32) error { if !util.IsAcdApp(pipeline.DeploymentAppName) { return nil } @@ -1589,7 +1303,7 @@ func (impl *TriggerServiceImpl) handleCustomGitOpsRepoValidation(runner *pipelin return nil } -func (impl *TriggerServiceImpl) getSanitizedK8sVersion(referenceTemplate string) (string, error) { +func (impl *HandlerServiceImpl) getSanitizedK8sVersion(referenceTemplate string) (string, error) { var sanitizedK8sVersion string //handle specific case for all cronjob charts from cronjob-chart_1-2-0 to cronjob-chart_1-5-0 where semverCompare //comparison func has wrong api version mentioned, so for already installed charts via these charts that comparison @@ -1606,7 +1320,7 @@ func (impl *TriggerServiceImpl) getSanitizedK8sVersion(referenceTemplate string) return sanitizedK8sVersion, nil } -func (impl *TriggerServiceImpl) getReferenceChartByteForHelmTypeApp(envOverride *bean10.EnvConfigOverride, +func (impl *HandlerServiceImpl) getReferenceChartByteForHelmTypeApp(envOverride *bean10.EnvConfigOverride, chartMetaData *chart.Metadata, referenceTemplatePath string, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse) ([]byte, error) { referenceChartByte := envOverride.Chart.ReferenceChart diff --git a/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go b/pkg/deployment/trigger/devtronApps/deployStageHandlerCode_ent.go similarity index 67% rename from pkg/deployment/trigger/devtronApps/TriggerService_ent1.go rename to pkg/deployment/trigger/devtronApps/deployStageHandlerCode_ent.go index f008f72adc..347f980a07 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go +++ b/pkg/deployment/trigger/devtronApps/deployStageHandlerCode_ent.go @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package devtronApps import ( @@ -17,29 +33,29 @@ import ( "helm.sh/helm/v3/pkg/chart" ) -func (impl *TriggerServiceImpl) getEnrichedWorkflowRunner(overrideRequest *bean3.ValuesOverrideRequest, artifact *repository3.CiArtifact, wfrId int) *pipelineConfig.CdWorkflowRunner { +func (impl *HandlerServiceImpl) getEnrichedWorkflowRunner(overrideRequest *bean3.ValuesOverrideRequest, artifact *repository3.CiArtifact, wfrId int) *pipelineConfig.CdWorkflowRunner { return nil } -func (impl *TriggerServiceImpl) postDeployHook(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, referenceChartByte []byte, err error) { +func (impl *HandlerServiceImpl) postDeployHook(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, referenceChartByte []byte, err error) { impl.logger.Debugw("no post deploy hook registered") } -func (impl *TriggerServiceImpl) isDevtronAsyncArgoCdInstallModeEnabledForApp(appId, envId int, forceSync bool) (bool, error) { +func (impl *HandlerServiceImpl) isDevtronAsyncArgoCdInstallModeEnabledForApp(appId, envId int, forceSync bool) (bool, error) { return impl.globalEnvVariables.EnableAsyncArgoCdInstallDevtronChart && !forceSync, nil } -func (impl *TriggerServiceImpl) getClusterGRPCConfig(cluster repository2.Cluster) *gRPC.ClusterConfig { +func (impl *HandlerServiceImpl) getClusterGRPCConfig(cluster repository2.Cluster) *gRPC.ClusterConfig { clusterConfig := helper.ConvertClusterBeanToGrpcConfig(cluster) return clusterConfig } -func (impl *TriggerServiceImpl) overrideReferenceChartByteForHelmTypeApp(valuesOverrideResponse *app.ValuesOverrideResponse, +func (impl *HandlerServiceImpl) overrideReferenceChartByteForHelmTypeApp(valuesOverrideResponse *app.ValuesOverrideResponse, chartMetaData *chart.Metadata, referenceTemplatePath string, referenceChartByte []byte) ([]byte, error) { return referenceChartByte, nil } -func (impl *TriggerServiceImpl) getManifestPushService(storageType string) publish.ManifestPushService { +func (impl *HandlerServiceImpl) getManifestPushService(storageType string) publish.ManifestPushService { var manifestPushService publish.ManifestPushService if storageType == bean2.ManifestStorageGit { manifestPushService = impl.gitOpsManifestPushService @@ -47,25 +63,25 @@ func (impl *TriggerServiceImpl) getManifestPushService(storageType string) publi return manifestPushService } -func (impl *TriggerServiceImpl) preStageHandlingForTriggerStageInBulk(triggerRequest *bean.TriggerRequest) error { +func (impl *HandlerServiceImpl) preStageHandlingForTriggerStageInBulk(triggerRequest *bean.TriggerRequest) error { return nil } -func (impl *TriggerServiceImpl) manifestGenerationFailedTimelineHandling(triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, err error) { +func (impl *HandlerServiceImpl) manifestGenerationFailedTimelineHandling(triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, err error) { } -func (impl *TriggerServiceImpl) getHelmManifestForTriggerRelease(ctx context.Context, triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, +func (impl *HandlerServiceImpl) getHelmManifestForTriggerRelease(ctx context.Context, triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string) ([]byte, error) { return nil, nil } -func (impl *TriggerServiceImpl) buildManifestPushTemplateForNonGitStorageType(overrideRequest *bean3.ValuesOverrideRequest, +func (impl *HandlerServiceImpl) buildManifestPushTemplateForNonGitStorageType(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, err error, manifestPushConfig *repository.ManifestPushConfig, manifestPushTemplate *bean4.ManifestPushTemplate) error { return nil } -func (impl *TriggerServiceImpl) triggerReleaseSuccessHandling(triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, +func (impl *HandlerServiceImpl) triggerReleaseSuccessHandling(triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, helmManifest []byte) error { return nil } diff --git a/pkg/deployment/trigger/devtronApps/feasibilityChecker.go b/pkg/deployment/trigger/devtronApps/feasibilityChecker.go index 43926bb86a..c03968d5de 100644 --- a/pkg/deployment/trigger/devtronApps/feasibilityChecker.go +++ b/pkg/deployment/trigger/devtronApps/feasibilityChecker.go @@ -24,7 +24,7 @@ type FeasibilityManager interface { CheckFeasibility(triggerRequirementRequest *bean.TriggerRequirementRequestDto) error } -func (impl *TriggerServiceImpl) CheckFeasibility(triggerRequirementRequest *bean.TriggerRequirementRequestDto) error { +func (impl *HandlerServiceImpl) CheckFeasibility(triggerRequirementRequest *bean.TriggerRequirementRequestDto) error { // have not implemented right now, will be implemented in future for security vulnerability return nil } diff --git a/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go b/pkg/deployment/trigger/devtronApps/postStageHandlerCode.go similarity index 97% rename from pkg/deployment/trigger/devtronApps/PostStageTriggerService.go rename to pkg/deployment/trigger/devtronApps/postStageHandlerCode.go index 8a11b06a6b..1bf2399ffc 100644 --- a/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/postStageHandlerCode.go @@ -30,7 +30,7 @@ import ( "time" ) -func (impl *TriggerServiceImpl) TriggerPostStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) { +func (impl *HandlerServiceImpl) TriggerPostStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) { request.WorkflowType = bean2.CD_WORKFLOW_TYPE_POST // setting triggeredAt variable to have consistent data for various audit log places in db for deployment time triggeredAt := time.Now() @@ -123,7 +123,7 @@ func (impl *TriggerServiceImpl) TriggerPostStage(request bean.TriggerRequest) (* return nil, err } - _, jobHelmPackagePath, err := impl.cdWorkflowService.SubmitWorkflow(cdStageWorkflowRequest) + _, jobHelmPackagePath, err := impl.workflowService.SubmitWorkflow(cdStageWorkflowRequest) if err != nil { impl.logger.Errorw("error in submitting workflow", "err", err, "workflowId", cdStageWorkflowRequest.WorkflowId, "pipeline", pipeline, "env", env) runner.Status = cdWorkflow.WorkflowFailed @@ -164,7 +164,7 @@ func (impl *TriggerServiceImpl) TriggerPostStage(request bean.TriggerRequest) (* return manifestPushTempate, nil } -func (impl *TriggerServiceImpl) buildWfRequestErrorHandler(runner *pipelineConfig.CdWorkflowRunner, err error, triggeredBy int32) (*bean4.ManifestPushTemplate, error) { +func (impl *HandlerServiceImpl) buildWfRequestErrorHandler(runner *pipelineConfig.CdWorkflowRunner, err error, triggeredBy int32) (*bean4.ManifestPushTemplate, error) { dbErr := impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, err, triggeredBy) if dbErr != nil { impl.logger.Errorw("error while updating current runner status to failed, buildWfRequestErrorHandler", "runner", runner.Id, "err", dbErr, "releaseErr", err) diff --git a/pkg/deployment/trigger/devtronApps/PostStageTriggerService_ent.go b/pkg/deployment/trigger/devtronApps/postStageHandlerCode_ent.go similarity index 92% rename from pkg/deployment/trigger/devtronApps/PostStageTriggerService_ent.go rename to pkg/deployment/trigger/devtronApps/postStageHandlerCode_ent.go index d99b813b1d..197aecd22d 100644 --- a/pkg/deployment/trigger/devtronApps/PostStageTriggerService_ent.go +++ b/pkg/deployment/trigger/devtronApps/postStageHandlerCode_ent.go @@ -26,13 +26,13 @@ import ( "time" ) -func (impl *TriggerServiceImpl) checkFeasibilityForPostStage(pipeline *pipelineConfig.Pipeline, request *bean.TriggerRequest, +func (impl *HandlerServiceImpl) checkFeasibilityForPostStage(pipeline *pipelineConfig.Pipeline, request *bean.TriggerRequest, env *repository.Environment, cdWf *pipelineConfig.CdWorkflow, triggeredBy int32) (interface{}, error) { //here return type is interface as ResourceFilterEvaluationAudit is not present in this version return nil, nil } -func (impl *TriggerServiceImpl) getManifestPushTemplateForPostStage(request bean.TriggerRequest, envDevploymentConfig *bean5.DeploymentConfig, +func (impl *HandlerServiceImpl) getManifestPushTemplateForPostStage(request bean.TriggerRequest, envDevploymentConfig *bean5.DeploymentConfig, jobHelmPackagePath string, cdStageWorkflowRequest *types.WorkflowRequest, cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner, pipeline *pipelineConfig.Pipeline, triggeredBy int32, triggeredAt time.Time) (*bean4.ManifestPushTemplate, error) { return nil, nil diff --git a/pkg/deployment/trigger/devtronApps/prePostWfAndLogsCode.go b/pkg/deployment/trigger/devtronApps/prePostWfAndLogsCode.go new file mode 100644 index 0000000000..d3ff8b5e5f --- /dev/null +++ b/pkg/deployment/trigger/devtronApps/prePostWfAndLogsCode.go @@ -0,0 +1,337 @@ +package devtronApps + +import ( + "bufio" + "errors" + "fmt" + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + blob_storage "github.com/devtron-labs/common-lib/blob-storage" + "github.com/devtron-labs/common-lib/utils/k8s" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + cdWorkflow2 "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" + "github.com/devtron-labs/devtron/internal/util" + bean2 "github.com/devtron-labs/devtron/pkg/bean" + "github.com/devtron-labs/devtron/pkg/cluster/adapter" + bean3 "github.com/devtron-labs/devtron/pkg/cluster/bean" + "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/pipeline/constants" + "github.com/devtron-labs/devtron/pkg/pipeline/types" + util2 "github.com/devtron-labs/devtron/pkg/pipeline/util" + "k8s.io/client-go/rest" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "time" +) + +func (impl *HandlerServiceImpl) CancelStage(workflowRunnerId int, forceAbort bool, userId int32) (int, error) { + workflowRunner, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(workflowRunnerId) + if err != nil { + impl.logger.Errorw("err", "err", err) + return 0, err + } + pipeline, err := impl.pipelineRepository.FindById(workflowRunner.CdWorkflow.PipelineId) + if err != nil { + impl.logger.Errorw("error while fetching cd pipeline", "err", err) + return 0, err + } + + env, err := impl.envRepository.FindById(pipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("could not fetch stage env", "err", err) + return 0, err + } + + var clusterBean bean3.ClusterBean + if env != nil && env.Cluster != nil { + clusterBean = adapter.GetClusterBean(*env.Cluster) + } + clusterConfig := clusterBean.GetClusterConfig() + var isExtCluster bool + if workflowRunner.WorkflowType == types.PRE { + isExtCluster = pipeline.RunPreStageInEnv + } else if workflowRunner.WorkflowType == types.POST { + isExtCluster = pipeline.RunPostStageInEnv + } + var restConfig *rest.Config + if isExtCluster { + restConfig, err = impl.K8sUtil.GetRestConfigByCluster(clusterConfig) + if err != nil { + impl.logger.Errorw("error in getting rest config by cluster id", "err", err) + return 0, err + } + } + // Terminate workflow + cancelWfDtoRequest := &types.CancelWfRequestDto{ + ExecutorType: workflowRunner.ExecutorType, + WorkflowName: workflowRunner.Name, + Namespace: workflowRunner.Namespace, + RestConfig: restConfig, + IsExt: isExtCluster, + Environment: nil, + } + err = impl.workflowService.TerminateWorkflow(cancelWfDtoRequest) + if err != nil && forceAbort { + impl.logger.Errorw("error in terminating workflow, with force abort flag as true", "workflowName", workflowRunner.Name, "err", err) + cancelWfDtoRequest.WorkflowGenerateName = fmt.Sprintf("%d-%s", workflowRunnerId, workflowRunner.Name) + err1 := impl.workflowService.TerminateDanglingWorkflows(cancelWfDtoRequest) + if err1 != nil { + impl.logger.Errorw("error in terminating dangling workflows", "cancelWfDtoRequest", cancelWfDtoRequest, "err", err) + // ignoring error here in case of force abort, confirmed from product + } + } else if err != nil && strings.Contains(err.Error(), "cannot find workflow") { + return 0, &util.ApiError{Code: "200", HttpStatusCode: http.StatusBadRequest, UserMessage: err.Error()} + } else if err != nil { + impl.logger.Error("cannot terminate wf runner", "err", err) + return 0, err + } + if forceAbort { + err = impl.handleForceAbortCaseForCdStage(workflowRunner, forceAbort) + if err != nil { + impl.logger.Errorw("error in handleForceAbortCaseForCdStage", "forceAbortFlag", forceAbort, "workflowRunner", workflowRunner, "err", err) + return 0, err + } + return workflowRunner.Id, nil + } + if len(workflowRunner.ImagePathReservationIds) > 0 { + err := impl.customTagService.DeactivateImagePathReservationByImageIds(workflowRunner.ImagePathReservationIds) + if err != nil { + impl.logger.Errorw("error in deactivating image path reservation ids", "err", err) + return 0, err + } + } + workflowRunner.Status = cdWorkflow2.WorkflowCancel + workflowRunner.UpdatedOn = time.Now() + workflowRunner.UpdatedBy = userId + err = impl.cdWorkflowRunnerService.UpdateCdWorkflowRunnerWithStage(workflowRunner) + if err != nil { + impl.logger.Error("cannot update deleted workflow runner status, but wf deleted", "err", err) + return 0, err + } + return workflowRunner.Id, nil +} + +func (impl *HandlerServiceImpl) updateWorkflowRunnerForForceAbort(workflowRunner *pipelineConfig.CdWorkflowRunner) error { + workflowRunner.Status = cdWorkflow2.WorkflowCancel + workflowRunner.PodStatus = string(bean2.Failed) + workflowRunner.Message = constants.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE + err := impl.cdWorkflowRunnerService.UpdateCdWorkflowRunnerWithStage(workflowRunner) + if err != nil { + impl.logger.Errorw("error in updating workflow status in cd workflow runner in force abort case", "err", err) + return err + } + return nil +} + +func (impl *HandlerServiceImpl) handleForceAbortCaseForCdStage(workflowRunner *pipelineConfig.CdWorkflowRunner, forceAbort bool) error { + isWorkflowInNonTerminalStage := workflowRunner.Status == string(v1alpha1.NodePending) || workflowRunner.Status == string(v1alpha1.NodeRunning) + if !isWorkflowInNonTerminalStage { + if forceAbort { + return impl.updateWorkflowRunnerForForceAbort(workflowRunner) + } else { + return &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "cannot cancel stage, stage not in progress"} + } + } + //this arises when someone deletes the workflow in resource browser and wants to force abort a cd stage(pre/post) + if workflowRunner.Status == string(v1alpha1.NodeRunning) && forceAbort { + return impl.updateWorkflowRunnerForForceAbort(workflowRunner) + } + return nil +} + +func (impl *HandlerServiceImpl) DownloadCdWorkflowArtifacts(buildId int) (*os.File, error) { + wfr, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(buildId) + if err != nil { + impl.logger.Errorw("unable to fetch ciWorkflow", "err", err) + return nil, err + } + useExternalBlobStorage := pipeline.IsExternalBlobStorageEnabled(wfr.IsExternalRun(), impl.config.UseBlobStorageConfigInCdWorkflow) + if !wfr.BlobStorageEnabled { + return nil, errors.New("logs-not-stored-in-repository") + } + + cdConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() + cdConfigCdCacheRegion := impl.config.GetDefaultCdLogsBucketRegion() + + item := strconv.Itoa(wfr.Id) + awsS3BaseConfig := &blob_storage.AwsS3BaseConfig{ + AccessKey: impl.config.BlobStorageS3AccessKey, + Passkey: impl.config.BlobStorageS3SecretKey, + EndpointUrl: impl.config.BlobStorageS3Endpoint, + IsInSecure: impl.config.BlobStorageS3EndpointInsecure, + BucketName: cdConfigLogsBucket, + Region: cdConfigCdCacheRegion, + VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, + } + azureBlobBaseConfig := &blob_storage.AzureBlobBaseConfig{ + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, + AccountKey: impl.config.AzureAccountKey, + AccountName: impl.config.AzureAccountName, + BlobContainerName: impl.config.AzureBlobContainerCiLog, + } + gcpBlobBaseConfig := &blob_storage.GcpBlobBaseConfig{ + BucketName: cdConfigLogsBucket, + CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, + } + cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() + key := fmt.Sprintf(cdArtifactLocationFormat, wfr.CdWorkflow.Id, wfr.Id) + if len(wfr.CdArtifactLocation) != 0 && util2.IsValidUrlSubPath(wfr.CdArtifactLocation) { + key = wfr.CdArtifactLocation + } else if util2.IsValidUrlSubPath(key) { + impl.cdWorkflowRepository.MigrateCdArtifactLocation(wfr.Id, key) + } + baseLogLocationPathConfig := impl.config.BaseLogLocationPath + blobStorageService := blob_storage.NewBlobStorageServiceImpl(nil) + destinationKey := filepath.Clean(filepath.Join(baseLogLocationPathConfig, item)) + request := &blob_storage.BlobStorageRequest{ + StorageType: impl.config.CloudProvider, + SourceKey: key, + DestinationKey: destinationKey, + AzureBlobBaseConfig: azureBlobBaseConfig, + AwsS3BaseConfig: awsS3BaseConfig, + GcpBlobBaseConfig: gcpBlobBaseConfig, + } + if useExternalBlobStorage { + clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(wfr.CdWorkflow.Pipeline.Environment.ClusterId) + if err != nil { + impl.logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig", "err", err, "clusterId", wfr.CdWorkflow.Pipeline.Environment.ClusterId) + return nil, err + } + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, wfr.Namespace) + if err != nil { + impl.logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, err + } + request = pipeline.UpdateRequestWithExtClusterCmAndSecret(request, cmConfig, secretConfig) + } + _, numBytes, err := blobStorageService.Get(request) + if err != nil { + impl.logger.Errorw("error occurred while downloading file", "request", request, "error", err) + return nil, errors.New("failed to download resource") + } + + file, err := os.Open(destinationKey) + if err != nil { + impl.logger.Errorw("unable to open file", "file", item, "err", err) + return nil, errors.New("unable to open file") + } + + impl.logger.Infow("Downloaded ", "name", file.Name(), "bytes", numBytes) + return file, nil +} + +func (impl *HandlerServiceImpl) GetRunningWorkflowLogs(environmentId int, pipelineId int, wfrId int) (*bufio.Reader, func() error, error) { + cdWorkflow, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(wfrId) + if err != nil { + impl.logger.Errorw("error on fetch wf runner", "err", err) + return nil, nil, err + } + + env, err := impl.envRepository.FindById(environmentId) + if err != nil { + impl.logger.Errorw("could not fetch stage env", "err", err) + return nil, nil, err + } + + pipeline, err := impl.pipelineRepository.FindById(cdWorkflow.CdWorkflow.PipelineId) + if err != nil { + impl.logger.Errorw("error while fetching cd pipeline", "err", err) + return nil, nil, err + } + var clusterBean bean3.ClusterBean + if env != nil && env.Cluster != nil { + clusterBean = adapter.GetClusterBean(*env.Cluster) + } + clusterConfig := clusterBean.GetClusterConfig() + var isExtCluster bool + if cdWorkflow.WorkflowType == types.PRE { + isExtCluster = pipeline.RunPreStageInEnv + } else if cdWorkflow.WorkflowType == types.POST { + isExtCluster = pipeline.RunPostStageInEnv + } + return impl.getWorkflowLogs(pipelineId, cdWorkflow, clusterConfig, isExtCluster) +} + +func (impl *HandlerServiceImpl) getWorkflowLogs(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, runStageInEnv bool) (*bufio.Reader, func() error, error) { + cdLogRequest := types.BuildLogRequest{ + PodName: cdWorkflow.PodName, + Namespace: cdWorkflow.Namespace, + } + + logStream, cleanUp, err := impl.ciLogService.FetchRunningWorkflowLogs(cdLogRequest, clusterConfig, runStageInEnv) + if logStream == nil || err != nil { + if !cdWorkflow.BlobStorageEnabled { + return nil, nil, errors.New("logs-not-stored-in-repository") + } else if string(v1alpha1.NodeSucceeded) == cdWorkflow.Status || string(v1alpha1.NodeError) == cdWorkflow.Status || string(v1alpha1.NodeFailed) == cdWorkflow.Status || cdWorkflow.Status == cdWorkflow2.WorkflowCancel { + impl.logger.Debugw("pod is not live", "podName", cdWorkflow.PodName, "err", err) + return impl.getLogsFromRepository(pipelineId, cdWorkflow, clusterConfig, runStageInEnv) + } + if err != nil { + impl.logger.Errorw("err on fetch workflow logs", "err", err) + return nil, nil, err + } else if logStream == nil { + return nil, cleanUp, fmt.Errorf("no logs found for pod %s", cdWorkflow.PodName) + } + } + logReader := bufio.NewReader(logStream) + return logReader, cleanUp, err +} + +func (impl *HandlerServiceImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { + impl.logger.Debug("getting historic logs", "pipelineId", pipelineId) + + cdConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() // TODO -fixme + cdConfigCdCacheRegion := impl.config.GetDefaultCdLogsBucketRegion() + + cdLogRequest := types.BuildLogRequest{ + PipelineId: cdWorkflow.CdWorkflow.PipelineId, + WorkflowId: cdWorkflow.Id, + PodName: cdWorkflow.PodName, + LogsFilePath: cdWorkflow.LogLocation, // impl.ciCdConfig.CiDefaultBuildLogsKeyPrefix + "/" + cdWorkflow.Name + "/main.log", //TODO - fixme + CloudProvider: impl.config.CloudProvider, + AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, + AccountName: impl.config.AzureAccountName, + BlobContainerName: impl.config.AzureBlobContainerCiLog, + AccountKey: impl.config.AzureAccountKey, + }, + AwsS3BaseConfig: &blob_storage.AwsS3BaseConfig{ + AccessKey: impl.config.BlobStorageS3AccessKey, + Passkey: impl.config.BlobStorageS3SecretKey, + EndpointUrl: impl.config.BlobStorageS3Endpoint, + IsInSecure: impl.config.BlobStorageS3EndpointInsecure, + BucketName: cdConfigLogsBucket, + Region: cdConfigCdCacheRegion, + VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, + }, + GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ + BucketName: cdConfigLogsBucket, + CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, + }, + } + useExternalBlobStorage := pipeline.IsExternalBlobStorageEnabled(isExt, impl.config.UseBlobStorageConfigInCdWorkflow) + if useExternalBlobStorage { + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, cdWorkflow.Namespace) + if err != nil { + impl.logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, nil, err + } + rq := &cdLogRequest + rq.SetBuildLogRequest(cmConfig, secretConfig) + } + + impl.logger.Debugw("s3 log req ", "pipelineId", pipelineId, "runnerId", cdWorkflow.Id) + oldLogsStream, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, cdLogRequest) + if err != nil { + impl.logger.Errorw("err", err) + return nil, nil, err + } + logReader := bufio.NewReader(oldLogsStream) + return logReader, cleanUp, err +} diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go b/pkg/deployment/trigger/devtronApps/preStageHandlerCode.go similarity index 97% rename from pkg/deployment/trigger/devtronApps/PreStageTriggerService.go rename to pkg/deployment/trigger/devtronApps/preStageHandlerCode.go index 580bbf4da9..ad0051e76d 100644 --- a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/preStageHandlerCode.go @@ -35,15 +35,14 @@ import ( bean7 "github.com/devtron-labs/devtron/pkg/auth/user/bean" bean4 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/bean/common" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" repository4 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" bean5 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" adapter2 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/adapter" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" - "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/adapter" pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" - "github.com/devtron-labs/devtron/pkg/pipeline/constants" repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/plugin" @@ -62,7 +61,7 @@ import ( "time" ) -func (impl *TriggerServiceImpl) TriggerPreStage(request bean.TriggerRequest) (*bean6.ManifestPushTemplate, error) { +func (impl *HandlerServiceImpl) TriggerPreStage(request bean.TriggerRequest) (*bean6.ManifestPushTemplate, error) { request.WorkflowType = bean2.CD_WORKFLOW_TYPE_PRE // setting triggeredAt variable to have consistent data for various audit log places in db for deployment time triggeredAt := time.Now() @@ -148,7 +147,7 @@ func (impl *TriggerServiceImpl) TriggerPreStage(request bean.TriggerRequest) (*b cdStageWorkflowRequest.Pipeline = pipeline cdStageWorkflowRequest.Env = env cdStageWorkflowRequest.Type = pipelineConfigBean.CD_WORKFLOW_PIPELINE_TYPE - _, jobHelmPackagePath, err := impl.cdWorkflowService.SubmitWorkflow(cdStageWorkflowRequest) + _, jobHelmPackagePath, err := impl.workflowService.SubmitWorkflow(cdStageWorkflowRequest) span.End() if err != nil { runner.Status = cdWorkflow.WorkflowFailed @@ -178,7 +177,7 @@ func (impl *TriggerServiceImpl) TriggerPreStage(request bean.TriggerRequest) (*b return manifestPushTemplate, nil } -func (impl *TriggerServiceImpl) TriggerAutoCDOnPreStageSuccess(triggerContext bean.TriggerContext, cdPipelineId, ciArtifactId, workflowId int) error { +func (impl *HandlerServiceImpl) TriggerAutoCDOnPreStageSuccess(triggerContext bean.TriggerContext, cdPipelineId, ciArtifactId, workflowId int) error { pipeline, err := impl.pipelineRepository.FindById(cdPipelineId) if err != nil { return err @@ -217,7 +216,7 @@ func (impl *TriggerServiceImpl) TriggerAutoCDOnPreStageSuccess(triggerContext be } return nil } -func (impl *TriggerServiceImpl) checkDeploymentTriggeredAlready(wfId int) bool { +func (impl *HandlerServiceImpl) checkDeploymentTriggeredAlready(wfId int) bool { deploymentTriggeredAlready := false // TODO : need to check this logic for status check in case of multiple deployments requirement for same workflow workflowRunner, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(context.Background(), wfId, bean2.CD_WORKFLOW_TYPE_DEPLOY) @@ -229,7 +228,7 @@ func (impl *TriggerServiceImpl) checkDeploymentTriggeredAlready(wfId int) bool { return deploymentTriggeredAlready } -func (impl *TriggerServiceImpl) createStartingWfAndRunner(request bean.TriggerRequest, triggeredAt time.Time) (*pipelineConfig.CdWorkflow, *pipelineConfig.CdWorkflowRunner, error) { +func (impl *HandlerServiceImpl) createStartingWfAndRunner(request bean.TriggerRequest, triggeredAt time.Time) (*pipelineConfig.CdWorkflow, *pipelineConfig.CdWorkflowRunner, error) { triggeredBy := request.TriggeredBy artifact := request.Artifact pipeline := request.Pipeline @@ -273,7 +272,7 @@ func (impl *TriggerServiceImpl) createStartingWfAndRunner(request bean.TriggerRe return cdWf, runner, nil } -func (impl *TriggerServiceImpl) getEnvAndNsIfRunStageInEnv(ctx context.Context, request bean.TriggerRequest) (*repository4.Environment, string, error) { +func (impl *HandlerServiceImpl) getEnvAndNsIfRunStageInEnv(ctx context.Context, request bean.TriggerRequest) (*repository4.Environment, string, error) { workflowStage := request.WorkflowType pipeline := request.Pipeline var env *repository4.Environment @@ -298,7 +297,7 @@ func (impl *TriggerServiceImpl) getEnvAndNsIfRunStageInEnv(ctx context.Context, return env, namespace, nil } -func (impl *TriggerServiceImpl) checkVulnerabilityStatusAndFailWfIfNeeded(ctx context.Context, artifact *repository.CiArtifact, +func (impl *HandlerServiceImpl) checkVulnerabilityStatusAndFailWfIfNeeded(ctx context.Context, artifact *repository.CiArtifact, cdPipeline *pipelineConfig.Pipeline, runner *pipelineConfig.CdWorkflowRunner, triggeredBy int32) error { //checking vulnerability for the selected image vulnerabilityCheckRequest := adapter2.GetVulnerabilityCheckRequest(cdPipeline, artifact.ImageDigest) @@ -325,9 +324,9 @@ func (impl *TriggerServiceImpl) checkVulnerabilityStatusAndFailWfIfNeeded(ctx co } // setCopyContainerImagePluginDataAndReserveImages sets required fields in cdStageWorkflowRequest and reserve images generated by plugin -func (impl *TriggerServiceImpl) setCopyContainerImagePluginDataAndReserveImages(cdStageWorkflowRequest *types.WorkflowRequest, pipelineId int, pipelineStage string, artifact *repository.CiArtifact) ([]int, error) { +func (impl *HandlerServiceImpl) setCopyContainerImagePluginDataAndReserveImages(cdStageWorkflowRequest *types.WorkflowRequest, pipelineId int, pipelineStage string, artifact *repository.CiArtifact) ([]int, error) { - copyContainerImagePluginDetail, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(pipeline.COPY_CONTAINER_IMAGE) + copyContainerImagePluginDetail, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(buildCommonBean.COPY_CONTAINER_IMAGE) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in getting copyContainerImage plugin id", "err", err) return nil, err @@ -363,7 +362,7 @@ func (impl *TriggerServiceImpl) setCopyContainerImagePluginDataAndReserveImages( impl.logger.Errorw("error in parsing copyContainerImage input variable", "err", err) return nil, err } - if version == pipeline.COPY_CONTAINER_IMAGE_VERSION_V1 { + if version == buildCommonBean.COPY_CONTAINER_IMAGE_VERSION_V1 { // this is needed in ci runner only for v1 cdStageWorkflowRequest.RegistryDestinationImageMap = registryDestinationImageMap } @@ -408,7 +407,7 @@ func (impl *TriggerServiceImpl) setCopyContainerImagePluginDataAndReserveImages( return imagePathReservationIds, nil } -func (impl *TriggerServiceImpl) getDockerTagAndCustomTagIdForPlugin(pipelineStage string, pipelineId int, artifact *repository.CiArtifact) (string, int, error) { +func (impl *HandlerServiceImpl) getDockerTagAndCustomTagIdForPlugin(pipelineStage string, pipelineId int, artifact *repository.CiArtifact) (string, int, error) { var pipelineStageEntityType int if pipelineStage == types.PRE { pipelineStageEntityType = pipelineConfigBean.EntityTypePreCD @@ -443,7 +442,7 @@ func (impl *TriggerServiceImpl) getDockerTagAndCustomTagIdForPlugin(pipelineStag return DockerImageTag, customTagId, nil } -func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflowRunner, cdWf *pipelineConfig.CdWorkflow, cdPipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean5.DeploymentConfig, triggeredBy int32) (*types.WorkflowRequest, error) { +func (impl *HandlerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflowRunner, cdWf *pipelineConfig.CdWorkflow, cdPipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean5.DeploymentConfig, triggeredBy int32) (*types.WorkflowRequest, error) { if cdPipeline.App.Id == 0 { appModel, err := impl.appRepository.FindById(cdPipeline.AppId) if err != nil { @@ -532,7 +531,7 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow return nil, err } ciProjectDetail.CommitTime = commitTime.Format(bean4.LayoutRFC3339) - } else if ciPipeline.PipelineType == string(constants.CI_JOB) { + } else if ciPipeline.PipelineType == string(buildCommonBean.CI_JOB) { // This has been done to resolve unmarshalling issue in ci-runner, in case of no commit time(eg- polling container images) ciProjectDetail.CommitTime = time.Time{}.Format(bean4.LayoutRFC3339) } else { @@ -911,7 +910,7 @@ getBuildRegistryConfigForArtifact performs the following logic to get Pre/Post C If the ci_pipeline_type type is CI_JOB We will always fetch the registry credentials from the ci_template_override table */ -func (impl *TriggerServiceImpl) getBuildRegistryConfigForArtifact(sourceCiPipeline *pipelineConfig.CiPipeline, artifact *repository.CiArtifact, appId int) (*types.DockerArtifactStoreBean, error) { +func (impl *HandlerServiceImpl) getBuildRegistryConfigForArtifact(sourceCiPipeline *pipelineConfig.CiPipeline, artifact *repository.CiArtifact, appId int) (*types.DockerArtifactStoreBean, error) { var buildRegistryConfig *types.DockerArtifactStoreBean var err error @@ -971,7 +970,7 @@ func (impl *TriggerServiceImpl) getBuildRegistryConfigForArtifact(sourceCiPipeli return buildRegistryConfig, nil } -func (impl *TriggerServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, destinationImages []string) ([]int, error) { +func (impl *HandlerServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, destinationImages []string) ([]int, error) { var imagePathReservationIds []int for _, image := range destinationImages { @@ -1005,7 +1004,7 @@ func setExtraEnvVariableInDeployStep(deploySteps []*pipelineConfigBean.StepObjec } } -func (impl *TriggerServiceImpl) getDeployStageDetails(pipelineId int) (pipelineConfig.CdWorkflowRunner, string, int, error) { +func (impl *HandlerServiceImpl) getDeployStageDetails(pipelineId int) (pipelineConfig.CdWorkflowRunner, string, int, error) { deployStageWfr := pipelineConfig.CdWorkflowRunner{} //getting deployment pipeline latest wfr by pipelineId deployStageWfr, err := impl.cdWorkflowRepository.FindLatestByPipelineIdAndRunnerType(pipelineId, bean2.CD_WORKFLOW_TYPE_DEPLOY) @@ -1026,7 +1025,7 @@ func (impl *TriggerServiceImpl) getDeployStageDetails(pipelineId int) (pipelineC return deployStageWfr, deployStageTriggeredByUserEmail, pipelineReleaseCounter, nil } -func (impl *TriggerServiceImpl) buildArtifactLocationForS3(cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) (string, string, string) { +func (impl *HandlerServiceImpl) buildArtifactLocationForS3(cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) (string, string, string) { cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() cdWorkflowConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() ArtifactLocation := fmt.Sprintf("s3://"+path.Join(cdWorkflowConfigLogsBucket, cdArtifactLocationFormat), cdWf.Id, runner.Id) @@ -1034,7 +1033,7 @@ func (impl *TriggerServiceImpl) buildArtifactLocationForS3(cdWf *pipelineConfig. return ArtifactLocation, cdWorkflowConfigLogsBucket, artifactFileName } -func (impl *TriggerServiceImpl) buildDefaultArtifactLocation(savedWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) string { +func (impl *HandlerServiceImpl) buildDefaultArtifactLocation(savedWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) string { cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() ArtifactLocation := fmt.Sprintf(cdArtifactLocationFormat, savedWf.Id, runner.Id) return ArtifactLocation @@ -1046,7 +1045,7 @@ func ReplaceImageTagWithDigest(image, digest string) string { return imageWithDigest } -func (impl *TriggerServiceImpl) sendPreStageNotification(ctx context.Context, cdWf *pipelineConfig.CdWorkflow, pipeline *pipelineConfig.Pipeline) error { +func (impl *HandlerServiceImpl) sendPreStageNotification(ctx context.Context, cdWf *pipelineConfig.CdWorkflow, pipeline *pipelineConfig.Pipeline) error { wfr, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, cdWf.Id, bean2.CD_WORKFLOW_TYPE_PRE) if err != nil { return err diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService_ent.go b/pkg/deployment/trigger/devtronApps/preStageHandlerCode_ent.go similarity index 82% rename from pkg/deployment/trigger/devtronApps/PreStageTriggerService_ent.go rename to pkg/deployment/trigger/devtronApps/preStageHandlerCode_ent.go index 8f5b3a5c39..d285ba8224 100644 --- a/pkg/deployment/trigger/devtronApps/PreStageTriggerService_ent.go +++ b/pkg/deployment/trigger/devtronApps/preStageHandlerCode_ent.go @@ -14,33 +14,33 @@ import ( "time" ) -func (impl *TriggerServiceImpl) checkFeasibilityForPreStage(pipeline *pipelineConfig.Pipeline, request *bean2.TriggerRequest, +func (impl *HandlerServiceImpl) checkFeasibilityForPreStage(pipeline *pipelineConfig.Pipeline, request *bean2.TriggerRequest, env *repository.Environment, artifact *repository2.CiArtifact, triggeredBy int32) (interface{}, error) { //here return type is interface as ResourceFilterEvaluationAudit is not present in this version return nil, nil } -func (impl *TriggerServiceImpl) createAuditDataForDeploymentWindowBypass(request bean2.TriggerRequest, wfrId int) error { +func (impl *HandlerServiceImpl) createAuditDataForDeploymentWindowBypass(request bean2.TriggerRequest, wfrId int) error { return nil } -func (impl *TriggerServiceImpl) getManifestPushTemplateForPreStage(ctx context.Context, envDeploymentConfig *bean3.DeploymentConfig, +func (impl *HandlerServiceImpl) getManifestPushTemplateForPreStage(ctx context.Context, envDeploymentConfig *bean3.DeploymentConfig, pipeline *pipelineConfig.Pipeline, artifact *repository2.CiArtifact, jobHelmPackagePath string, cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner, triggeredBy int32, triggeredAt time.Time, request bean2.TriggerRequest) (*bean6.ManifestPushTemplate, error) { return nil, nil } -func (impl *TriggerServiceImpl) setCloningModeInCIProjectDetail(ciProjectDetail *bean.CiProjectDetails, appId int, +func (impl *HandlerServiceImpl) setCloningModeInCIProjectDetail(ciProjectDetail *bean.CiProjectDetails, appId int, m *pipelineConfig.CiPipelineMaterial) error { return nil } -func (impl *TriggerServiceImpl) getPreStageBuildRegistryConfigIfSourcePipelineNotPresent(appId int) (*types.DockerArtifactStoreBean, error) { +func (impl *HandlerServiceImpl) getPreStageBuildRegistryConfigIfSourcePipelineNotPresent(appId int) (*types.DockerArtifactStoreBean, error) { return nil, fmt.Errorf("soucePipeline is mandatory, corrupt data") } -func (impl *TriggerServiceImpl) handlerFilterEvaluationAudit(filterEvaluationAudit interface{}, +func (impl *HandlerServiceImpl) handlerFilterEvaluationAudit(filterEvaluationAudit interface{}, runner *pipelineConfig.CdWorkflowRunner) error { //here ip type of filterEvaluationAudit is interface as ResourceFilterEvaluationAudit is not present in this version return nil diff --git a/pkg/deployment/trigger/devtronApps/wire_devtronAppsDeployTrigger.go b/pkg/deployment/trigger/devtronApps/wire_devtronAppsDeployTrigger.go index 4bf4174cfe..953b46422f 100644 --- a/pkg/deployment/trigger/devtronApps/wire_devtronAppsDeployTrigger.go +++ b/pkg/deployment/trigger/devtronApps/wire_devtronAppsDeployTrigger.go @@ -23,6 +23,6 @@ import ( var DevtronAppsDeployTriggerWireSet = wire.NewSet( userDeploymentRequest.WireSet, - NewTriggerServiceImpl, - wire.Bind(new(TriggerService), new(*TriggerServiceImpl)), + NewHandlerServiceImpl, + wire.Bind(new(HandlerService), new(*HandlerServiceImpl)), ) diff --git a/pkg/eventProcessor/bean/workflowEventBean.go b/pkg/eventProcessor/bean/workflowEventBean.go index 4b33b8cba4..ad150af539 100644 --- a/pkg/eventProcessor/bean/workflowEventBean.go +++ b/pkg/eventProcessor/bean/workflowEventBean.go @@ -19,6 +19,7 @@ package bean import ( "context" "encoding/json" + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/devtron-labs/common-lib/utils/registry" "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" @@ -114,3 +115,14 @@ type DevtronAppReleaseContextType struct { CancelContext context.CancelCauseFunc RunnerId int } + +type CiCdStatus struct { + DevtronOwnerInstance string `json:"devtronOwnerInstance"` + *v1alpha1.WorkflowStatus +} + +func NewCiCdStatus() CiCdStatus { + return CiCdStatus{ + WorkflowStatus: &v1alpha1.WorkflowStatus{}, + } +} diff --git a/pkg/eventProcessor/in/CDPipelineEventProcessorService.go b/pkg/eventProcessor/in/CDPipelineEventProcessorService.go index 5a54759107..cb18e00066 100644 --- a/pkg/eventProcessor/in/CDPipelineEventProcessorService.go +++ b/pkg/eventProcessor/in/CDPipelineEventProcessorService.go @@ -38,7 +38,7 @@ type CDPipelineEventProcessorImpl struct { pubSubClient *pubsub.PubSubClientServiceImpl cdWorkflowCommonService cd.CdWorkflowCommonService workflowStatusService status.WorkflowStatusService - cdTriggerService devtronApps.TriggerService + cdHandlerService devtronApps.HandlerService pipelineRepository pipelineConfig.PipelineRepository installedAppReadService installedAppReader.InstalledAppReadService } @@ -47,7 +47,7 @@ func NewCDPipelineEventProcessorImpl(logger *zap.SugaredLogger, pubSubClient *pubsub.PubSubClientServiceImpl, cdWorkflowCommonService cd.CdWorkflowCommonService, workflowStatusService status.WorkflowStatusService, - cdTriggerService devtronApps.TriggerService, + cdHandlerService devtronApps.HandlerService, pipelineRepository pipelineConfig.PipelineRepository, installedAppReadService installedAppReader.InstalledAppReadService) *CDPipelineEventProcessorImpl { cdPipelineEventProcessorImpl := &CDPipelineEventProcessorImpl{ @@ -55,7 +55,7 @@ func NewCDPipelineEventProcessorImpl(logger *zap.SugaredLogger, pubSubClient: pubSubClient, cdWorkflowCommonService: cdWorkflowCommonService, workflowStatusService: workflowStatusService, - cdTriggerService: cdTriggerService, + cdHandlerService: cdHandlerService, pipelineRepository: pipelineRepository, installedAppReadService: installedAppReadService, } @@ -78,7 +78,7 @@ func (impl *CDPipelineEventProcessorImpl) SubscribeCDBulkTriggerTopic() error { ReferenceId: pointer.String(msg.MsgId), Context: context2.Background(), } - _, _, _, err = impl.cdTriggerService.ManualCdTrigger(triggerContext, event.ValuesOverrideRequest) + _, _, _, err = impl.cdHandlerService.ManualCdTrigger(triggerContext, event.ValuesOverrideRequest) if err != nil { impl.logger.Errorw("Error triggering CD", "topic", pubsub.CD_BULK_DEPLOY_TRIGGER_TOPIC, "msg", msg.Data, "err", err) } diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index 2800e9a3bb..cb34134dd1 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -27,6 +27,7 @@ import ( "github.com/devtron-labs/common-lib/utils/registry" apiBean "github.com/devtron-labs/devtron/api/bean" client "github.com/devtron-labs/devtron/client/events" + "github.com/devtron-labs/devtron/internal/middleware" "github.com/devtron-labs/devtron/internal/sql/constants" "github.com/devtron-labs/devtron/internal/sql/models" "github.com/devtron-labs/devtron/internal/sql/repository" @@ -35,6 +36,7 @@ import ( util3 "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" userBean "github.com/devtron-labs/devtron/pkg/auth/user/bean" + "github.com/devtron-labs/devtron/pkg/build/trigger" "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/deployedApp" deploymentBean "github.com/devtron-labs/devtron/pkg/deployment/deployedApp/bean" @@ -46,6 +48,7 @@ import ( eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/out/bean" "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/executors" + "github.com/devtron-labs/devtron/pkg/ucid" "github.com/devtron-labs/devtron/pkg/workflow/cd" "github.com/devtron-labs/devtron/pkg/workflow/cd/adapter" cdWorkflowBean "github.com/devtron-labs/devtron/pkg/workflow/cd/bean" @@ -78,7 +81,7 @@ type WorkflowEventProcessorImpl struct { cdHandler pipeline.CdHandler eventFactory client.EventFactory eventClient client.EventClient - cdTriggerService devtronApps.TriggerService + cdHandlerService devtronApps.HandlerService deployedAppService deployedApp.DeployedAppService webhookService pipeline.WebhookService validator *validator.Validate @@ -86,11 +89,15 @@ type WorkflowEventProcessorImpl struct { cdWorkflowCommonService cd.CdWorkflowCommonService cdPipelineConfigService pipeline.CdPipelineConfigService userDeploymentRequestService service.UserDeploymentRequestService + ucid ucid.Service devtronAppReleaseContextMap map[int]bean.DevtronAppReleaseContextType devtronAppReleaseContextMapLock *sync.Mutex appServiceConfig *app.AppServiceConfig + //ent only + ciHandlerService trigger.HandlerService + // repositories import to be removed pipelineRepository pipelineConfig.PipelineRepository ciArtifactRepository repository.CiArtifactRepository @@ -107,7 +114,7 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, workflowDagExecutor dag.WorkflowDagExecutor, ciHandler pipeline.CiHandler, cdHandler pipeline.CdHandler, eventFactory client.EventFactory, eventClient client.EventClient, - cdTriggerService devtronApps.TriggerService, + cdHandlerService devtronApps.HandlerService, deployedAppService deployedApp.DeployedAppService, webhookService pipeline.WebhookService, validator *validator.Validate, @@ -115,10 +122,12 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, cdWorkflowCommonService cd.CdWorkflowCommonService, cdPipelineConfigService pipeline.CdPipelineConfigService, userDeploymentRequestService service.UserDeploymentRequestService, + ucid ucid.Service, pipelineRepository pipelineConfig.PipelineRepository, ciArtifactRepository repository.CiArtifactRepository, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, - deploymentConfigService common.DeploymentConfigService) (*WorkflowEventProcessorImpl, error) { + deploymentConfigService common.DeploymentConfigService, + ciHandlerService trigger.HandlerService) (*WorkflowEventProcessorImpl, error) { impl := &WorkflowEventProcessorImpl{ logger: logger, pubSubClient: pubSubClient, @@ -131,7 +140,7 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, eventFactory: eventFactory, eventClient: eventClient, workflowDagExecutor: workflowDagExecutor, - cdTriggerService: cdTriggerService, + cdHandlerService: cdHandlerService, deployedAppService: deployedAppService, webhookService: webhookService, validator: validator, @@ -139,12 +148,14 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, cdWorkflowCommonService: cdWorkflowCommonService, cdPipelineConfigService: cdPipelineConfigService, userDeploymentRequestService: userDeploymentRequestService, + ucid: ucid, devtronAppReleaseContextMap: make(map[int]bean.DevtronAppReleaseContextType), devtronAppReleaseContextMapLock: &sync.Mutex{}, pipelineRepository: pipelineRepository, ciArtifactRepository: ciArtifactRepository, cdWorkflowRepository: cdWorkflowRepository, deploymentConfigService: deploymentConfigService, + ciHandlerService: ciHandlerService, } appServiceConfig, err := app.GetAppServiceConfig() if err != nil { @@ -320,7 +331,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeTriggerBulkAction() error { ApplyAuth: false, TriggerContext: triggerContext, } - err = impl.cdTriggerService.TriggerStageForBulk(triggerRequest) + err = impl.cdHandlerService.TriggerStageForBulk(triggerRequest) if err != nil { impl.logger.Errorw("error in cd trigger ", "err", err) wf.WorkflowStatus = cdWorkflowModelBean.TRIGGER_ERROR @@ -387,30 +398,45 @@ func (impl *WorkflowEventProcessorImpl) SubscribeHibernateBulkAction() error { func (impl *WorkflowEventProcessorImpl) SubscribeCIWorkflowStatusUpdate() error { callback := func(msg *model.PubSubMsg) { - wfStatus := v1alpha1.WorkflowStatus{} + wfStatus := bean.NewCiCdStatus() err := json.Unmarshal([]byte(msg.Data), &wfStatus) if err != nil { impl.logger.Errorw("error while unmarshalling wf status update", "err", err, "msg", msg.Data) return } - - err = impl.ciHandler.CheckAndReTriggerCI(wfStatus) - if err != nil { - impl.logger.Errorw("error in checking and re triggering ci", "err", err) - //don't return as we have to update the workflow status + if len(wfStatus.DevtronOwnerInstance) != 0 { + devtronUCID, _, err := impl.ucid.GetUCIDWithOutCache() + if err != nil { + impl.logger.Errorw("error in getting UCID", "err", err) + return + } + if wfStatus.DevtronOwnerInstance != devtronUCID { + impl.logger.Warnw("mis match in UCID. skipping...", "devtronAdministratorInstance", wfStatus.DevtronOwnerInstance, "devtronUCID", devtronUCID) + return + } } - - _, err = impl.ciHandler.UpdateWorkflow(wfStatus) + // update the ci workflow status + ciWfId, stateChanged, err := impl.ciHandler.UpdateWorkflow(wfStatus) if err != nil { - impl.logger.Errorw("error on update workflow status", "err", err, "msg", msg.Data) + impl.logger.Errorw("error on update workflow status", "msg", msg.Data, "err", err) return } - + if stateChanged { + // check if we need to re-trigger the ci + err = impl.ciHandlerService.CheckAndReTriggerCI(wfStatus) + if err != nil { + middleware.ReTriggerFailedCounter.WithLabelValues("CI", strconv.Itoa(ciWfId)).Inc() + impl.logger.Errorw("error in checking and re triggering ci", "wfStatus", wfStatus, "err", err) + return + } + } else { + impl.logger.Debugw("no state change detected for the ci workflow status update, ignoring this event", "workflowRunnerId", ciWfId, "wfStatus", wfStatus) + } } // add required logging here var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) { - wfStatus := v1alpha1.WorkflowStatus{} + wfStatus := bean.NewCiCdStatus() err := json.Unmarshal([]byte(msg.Data), &wfStatus) if err != nil { return "error while unmarshalling wf status update", []interface{}{"err", err, "msg", msg.Data} @@ -423,7 +449,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCIWorkflowStatusUpdate() error err := impl.pubSubClient.Subscribe(pubsub.WORKFLOW_STATUS_UPDATE_TOPIC, callback, loggerFunc) if err != nil { - impl.logger.Error("err", err) + impl.logger.Error("error in subscribing to ci workflow status update topic", "err", err) return err } return nil @@ -431,17 +457,27 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCIWorkflowStatusUpdate() error func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error { callback := func(msg *model.PubSubMsg) { - wfStatus := v1alpha1.WorkflowStatus{} + wfStatus := bean.NewCiCdStatus() err := json.Unmarshal([]byte(msg.Data), &wfStatus) if err != nil { impl.logger.Error("Error while unmarshalling wfStatus json object", "error", err) return } - - wfrId, wfrStatus, stateChanged, err := impl.cdHandler.UpdateWorkflow(wfStatus) - impl.logger.Debugw("UpdateWorkflow", "wfrId", wfrId, "wfrStatus", wfrStatus) + if len(wfStatus.DevtronOwnerInstance) != 0 { + devtronUCID, _, err := impl.ucid.GetUCIDWithOutCache() + if err != nil { + impl.logger.Errorw("error in getting UCID", "err", err) + return + } + if wfStatus.DevtronOwnerInstance != devtronUCID { + impl.logger.Warnw("mis match in UCID. skipping...", "devtronAdministratorInstance", wfStatus.DevtronOwnerInstance, "devtronUCID", devtronUCID) + return + } + } + wfrId, status, stateChanged, err := impl.cdHandler.UpdateWorkflow(wfStatus) + impl.logger.Debugw("cd UpdateWorkflow for wfStatus", "wfrId", wfrId, "status", status, "wfStatus", wfStatus) if err != nil { - impl.logger.Error("err", err) + impl.logger.Errorw("error in cd workflow status update", "wfrId", wfrId, "status", status, "wfStatus", wfStatus, "err", err) return } @@ -452,11 +488,12 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error return } - if wfrStatus == string(v1alpha1.NodeFailed) || wfrStatus == string(v1alpha1.NodeError) { + if wfr.Status == string(v1alpha1.NodeFailed) || wfr.Status == string(v1alpha1.NodeError) { if len(wfr.ImagePathReservationIds) > 0 { err := impl.cdHandler.DeactivateImageReservationPathsOnFailure(wfr.ImagePathReservationIds) if err != nil { impl.logger.Errorw("error in removing image path reservation ", "imagePathReservationIds", wfr.ImagePathReservationIds, "err", err) + // not returning here as we need to send the notification event and re-trigger the cd stage (if required) } } } @@ -464,13 +501,16 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error wfStatusInEvent := string(wfStatus.Phase) if wfStatusInEvent == string(v1alpha1.NodeSucceeded) || wfStatusInEvent == string(v1alpha1.NodeFailed) || wfStatusInEvent == string(v1alpha1.NodeError) { // the re-trigger should only happen when we get a pod deleted event. - if wfr != nil && executors.CheckIfReTriggerRequired(wfrStatus, wfStatus.Message, wfr.Status) { + if executors.CheckIfReTriggerRequired(status, wfStatus.Message, wfr.Status) { err = impl.workflowDagExecutor.HandleCdStageReTrigger(wfr) if err != nil { // check if this log required or not - impl.logger.Errorw("error in HandleCdStageReTrigger", "workflowRunnerId", wfr.Id, "workflowStatus", wfrStatus, "workflowStatusMessage", wfStatus.Message, "error", err) + workflowType := fmt.Sprintf("%s-CD", wfr.WorkflowType) + middleware.ReTriggerFailedCounter.WithLabelValues(workflowType, strconv.Itoa(wfrId)).Inc() + impl.logger.Errorw("error in HandleCdStageReTrigger", "workflowRunnerId", wfr.Id, "status", status, "message", wfStatus.Message, "error", err) + return } - impl.logger.Debugw("re-triggered cd stage", "workflowRunnerId", wfr.Id, "workflowStatus", wfrStatus, "workflowStatusMessage", wfStatus.Message) + impl.logger.Infow("re-triggered cd stage", "workflowRunnerId", wfr.Id, "status", status, "message", wfStatus.Message) } else { // we send this notification on *workflow_runner* status, both success and failure // during workflow runner failure, particularly when failure occurred due to pod deletion , we get two events from kubewatch. @@ -486,13 +526,13 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error } } } else { - impl.logger.Debugw("no state change detected for the cd workflow status update, ignoring this event", "workflowRunnerId", wfrId, "wfrStatus", wfrStatus) + impl.logger.Debugw("no state change detected for the cd workflow status update, ignoring this event", "workflowRunnerId", wfrId, "status", status) } } // add required logging here var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) { - wfStatus := v1alpha1.WorkflowStatus{} + wfStatus := bean.NewCiCdStatus() err := json.Unmarshal([]byte(msg.Data), &wfStatus) if err != nil { return "error while unmarshalling wfStatus json object", []interface{}{"error", err} @@ -503,7 +543,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error err := impl.pubSubClient.Subscribe(pubsub.CD_WORKFLOW_STATUS_UPDATE, callback, loggerFunc) if err != nil { - impl.logger.Error("err", err) + impl.logger.Error("error in subscribing to cd workflow status update topic", "err", err) return err } return nil diff --git a/pkg/pipeline/WorkflowService.go b/pkg/executor/WorkflowService.go similarity index 93% rename from pkg/pipeline/WorkflowService.go rename to pkg/executor/WorkflowService.go index 3cc2215867..14ff02b5b8 100644 --- a/pkg/pipeline/WorkflowService.go +++ b/pkg/executor/WorkflowService.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package pipeline +package executor import ( "context" @@ -28,16 +28,18 @@ import ( "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" - bean2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/config/read" v1 "github.com/devtron-labs/devtron/pkg/infraConfig/bean/v1" k8s2 "github.com/devtron-labs/devtron/pkg/k8s" + "github.com/devtron-labs/devtron/pkg/pipeline" bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders" "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders/infraGetters" "github.com/devtron-labs/devtron/pkg/pipeline/types" + "github.com/devtron-labs/devtron/pkg/ucid" "go.uber.org/zap" v12 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -66,12 +68,13 @@ type WorkflowServiceImpl struct { ciCdConfig *types.CiCdConfig configMapService read.ConfigReadService envRepository repository2.EnvironmentRepository - globalCMCSService GlobalCMCSService + globalCMCSService pipeline.GlobalCMCSService argoWorkflowExecutor executors.ArgoWorkflowExecutor systemWorkflowExecutor executors.SystemWorkflowExecutor - k8sUtil *k8s.K8sServiceImpl k8sCommonService k8s2.K8sCommonService infraProvider infraProviders.InfraProvider + ucid ucid.Service + k8sUtil *k8s.K8sServiceImpl } // TODO: Move to bean @@ -80,12 +83,14 @@ func NewWorkflowServiceImpl(Logger *zap.SugaredLogger, envRepository repository2.EnvironmentRepository, ciCdConfig *types.CiCdConfig, configMapService read.ConfigReadService, - globalCMCSService GlobalCMCSService, + globalCMCSService pipeline.GlobalCMCSService, argoWorkflowExecutor executors.ArgoWorkflowExecutor, - k8sUtil *k8s.K8sServiceImpl, systemWorkflowExecutor executors.SystemWorkflowExecutor, k8sCommonService k8s2.K8sCommonService, - infraProvider infraProviders.InfraProvider) (*WorkflowServiceImpl, error) { + infraProvider infraProviders.InfraProvider, + ucid ucid.Service, + k8sUtil *k8s.K8sServiceImpl, +) (*WorkflowServiceImpl, error) { commonWorkflowService := &WorkflowServiceImpl{ Logger: Logger, ciCdConfig: ciCdConfig, @@ -97,6 +102,7 @@ func NewWorkflowServiceImpl(Logger *zap.SugaredLogger, systemWorkflowExecutor: systemWorkflowExecutor, k8sCommonService: k8sCommonService, infraProvider: infraProvider, + ucid: ucid, } restConfig, err := k8sUtil.GetK8sInClusterRestConfig() if err != nil { @@ -109,9 +115,6 @@ func NewWorkflowServiceImpl(Logger *zap.SugaredLogger, const ( CI_NODE_SELECTOR_APP_LABEL_KEY = "devtron.ai/node-selector" - - preCdStage = "preCD" - postCdStage = "postCD" ) func (impl *WorkflowServiceImpl) SubmitWorkflow(workflowRequest *types.WorkflowRequest) (*unstructured.UnstructuredList, string, error) { @@ -119,13 +122,16 @@ func (impl *WorkflowServiceImpl) SubmitWorkflow(workflowRequest *types.WorkflowR if err != nil { return nil, "", err } - workflowExecutor := impl.getWorkflowExecutor(workflowRequest.WorkflowExecutor) - if workflowExecutor == nil { - return nil, "", errors.New("workflow executor not found") + var createdWf *unstructured.UnstructuredList + canExecuteWorkflow, jobHelmChartPath, err := impl.checkIfCanExecuteWorkflowAndHandleVirtualExec(workflowRequest, workflowTemplate) + if canExecuteWorkflow { + workflowExecutor := impl.getWorkflowExecutor(workflowRequest.WorkflowExecutor) + if workflowExecutor == nil { + return nil, "", errors.New("workflow executor not found") + } + createdWf, err = workflowExecutor.ExecuteWorkflow(workflowTemplate) } - createdWf, err := workflowExecutor.ExecuteWorkflow(workflowTemplate) - jobHelmPackagePath := "" // due to ENT - return createdWf, jobHelmPackagePath, err + return createdWf, jobHelmChartPath, err } func (impl *WorkflowServiceImpl) createWorkflowTemplate(workflowRequest *types.WorkflowRequest) (bean3.WorkflowTemplate, error) { @@ -141,6 +147,11 @@ func (impl *WorkflowServiceImpl) createWorkflowTemplate(workflowRequest *types.W return bean3.WorkflowTemplate{}, err } + workflowTemplate, err = impl.updateWorkflowTemplateWithLabels(workflowRequest, workflowTemplate) + if err != nil { + impl.Logger.Errorw("error occurred while updating workflow template with labels", "err", err) + return bean3.WorkflowTemplate{}, err + } workflowRequest.AddNodeConstraintsFromConfig(&workflowTemplate, impl.ciCdConfig) infraConfiguration := &v1.InfraConfig{} shouldAddExistingCmCsInWorkflow := impl.shouldAddExistingCmCsInWorkflow(workflowRequest) @@ -213,12 +224,18 @@ func (impl *WorkflowServiceImpl) createWorkflowTemplate(workflowRequest *types.W clusterConfig, err := impl.getClusterConfig(workflowRequest) workflowTemplate.ClusterConfig = clusterConfig workflowTemplate.WorkflowType = workflowRequest.GetWorkflowTypeForWorkflowRequest() + devtronUCID, _, err := impl.ucid.GetUCIDWithOutCache() + if err != nil { + impl.Logger.Errorw("error in getting UCID", "err", err) + return bean3.WorkflowTemplate{}, err + } + workflowTemplate.DevtronInstanceUID = devtronUCID return workflowTemplate, nil } func (impl *WorkflowServiceImpl) shouldAddExistingCmCsInWorkflow(workflowRequest *types.WorkflowRequest) bool { // CmCs are not added for CI_JOB if IgnoreCmCsInCiJob is true - if workflowRequest.CiPipelineType == string(bean2.CI_JOB) && impl.ciCdConfig.IgnoreCmCsInCiJob { + if workflowRequest.CiPipelineType == string(common.CI_JOB) && impl.ciCdConfig.IgnoreCmCsInCiJob { return false } return true @@ -378,6 +395,7 @@ func (impl *WorkflowServiceImpl) getWorkflowExecutor(executorType cdWorkflow.Wor impl.Logger.Warnw("workflow executor not found", "type", executorType) return nil } + func (impl *WorkflowServiceImpl) GetWorkflow(executorType cdWorkflow.WorkflowExecutorType, name string, namespace string, restConfig *rest.Config) (*unstructured.UnstructuredList, error) { impl.Logger.Debug("getting wf", name) workflowExecutor := impl.getWorkflowExecutor(executorType) diff --git a/pkg/pipeline/WorkflowServiceIT_test.go b/pkg/executor/WorkflowServiceIT_test.go similarity index 99% rename from pkg/pipeline/WorkflowServiceIT_test.go rename to pkg/executor/WorkflowServiceIT_test.go index c025abe1fe..b1d569487f 100644 --- a/pkg/pipeline/WorkflowServiceIT_test.go +++ b/pkg/executor/WorkflowServiceIT_test.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package pipeline +package executor import ( "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" @@ -35,6 +35,7 @@ import ( "github.com/devtron-labs/devtron/pkg/commonService" k8s2 "github.com/devtron-labs/devtron/pkg/k8s" "github.com/devtron-labs/devtron/pkg/k8s/informer" + "github.com/devtron-labs/devtron/pkg/pipeline" bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/types" @@ -59,10 +60,10 @@ var cmManifest3 = "{\"kind\":\"ConfigMap\",\"apiVersion\":\"v1\",\"metadata\":{\ var cmManifest4 = "{\"kind\":\"ConfigMap\",\"apiVersion\":\"v1\",\"metadata\":{\"name\":\"cm4-5-ci\",\"creationTimestamp\":null,\"ownerReferences\":[{\"apiVersion\":\"argoproj.io/v1alpha1\",\"kind\":\"Workflow\",\"name\":\"{{workflow.name}}\",\"uid\":\"{{workflow.uid}}\",\"blockOwnerDeletion\":true}]},\"data\":{\"key4\":\"value4\"}}" func getWorkflowServiceImpl(t *testing.T) *WorkflowServiceImpl { - logger, dbConnection := getDbConnAndLoggerService(t) + logger, dbConnection := pipeline.getDbConnAndLoggerService(t) ciCdConfig, _ := types.GetCiCdConfig() newGlobalCMCSRepositoryImpl := repository.NewGlobalCMCSRepositoryImpl(logger, dbConnection) - globalCMCSServiceImpl := NewGlobalCMCSServiceImpl(logger, newGlobalCMCSRepositoryImpl) + globalCMCSServiceImpl := pipeline.NewGlobalCMCSServiceImpl(logger, newGlobalCMCSRepositoryImpl) newEnvConfigOverrideRepository := chartConfig.NewEnvConfigOverrideRepository(dbConnection) newConfigMapRepositoryImpl := chartConfig.NewConfigMapRepositoryImpl(logger, dbConnection) newChartRepository := chartRepoRepository.NewChartRepository(dbConnection) diff --git a/pkg/executor/WorkflowService_ent.go b/pkg/executor/WorkflowService_ent.go new file mode 100644 index 0000000000..f01a801a90 --- /dev/null +++ b/pkg/executor/WorkflowService_ent.go @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2020-2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package executor + +import ( + bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/types" +) + +func (impl *WorkflowServiceImpl) checkIfCanExecuteWorkflowAndHandleVirtualExec(workflowRequest *types.WorkflowRequest, workflowTemplate bean3.WorkflowTemplate) (canExecuteWorkflow bool, jobHelmChartPath string, err error) { + return true, "", nil +} + +func (impl *WorkflowServiceImpl) updateWorkflowTemplateWithLabels(workflowRequest *types.WorkflowRequest, workflowTemplate bean3.WorkflowTemplate) (bean3.WorkflowTemplate, error) { + return workflowTemplate, nil +} diff --git a/pkg/pipeline/WorkflowService_test.go b/pkg/executor/WorkflowService_test.go similarity index 99% rename from pkg/pipeline/WorkflowService_test.go rename to pkg/executor/WorkflowService_test.go index 4b4e937d68..1ebcb9639c 100644 --- a/pkg/pipeline/WorkflowService_test.go +++ b/pkg/executor/WorkflowService_test.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package pipeline +package executor import ( "fmt" diff --git a/pkg/executor/wire_executor.go b/pkg/executor/wire_executor.go new file mode 100644 index 0000000000..a99bf31f61 --- /dev/null +++ b/pkg/executor/wire_executor.go @@ -0,0 +1,8 @@ +package executor + +import "github.com/google/wire" + +var ExecutorWireSet = wire.NewSet( + NewWorkflowServiceImpl, + wire.Bind(new(WorkflowService), new(*WorkflowServiceImpl)), +) diff --git a/pkg/gitops/GitOpsConfigService.go b/pkg/gitops/GitOpsConfigService.go index a0776f75b3..f095136674 100644 --- a/pkg/gitops/GitOpsConfigService.go +++ b/pkg/gitops/GitOpsConfigService.go @@ -251,7 +251,7 @@ func (impl *GitOpsConfigServiceImpl) registerGitOpsClientConfig(ctx context.Cont } } else { - clusterBean, err := impl.clusterReadService.FindOne(bean2.DEFAULT_CLUSTER) + clusterBean, err := impl.clusterReadService.FindOne(bean2.DefaultCluster) if err != nil { return nil, err } @@ -538,7 +538,7 @@ func (impl *GitOpsConfigServiceImpl) patchGitOpsClientConfig(model *repository.G } } else { - clusterBean, err := impl.clusterReadService.FindOne(bean2.DEFAULT_CLUSTER) + clusterBean, err := impl.clusterReadService.FindOne(bean2.DefaultCluster) if err != nil { return err } diff --git a/pkg/pipeline/BlobStorageConfigService.go b/pkg/pipeline/BlobStorageConfigService.go index fd6141c111..53ea2efde6 100644 --- a/pkg/pipeline/BlobStorageConfigService.go +++ b/pkg/pipeline/BlobStorageConfigService.go @@ -91,7 +91,7 @@ func (impl *BlobStorageConfigServiceImpl) FetchCmAndSecretBlobConfigFromExternal return cmConfig, secretConfig, nil } -func updateRequestWithExtClusterCmAndSecret(request *blob_storage.BlobStorageRequest, cmConfig *bean2.CmBlobStorageConfig, secretConfig *bean2.SecretBlobStorageConfig) *blob_storage.BlobStorageRequest { +func UpdateRequestWithExtClusterCmAndSecret(request *blob_storage.BlobStorageRequest, cmConfig *bean2.CmBlobStorageConfig, secretConfig *bean2.SecretBlobStorageConfig) *blob_storage.BlobStorageRequest { request.StorageType = cmConfig.CloudProvider request.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey @@ -113,3 +113,8 @@ func updateRequestWithExtClusterCmAndSecret(request *blob_storage.BlobStorageReq return request } + +func IsExternalBlobStorageEnabled(isExternalRun bool, useBlobStorageConfigInCdWorkflow bool) bool { + // TODO impl.config.UseBlobStorageConfigInCdWorkflow fetches the live status, we need to check from db as well, we should put useExternalBlobStorage in db + return isExternalRun && !useBlobStorageConfigInCdWorkflow +} diff --git a/pkg/pipeline/BuildPipelineConfigService.go b/pkg/pipeline/BuildPipelineConfigService.go index c320898bdc..b003cbeedb 100644 --- a/pkg/pipeline/BuildPipelineConfigService.go +++ b/pkg/pipeline/BuildPipelineConfigService.go @@ -36,6 +36,7 @@ import ( repository3 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/repository" "github.com/devtron-labs/devtron/pkg/build/pipeline" bean3 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" "github.com/devtron-labs/devtron/pkg/build/pipeline/read" pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/history" @@ -307,11 +308,11 @@ func (impl *CiPipelineConfigServiceImpl) patchCiPipelineUpdateSource(baseCiConfi } // updating PipelineType from db if not present in request if modifiedCiPipeline.PipelineType == "" { - if bean3.PipelineType(pipeline.PipelineType) != "" { - modifiedCiPipeline.PipelineType = bean3.PipelineType(pipeline.PipelineType) + if common.PipelineType(pipeline.PipelineType) != "" { + modifiedCiPipeline.PipelineType = common.PipelineType(pipeline.PipelineType) } else { // updating default pipelineType if not present in request - modifiedCiPipeline.PipelineType = bean3.DefaultPipelineType + modifiedCiPipeline.PipelineType = common.DefaultPipelineType } } @@ -595,9 +596,9 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipeline(appId int) (ciConfig *bea Script: ciScript.Script, OutputLocation: ciScript.OutputLocation, } - if ciScript.Stage == BEFORE_DOCKER_BUILD { + if ciScript.Stage == common.BEFORE_DOCKER_BUILD { beforeDockerBuildScripts = append(beforeDockerBuildScripts, ciScriptResp) - } else if ciScript.Stage == AFTER_DOCKER_BUILD { + } else if ciScript.Stage == common.AFTER_DOCKER_BUILD { afterDockerBuildScripts = append(afterDockerBuildScripts, ciScriptResp) } } @@ -622,7 +623,7 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipeline(appId int) (ciConfig *bea AfterDockerBuildScripts: afterDockerBuildScripts, ScanEnabled: pipeline.ScanEnabled, IsDockerConfigOverridden: pipeline.IsDockerConfigOverridden, - PipelineType: bean3.PipelineType(pipeline.PipelineType), + PipelineType: common.PipelineType(pipeline.PipelineType), } ciEnvMapping, err := impl.ciPipelineRepository.FindCiEnvMappingByCiPipelineId(pipeline.Id) if err != nil && err != pg.ErrNoRows { @@ -750,9 +751,9 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipelineById(pipelineId int) (ciPi Script: ciScript.Script, OutputLocation: ciScript.OutputLocation, } - if ciScript.Stage == BEFORE_DOCKER_BUILD { + if ciScript.Stage == common.BEFORE_DOCKER_BUILD { beforeDockerBuildScripts = append(beforeDockerBuildScripts, ciScriptResp) - } else if ciScript.Stage == AFTER_DOCKER_BUILD { + } else if ciScript.Stage == common.AFTER_DOCKER_BUILD { afterDockerBuildScripts = append(afterDockerBuildScripts, ciScriptResp) } } @@ -778,7 +779,7 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipelineById(pipelineId int) (ciPi AfterDockerBuildScripts: afterDockerBuildScripts, ScanEnabled: pipeline.ScanEnabled, IsDockerConfigOverridden: pipeline.IsDockerConfigOverridden, - PipelineType: bean3.PipelineType(pipeline.PipelineType), + PipelineType: common.PipelineType(pipeline.PipelineType), } customTag, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id)) if err != nil && err != pg.ErrNoRows { @@ -902,7 +903,7 @@ func (impl *CiPipelineConfigServiceImpl) GetTriggerViewCiPipeline(appId int) (*b ParentCiPipeline: pipeline.ParentCiPipeline, ScanEnabled: pipeline.ScanEnabled, IsDockerConfigOverridden: pipeline.IsDockerConfigOverridden, - PipelineType: bean3.PipelineType(pipeline.PipelineType), + PipelineType: common.PipelineType(pipeline.PipelineType), } if ciTemplateBean, ok := ciOverrideTemplateMap[pipeline.Id]; ok { templateOverride := ciTemplateBean.CiTemplateOverride @@ -1268,7 +1269,7 @@ func (impl *CiPipelineConfigServiceImpl) UpdateCiTemplate(updateRequest *bean.Ci } for _, ciTemplateOverride := range ciTemplateOverrides { if _, ok := ciPipelineIdsMap[ciTemplateOverride.CiPipelineId]; ok { - if ciPipelineIdsMap[ciTemplateOverride.CiPipelineId].PipelineType == string(bean3.CI_JOB) { + if ciPipelineIdsMap[ciTemplateOverride.CiPipelineId].PipelineType == string(common.CI_JOB) { ciTemplateOverride.DockerRepository = updateRequest.DockerRepository ciTemplateOverride.DockerRegistryId = updateRequest.DockerRegistry _, err = impl.ciTemplateOverrideRepository.Update(ciTemplateOverride) @@ -1324,7 +1325,7 @@ func (impl *CiPipelineConfigServiceImpl) PatchCiPipeline(request *bean.CiPatchRe impl.logger.Errorw("err in fetching template for pipeline patch, ", "err", err, "appId", request.AppId) return nil, err } - if request.CiPipeline.PipelineType == bean3.CI_JOB { + if request.CiPipeline.PipelineType == common.CI_JOB { request.CiPipeline.IsDockerConfigOverridden = true request.CiPipeline.DockerConfigOverride = bean.DockerConfigOverride{ DockerRegistry: ciConfig.DockerRegistry, @@ -1517,15 +1518,15 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipelineMin(appId int, envIds []in var ciPipelineResp []*bean.CiPipelineMin for _, pipeline := range pipelines { parentCiPipeline := pipelineConfig.CiPipeline{} - pipelineType := bean3.CI_BUILD + pipelineType := common.CI_BUILD if pipelineParentCiMap[pipeline.Id] != nil { parentCiPipeline = *pipelineParentCiMap[pipeline.Id] - pipelineType = bean3.LINKED + pipelineType = common.LINKED } else if pipeline.IsExternal == true { - pipelineType = bean3.EXTERNAL - } else if pipeline.PipelineType == string(bean3.CI_JOB) { - pipelineType = bean3.CI_JOB + pipelineType = common.EXTERNAL + } else if pipeline.PipelineType == string(common.CI_JOB) { + pipelineType = common.CI_JOB } ciPipeline := &bean.CiPipelineMin{ @@ -1741,7 +1742,7 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipelineByEnvironment(request reso ExternalCiConfig: externalCiConfig, ScanEnabled: pipeline.ScanEnabled, IsDockerConfigOverridden: pipeline.IsDockerConfigOverridden, - PipelineType: bean3.PipelineType(pipeline.PipelineType), + PipelineType: common.PipelineType(pipeline.PipelineType), } parentPipelineAppId, ok := pipelineIdVsAppId[parentCiPipelineId] if ok { diff --git a/pkg/pipeline/BuildPipelineSwitchService.go b/pkg/pipeline/BuildPipelineSwitchService.go index 076d95a12e..858f03473c 100644 --- a/pkg/pipeline/BuildPipelineSwitchService.go +++ b/pkg/pipeline/BuildPipelineSwitchService.go @@ -21,7 +21,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/bean" - pipelineConfigBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" "github.com/devtron-labs/devtron/pkg/build/pipeline/read" "github.com/devtron-labs/devtron/pkg/pipeline/adapter" "github.com/devtron-labs/devtron/pkg/pipeline/history" @@ -124,10 +124,10 @@ func (impl *BuildPipelineSwitchServiceImpl) SwitchToCiPipelineExceptExternal(req } //get the ciPipeline - var switchFromType pipelineConfigBean.PipelineType + var switchFromType common.PipelineType var switchFromPipelineId int if request.SwitchFromExternalCiPipelineId != 0 { - switchFromType = pipelineConfigBean.EXTERNAL + switchFromType = common.EXTERNAL switchFromPipelineId = request.SwitchFromExternalCiPipelineId } else { switchFromPipelineId = request.SwitchFromCiPipelineId @@ -145,9 +145,9 @@ func (impl *BuildPipelineSwitchServiceImpl) SwitchToCiPipelineExceptExternal(req return impl.createNewPipelineAndReplaceOldPipelineLinks(request.CiPipeline, ciConfig, switchFromPipelineId, switchFromType, request.UserId) } -func (impl *BuildPipelineSwitchServiceImpl) createNewPipelineAndReplaceOldPipelineLinks(ciPipelineReq *bean.CiPipeline, ciConfig *bean.CiConfigRequest, switchFromPipelineId int, switchFromType pipelineConfigBean.PipelineType, userId int32) (*bean.CiConfigRequest, error) { +func (impl *BuildPipelineSwitchServiceImpl) createNewPipelineAndReplaceOldPipelineLinks(ciPipelineReq *bean.CiPipeline, ciConfig *bean.CiConfigRequest, switchFromPipelineId int, switchFromType common.PipelineType, userId int32) (*bean.CiConfigRequest, error) { - isSelfLinkedCiPipeline := switchFromType != pipelineConfigBean.EXTERNAL && ciPipelineReq.IsLinkedCi() && ciPipelineReq.ParentCiPipeline == switchFromPipelineId + isSelfLinkedCiPipeline := switchFromType != common.EXTERNAL && ciPipelineReq.IsLinkedCi() && ciPipelineReq.ParentCiPipeline == switchFromPipelineId if isSelfLinkedCiPipeline { errMsg := "cannot create linked ci pipeline from the same source" return nil, util.DefaultApiError().WithInternalMessage(errMsg).WithUserMessage(errMsg).WithHttpStatusCode(http.StatusBadRequest) @@ -208,7 +208,7 @@ func (impl *BuildPipelineSwitchServiceImpl) createNewPipelineAndReplaceOldPipeli // add switchType and remove other id // make constants for error msgs -func (impl *BuildPipelineSwitchServiceImpl) validateCiPipelineSwitch(switchFromCiPipelineId int, switchToType, switchFromType pipelineConfigBean.PipelineType) error { +func (impl *BuildPipelineSwitchServiceImpl) validateCiPipelineSwitch(switchFromCiPipelineId int, switchToType, switchFromType common.PipelineType) error { // this will only allow below conversions // ext -> {ci_job,direct,linked} // direct -> {ci_job,linked} @@ -220,13 +220,13 @@ func (impl *BuildPipelineSwitchServiceImpl) validateCiPipelineSwitch(switchFromC } // refer SwitchToExternalCi - if switchToType == pipelineConfigBean.EXTERNAL { + if switchToType == common.EXTERNAL { return errors.New(string(cannotConvertToExternalCi)) } // we should not check the below logic for external_ci type as builds are not built in devtron and // linked pipelines won't be there as per current external-ci-pipeline architecture - if switchFromCiPipelineId > 0 && switchFromType != pipelineConfigBean.EXTERNAL { + if switchFromCiPipelineId > 0 && switchFromType != common.EXTERNAL { err := impl.validateSwitchPreConditions(switchFromCiPipelineId) if err != nil { return err @@ -252,13 +252,13 @@ func (impl *BuildPipelineSwitchServiceImpl) deleteCiAndItsWorkflowMappings(tx *p return err } -func (impl *BuildPipelineSwitchServiceImpl) deleteOldCiPipelineAndWorkflowMappingBeforeSwitch(tx *pg.Tx, switchFromPipelineId int, switchFromType pipelineConfigBean.PipelineType, userId int32) (*appWorkflow.AppWorkflowMapping, error) { +func (impl *BuildPipelineSwitchServiceImpl) deleteOldCiPipelineAndWorkflowMappingBeforeSwitch(tx *pg.Tx, switchFromPipelineId int, switchFromType common.PipelineType, userId int32) (*appWorkflow.AppWorkflowMapping, error) { // 1) delete build pipelines // 2) delete app workflowMappings var err error pipelineId := switchFromPipelineId pipelineType := "" - if switchFromType == pipelineConfigBean.EXTERNAL { + if switchFromType == common.EXTERNAL { err = impl.deleteExternalCi(tx, switchFromPipelineId, userId) pipelineType = appWorkflow.WEBHOOK } else { diff --git a/pkg/pipeline/CIHandler_ent.go b/pkg/pipeline/CIHandler_ent.go new file mode 100644 index 0000000000..4d41cf80a3 --- /dev/null +++ b/pkg/pipeline/CIHandler_ent.go @@ -0,0 +1,21 @@ +package pipeline + +import ( + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" +) + +type CiHandlerEnt interface { +} + +func (impl *CiHandlerImpl) updateResourceStatusInCache(ciWorkflowId int, podName string, namespace string, status string) { + //do nothing +} + +func (impl *CiHandlerImpl) getPipelineIdForTriggerView(pipeline *pipelineConfig.CiPipeline) (pipelineId int) { + if pipeline.ParentCiPipeline == 0 { + pipelineId = pipeline.Id + } else { + pipelineId = pipeline.ParentCiPipeline + } + return pipelineId +} diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index ba2618db49..7ceacc1c2e 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -17,49 +17,40 @@ package pipeline import ( - "bufio" "errors" "fmt" "github.com/devtron-labs/common-lib/utils" bean4 "github.com/devtron-labs/common-lib/utils/bean" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/adapter/cdWorkflow" - cdWorkflow2 "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" - bean2 "github.com/devtron-labs/devtron/pkg/bean" + pipelineAdapter "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/adapter/cdWorkflow" + cdWorkflowBean "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/pkg/build/artifacts/imageTagging" - "github.com/devtron-labs/devtron/pkg/cluster/adapter" - bean3 "github.com/devtron-labs/devtron/pkg/cluster/bean" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" repository3 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" - "github.com/devtron-labs/devtron/pkg/pipeline/constants" - util2 "github.com/devtron-labs/devtron/pkg/pipeline/util" + eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" bean5 "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus/bean" "github.com/devtron-labs/devtron/pkg/workflow/cd" - "net/http" - "os" - "path/filepath" + "slices" "strconv" "strings" "time" "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" - blob_storage "github.com/devtron-labs/common-lib/blob-storage" "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/auth/user" - "github.com/devtron-labs/devtron/pkg/cluster" pipelineBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/types" resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup" - util3 "github.com/devtron-labs/devtron/util" + globalUtil "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/rbac" "github.com/go-pg/pg" "go.opentelemetry.io/otel" "go.uber.org/zap" - "k8s.io/client-go/rest" ) const ( @@ -69,13 +60,10 @@ const ( ) type CdHandler interface { - UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, string, bool, error) + UpdateWorkflow(workflowStatus eventProcessorBean.CiCdStatus) (int, string, bool, error) GetCdBuildHistory(appId int, environmentId int, pipelineId int, offset int, size int) ([]pipelineBean.CdWorkflowWithArtifact, error) - GetRunningWorkflowLogs(environmentId int, pipelineId int, workflowId int) (*bufio.Reader, func() error, error) FetchCdWorkflowDetails(appId int, environmentId int, pipelineId int, buildId int) (types.WorkflowResponse, error) - DownloadCdWorkflowArtifacts(buildId int) (*os.File, error) FetchCdPrePostStageStatus(pipelineId int) ([]pipelineBean.CdWorkflowWithArtifact, error) - CancelStage(workflowRunnerId int, forceAbort bool, userId int32) (int, error) FetchAppWorkflowStatusForTriggerView(appId int) ([]*pipelineConfig.CdWorkflowStatus, error) FetchAppWorkflowStatusForTriggerViewForEnvironment(request resourceGroup2.ResourceGroupingRequest, token string) ([]*pipelineConfig.CdWorkflowStatus, error) FetchAppDeploymentStatusForEnvironments(request resourceGroup2.ResourceGroupingRequest, token string) ([]*pipelineConfig.AppDeploymentStatus, error) @@ -85,7 +73,6 @@ type CdHandler interface { type CdHandlerImpl struct { Logger *zap.SugaredLogger userService user.UserService - ciLogService CiLogService ciArtifactRepository repository.CiArtifactRepository ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository cdWorkflowRepository pipelineConfig.CdWorkflowRepository @@ -96,10 +83,7 @@ type CdHandlerImpl struct { resourceGroupService resourceGroup2.ResourceGroupService imageTaggingService imageTagging.ImageTaggingService k8sUtil *k8s.K8sServiceImpl - workflowService WorkflowService config *types.CdConfig - clusterService cluster.ClusterService - blobConfigStorageService BlobStorageConfigService customTagService CustomTagService deploymentConfigService common2.DeploymentConfigService workflowStageStatusService workflowStatus.WorkFlowStageStatusService @@ -107,15 +91,14 @@ type CdHandlerImpl struct { } func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, - cdWorkflowRepository pipelineConfig.CdWorkflowRepository, ciLogService CiLogService, + cdWorkflowRepository pipelineConfig.CdWorkflowRepository, ciArtifactRepository repository.CiArtifactRepository, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, pipelineRepository pipelineConfig.PipelineRepository, envRepository repository3.EnvironmentRepository, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, enforcerUtil rbac.EnforcerUtil, resourceGroupService resourceGroup2.ResourceGroupService, imageTaggingService imageTagging.ImageTaggingService, k8sUtil *k8s.K8sServiceImpl, - workflowService WorkflowService, clusterService cluster.ClusterService, - blobConfigStorageService BlobStorageConfigService, customTagService CustomTagService, + customTagService CustomTagService, deploymentConfigService common2.DeploymentConfigService, workflowStageStatusService workflowStatus.WorkFlowStageStatusService, cdWorkflowRunnerService cd.CdWorkflowRunnerService, @@ -123,7 +106,6 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, cdh := &CdHandlerImpl{ Logger: Logger, userService: userService, - ciLogService: ciLogService, cdWorkflowRepository: cdWorkflowRepository, ciArtifactRepository: ciArtifactRepository, ciPipelineMaterialRepository: ciPipelineMaterialRepository, @@ -134,9 +116,6 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, resourceGroupService: resourceGroupService, imageTaggingService: imageTaggingService, k8sUtil: k8sUtil, - workflowService: workflowService, - clusterService: clusterService, - blobConfigStorageService: blobConfigStorageService, customTagService: customTagService, deploymentConfigService: deploymentConfigService, workflowStageStatusService: workflowStageStatusService, @@ -150,177 +129,69 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, return cdh } -func (impl *CdHandlerImpl) CancelStage(workflowRunnerId int, forceAbort bool, userId int32) (int, error) { - workflowRunner, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(workflowRunnerId) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return 0, err - } - pipeline, err := impl.pipelineRepository.FindById(workflowRunner.CdWorkflow.PipelineId) - if err != nil { - impl.Logger.Errorw("error while fetching cd pipeline", "err", err) - return 0, err - } - - env, err := impl.envRepository.FindById(pipeline.EnvironmentId) - if err != nil { - impl.Logger.Errorw("could not fetch stage env", "err", err) - return 0, err - } - - var clusterBean bean3.ClusterBean - if env != nil && env.Cluster != nil { - clusterBean = adapter.GetClusterBean(*env.Cluster) - } - clusterConfig := clusterBean.GetClusterConfig() - var isExtCluster bool - if workflowRunner.WorkflowType == types.PRE { - isExtCluster = pipeline.RunPreStageInEnv - } else if workflowRunner.WorkflowType == types.POST { - isExtCluster = pipeline.RunPostStageInEnv - } - var restConfig *rest.Config - if isExtCluster { - restConfig, err = impl.k8sUtil.GetRestConfigByCluster(clusterConfig) - if err != nil { - impl.Logger.Errorw("error in getting rest config by cluster id", "err", err) - return 0, err - } - } - // Terminate workflow - cancelWfDtoRequest := &types.CancelWfRequestDto{ - ExecutorType: workflowRunner.ExecutorType, - WorkflowName: workflowRunner.Name, - Namespace: workflowRunner.Namespace, - RestConfig: restConfig, - IsExt: isExtCluster, - Environment: nil, - } - err = impl.workflowService.TerminateWorkflow(cancelWfDtoRequest) - if err != nil && forceAbort { - impl.Logger.Errorw("error in terminating workflow, with force abort flag as true", "workflowName", workflowRunner.Name, "err", err) - cancelWfDtoRequest.WorkflowGenerateName = fmt.Sprintf("%d-%s", workflowRunnerId, workflowRunner.Name) - err1 := impl.workflowService.TerminateDanglingWorkflows(cancelWfDtoRequest) - if err1 != nil { - impl.Logger.Errorw("error in terminating dangling workflows", "cancelWfDtoRequest", cancelWfDtoRequest, "err", err) - // ignoring error here in case of force abort, confirmed from product - } - } else if err != nil && strings.Contains(err.Error(), "cannot find workflow") { - return 0, &util.ApiError{Code: "200", HttpStatusCode: http.StatusBadRequest, UserMessage: err.Error()} - } else if err != nil { - impl.Logger.Error("cannot terminate wf runner", "err", err) - return 0, err - } - if forceAbort { - err = impl.handleForceAbortCaseForCdStage(workflowRunner, forceAbort) - if err != nil { - impl.Logger.Errorw("error in handleForceAbortCaseForCdStage", "forceAbortFlag", forceAbort, "workflowRunner", workflowRunner, "err", err) - return 0, err - } - return workflowRunner.Id, nil - } - if len(workflowRunner.ImagePathReservationIds) > 0 { - err := impl.customTagService.DeactivateImagePathReservationByImageIds(workflowRunner.ImagePathReservationIds) - if err != nil { - impl.Logger.Errorw("error in deactivating image path reservation ids", "err", err) - return 0, err - } - } - workflowRunner.Status = cdWorkflow2.WorkflowCancel - workflowRunner.UpdatedOn = time.Now() - workflowRunner.UpdatedBy = userId - err = impl.cdWorkflowRunnerService.UpdateCdWorkflowRunnerWithStage(workflowRunner) - if err != nil { - impl.Logger.Error("cannot update deleted workflow runner status, but wf deleted", "err", err) - return 0, err - } - return workflowRunner.Id, nil -} - -func (impl *CdHandlerImpl) updateWorkflowRunnerForForceAbort(workflowRunner *pipelineConfig.CdWorkflowRunner) error { - workflowRunner.Status = cdWorkflow2.WorkflowCancel - workflowRunner.PodStatus = string(bean2.Failed) - workflowRunner.Message = constants.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE - err := impl.cdWorkflowRunnerService.UpdateCdWorkflowRunnerWithStage(workflowRunner) - if err != nil { - impl.Logger.Errorw("error in updating workflow status in cd workflow runner in force abort case", "err", err) - return err - } - return nil -} - -func (impl *CdHandlerImpl) handleForceAbortCaseForCdStage(workflowRunner *pipelineConfig.CdWorkflowRunner, forceAbort bool) error { - isWorkflowInNonTerminalStage := workflowRunner.Status == string(v1alpha1.NodePending) || workflowRunner.Status == string(v1alpha1.NodeRunning) - if !isWorkflowInNonTerminalStage { - if forceAbort { - return impl.updateWorkflowRunnerForForceAbort(workflowRunner) - } else { - return &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "cannot cancel stage, stage not in progress"} - } - } - //this arises when someone deletes the workflow in resource browser and wants to force abort a cd stage(pre/post) - if workflowRunner.Status == string(v1alpha1.NodeRunning) && forceAbort { - return impl.updateWorkflowRunnerForForceAbort(workflowRunner) - } - return nil -} - -func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, string, bool, error) { - wfStatusRs := impl.extractWorkfowStatus(workflowStatus) +func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus eventProcessorBean.CiCdStatus) (int, string, bool, error) { + wfStatusRs := impl.extractWorkflowStatus(workflowStatus) workflowName, status, podStatus, message, podName := wfStatusRs.WorkflowName, wfStatusRs.Status, wfStatusRs.PodStatus, wfStatusRs.Message, wfStatusRs.PodName - impl.Logger.Debugw("cd update for ", "wf ", workflowName, "status", status) + impl.Logger.Debugw("cd workflow status update event for", "wf ", workflowName, "status", status) if workflowName == "" { return 0, "", false, errors.New("invalid wf name") } workflowId, err := strconv.Atoi(workflowName[:strings.Index(workflowName, "-")]) if err != nil { - impl.Logger.Error("invalid wf status update req", "err", err) + impl.Logger.Errorw("invalid wf status update req", "workflowName", workflowName, "err", err) return 0, "", false, err } - savedWorkflow, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(workflowId) + savedWorkflow, err := impl.cdWorkflowRepository.FindPreOrPostCdWorkflowRunnerById(workflowId) if err != nil { - impl.Logger.Error("cannot get saved wf", "err", err) + impl.Logger.Error("cannot get saved wf", "workflowId", workflowId, "err", err) return 0, "", false, err } cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() cdArtifactLocation := fmt.Sprintf(cdArtifactLocationFormat, savedWorkflow.CdWorkflowId, savedWorkflow.Id) if impl.stateChanged(status, podStatus, message, workflowStatus.FinishedAt.Time, savedWorkflow) { - if savedWorkflow.Status != cdWorkflow2.WorkflowCancel { + if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.PodStatus) { + savedWorkflow.Message = message + if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.Status) { + savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time + } + } else { + impl.Logger.Warnw("cd stage already in terminal state. skipped message and finishedOn from being updated", + "wfId", savedWorkflow.Id, "podStatus", savedWorkflow.PodStatus, "status", savedWorkflow.Status, "message", message, "finishedOn", workflowStatus.FinishedAt.Time) + } + if savedWorkflow.Status != cdWorkflowBean.WorkflowCancel { savedWorkflow.Status = status } savedWorkflow.CdArtifactLocation = cdArtifactLocation savedWorkflow.PodStatus = podStatus - savedWorkflow.Message = message - savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time savedWorkflow.Name = workflowName // removed log location from here since we are saving it at trigger savedWorkflow.PodName = podName savedWorkflow.UpdateAuditLog(1) - impl.Logger.Debugw("updating workflow ", "workflow", savedWorkflow) + impl.Logger.Debugw("updating cd workflow runner", "workflow", savedWorkflow) err = impl.cdWorkflowRunnerService.UpdateCdWorkflowRunnerWithStage(savedWorkflow) if err != nil { - impl.Logger.Error("update wf failed for id " + strconv.Itoa(savedWorkflow.Id)) - return 0, "", true, err + impl.Logger.Errorw("update wf failed for id", "wfId", savedWorkflow.Id, "err", err) + return savedWorkflow.Id, "", true, err } appId := savedWorkflow.CdWorkflow.Pipeline.AppId envId := savedWorkflow.CdWorkflow.Pipeline.EnvironmentId envDeploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(appId, envId) if err != nil { impl.Logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", appId, "envId", envId, "err", err) - return 0, "", true, err + return savedWorkflow.Id, savedWorkflow.Status, true, err } - util3.TriggerCDMetrics(cdWorkflow.GetTriggerMetricsFromRunnerObj(savedWorkflow, envDeploymentConfig), impl.config.ExposeCDMetrics) + globalUtil.TriggerCDMetrics(pipelineAdapter.GetTriggerMetricsFromRunnerObj(savedWorkflow, envDeploymentConfig), impl.config.ExposeCDMetrics) if string(v1alpha1.NodeError) == savedWorkflow.Status || string(v1alpha1.NodeFailed) == savedWorkflow.Status { - impl.Logger.Warnw("cd stage failed for workflow: ", "wfId", savedWorkflow.Id) + impl.Logger.Warnw("cd stage failed for workflow", "wfId", savedWorkflow.Id) } return savedWorkflow.Id, savedWorkflow.Status, true, nil } - return savedWorkflow.Id, savedWorkflow.Status, false, nil + return savedWorkflow.Id, status, false, nil } -func (impl *CdHandlerImpl) extractWorkfowStatus(workflowStatus v1alpha1.WorkflowStatus) *types.WorkflowStatus { +func (impl *CdHandlerImpl) extractWorkflowStatus(workflowStatus eventProcessorBean.CiCdStatus) *types.WorkflowStatus { workflowName := "" status := string(workflowStatus.Phase) podStatus := "Pending" @@ -435,9 +306,9 @@ func (impl *CdHandlerImpl) GetCdBuildHistory(appId int, environmentId int, pipel return cdWorkflowArtifact, err } - var ciMaterialsArr []pipelineConfig.CiPipelineMaterialResponse + var ciMaterialsArr []buildBean.CiPipelineMaterialResponse for _, ciMaterial := range ciMaterials { - res := pipelineConfig.CiPipelineMaterialResponse{ + res := buildBean.CiPipelineMaterialResponse{ Id: ciMaterial.Id, GitMaterialId: ciMaterial.GitMaterialId, GitMaterialName: ciMaterial.GitMaterial.Name[strings.Index(ciMaterial.GitMaterial.Name, "-")+1:], @@ -515,122 +386,6 @@ func (impl *CdHandlerImpl) GetCdBuildHistory(appId int, environmentId int, pipel return cdWorkflowArtifact, nil } -func (impl *CdHandlerImpl) GetRunningWorkflowLogs(environmentId int, pipelineId int, wfrId int) (*bufio.Reader, func() error, error) { - cdWorkflow, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(wfrId) - if err != nil { - impl.Logger.Errorw("error on fetch wf runner", "err", err) - return nil, nil, err - } - - env, err := impl.envRepository.FindById(environmentId) - if err != nil { - impl.Logger.Errorw("could not fetch stage env", "err", err) - return nil, nil, err - } - - pipeline, err := impl.pipelineRepository.FindById(cdWorkflow.CdWorkflow.PipelineId) - if err != nil { - impl.Logger.Errorw("error while fetching cd pipeline", "err", err) - return nil, nil, err - } - var clusterBean bean3.ClusterBean - if env != nil && env.Cluster != nil { - clusterBean = adapter.GetClusterBean(*env.Cluster) - } - clusterConfig := clusterBean.GetClusterConfig() - var isExtCluster bool - if cdWorkflow.WorkflowType == types.PRE { - isExtCluster = pipeline.RunPreStageInEnv - } else if cdWorkflow.WorkflowType == types.POST { - isExtCluster = pipeline.RunPostStageInEnv - } - return impl.getWorkflowLogs(pipelineId, cdWorkflow, clusterConfig, isExtCluster) -} - -func (impl *CdHandlerImpl) getWorkflowLogs(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, runStageInEnv bool) (*bufio.Reader, func() error, error) { - cdLogRequest := types.BuildLogRequest{ - PodName: cdWorkflow.PodName, - Namespace: cdWorkflow.Namespace, - } - - logStream, cleanUp, err := impl.ciLogService.FetchRunningWorkflowLogs(cdLogRequest, clusterConfig, runStageInEnv) - if logStream == nil || err != nil { - if !cdWorkflow.BlobStorageEnabled { - return nil, nil, errors.New("logs-not-stored-in-repository") - } else if string(v1alpha1.NodeSucceeded) == cdWorkflow.Status || string(v1alpha1.NodeError) == cdWorkflow.Status || string(v1alpha1.NodeFailed) == cdWorkflow.Status || cdWorkflow.Status == cdWorkflow2.WorkflowCancel { - impl.Logger.Debugw("pod is not live", "podName", cdWorkflow.PodName, "err", err) - return impl.getLogsFromRepository(pipelineId, cdWorkflow, clusterConfig, runStageInEnv) - } - if err != nil { - impl.Logger.Errorw("err on fetch workflow logs", "err", err) - return nil, nil, err - } else if logStream == nil { - return nil, cleanUp, fmt.Errorf("no logs found for pod %s", cdWorkflow.PodName) - } - } - logReader := bufio.NewReader(logStream) - return logReader, cleanUp, err -} - -func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { - impl.Logger.Debug("getting historic logs", "pipelineId", pipelineId) - - cdConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() // TODO -fixme - cdConfigCdCacheRegion := impl.config.GetDefaultCdLogsBucketRegion() - - cdLogRequest := types.BuildLogRequest{ - PipelineId: cdWorkflow.CdWorkflow.PipelineId, - WorkflowId: cdWorkflow.Id, - PodName: cdWorkflow.PodName, - LogsFilePath: cdWorkflow.LogLocation, // impl.ciCdConfig.CiDefaultBuildLogsKeyPrefix + "/" + cdWorkflow.Name + "/main.log", //TODO - fixme - CloudProvider: impl.config.CloudProvider, - AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ - Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, - AccountName: impl.config.AzureAccountName, - BlobContainerName: impl.config.AzureBlobContainerCiLog, - AccountKey: impl.config.AzureAccountKey, - }, - AwsS3BaseConfig: &blob_storage.AwsS3BaseConfig{ - AccessKey: impl.config.BlobStorageS3AccessKey, - Passkey: impl.config.BlobStorageS3SecretKey, - EndpointUrl: impl.config.BlobStorageS3Endpoint, - IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - BucketName: cdConfigLogsBucket, - Region: cdConfigCdCacheRegion, - VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, - }, - GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ - BucketName: cdConfigLogsBucket, - CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - }, - } - useExternalBlobStorage := isExternalBlobStorageEnabled(isExt, impl.config.UseBlobStorageConfigInCdWorkflow) - if useExternalBlobStorage { - // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - // from them else return. - cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, cdWorkflow.Namespace) - if err != nil { - impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) - return nil, nil, err - } - rq := &cdLogRequest - rq.SetBuildLogRequest(cmConfig, secretConfig) - } - - impl.Logger.Debugw("s3 log req ", "pipelineId", pipelineId, "runnerId", cdWorkflow.Id) - oldLogsStream, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, cdLogRequest) - if err != nil { - impl.Logger.Errorw("err", err) - return nil, nil, err - } - logReader := bufio.NewReader(oldLogsStream) - return logReader, cleanUp, err -} -func isExternalBlobStorageEnabled(isExternalRun bool, useBlobStorageConfigInCdWorkflow bool) bool { - // TODO impl.config.UseBlobStorageConfigInCdWorkflow fetches the live status, we need to check from db as well, we should put useExternalBlobStorage in db - return isExternalRun && !useBlobStorageConfigInCdWorkflow -} - func (impl *CdHandlerImpl) FetchCdWorkflowDetails(appId int, environmentId int, pipelineId int, buildId int) (types.WorkflowResponse, error) { workflowR, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(buildId) if err != nil && err != pg.ErrNoRows { @@ -689,9 +444,9 @@ func (impl *CdHandlerImpl) FetchCdWorkflowDetails(appId int, environmentId int, return types.WorkflowResponse{}, err } - var ciMaterialsArr []pipelineConfig.CiPipelineMaterialResponse + var ciMaterialsArr []buildBean.CiPipelineMaterialResponse for _, m := range ciMaterials { - res := pipelineConfig.CiPipelineMaterialResponse{ + res := buildBean.CiPipelineMaterialResponse{ Id: m.Id, GitMaterialId: m.GitMaterialId, GitMaterialName: m.GitMaterial.Name[strings.Index(m.GitMaterial.Name, "-")+1:], @@ -735,89 +490,6 @@ func (impl *CdHandlerImpl) FetchCdWorkflowDetails(appId int, environmentId int, } -func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(buildId int) (*os.File, error) { - wfr, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(buildId) - if err != nil { - impl.Logger.Errorw("unable to fetch ciWorkflow", "err", err) - return nil, err - } - useExternalBlobStorage := isExternalBlobStorageEnabled(wfr.IsExternalRun(), impl.config.UseBlobStorageConfigInCdWorkflow) - if !wfr.BlobStorageEnabled { - return nil, errors.New("logs-not-stored-in-repository") - } - - cdConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() - cdConfigCdCacheRegion := impl.config.GetDefaultCdLogsBucketRegion() - - item := strconv.Itoa(wfr.Id) - awsS3BaseConfig := &blob_storage.AwsS3BaseConfig{ - AccessKey: impl.config.BlobStorageS3AccessKey, - Passkey: impl.config.BlobStorageS3SecretKey, - EndpointUrl: impl.config.BlobStorageS3Endpoint, - IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - BucketName: cdConfigLogsBucket, - Region: cdConfigCdCacheRegion, - VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, - } - azureBlobBaseConfig := &blob_storage.AzureBlobBaseConfig{ - Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, - AccountKey: impl.config.AzureAccountKey, - AccountName: impl.config.AzureAccountName, - BlobContainerName: impl.config.AzureBlobContainerCiLog, - } - gcpBlobBaseConfig := &blob_storage.GcpBlobBaseConfig{ - BucketName: cdConfigLogsBucket, - CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - } - cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() - key := fmt.Sprintf(cdArtifactLocationFormat, wfr.CdWorkflow.Id, wfr.Id) - if len(wfr.CdArtifactLocation) != 0 && util2.IsValidUrlSubPath(wfr.CdArtifactLocation) { - key = wfr.CdArtifactLocation - } else if util2.IsValidUrlSubPath(key) { - impl.cdWorkflowRepository.MigrateCdArtifactLocation(wfr.Id, key) - } - baseLogLocationPathConfig := impl.config.BaseLogLocationPath - blobStorageService := blob_storage.NewBlobStorageServiceImpl(nil) - destinationKey := filepath.Clean(filepath.Join(baseLogLocationPathConfig, item)) - request := &blob_storage.BlobStorageRequest{ - StorageType: impl.config.CloudProvider, - SourceKey: key, - DestinationKey: destinationKey, - AzureBlobBaseConfig: azureBlobBaseConfig, - AwsS3BaseConfig: awsS3BaseConfig, - GcpBlobBaseConfig: gcpBlobBaseConfig, - } - if useExternalBlobStorage { - clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(wfr.CdWorkflow.Pipeline.Environment.ClusterId) - if err != nil { - impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig", "err", err, "clusterId", wfr.CdWorkflow.Pipeline.Environment.ClusterId) - return nil, err - } - // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - // from them else return. - cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, wfr.Namespace) - if err != nil { - impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) - return nil, err - } - request = updateRequestWithExtClusterCmAndSecret(request, cmConfig, secretConfig) - } - _, numBytes, err := blobStorageService.Get(request) - if err != nil { - impl.Logger.Errorw("error occurred while downloading file", "request", request, "error", err) - return nil, errors.New("failed to download resource") - } - - file, err := os.Open(destinationKey) - if err != nil { - impl.Logger.Errorw("unable to open file", "file", item, "err", err) - return nil, errors.New("unable to open file") - } - - impl.Logger.Infow("Downloaded ", "name", file.Name(), "bytes", numBytes) - return file, nil -} - func (impl *CdHandlerImpl) converterWFR(wfr pipelineConfig.CdWorkflowRunner) pipelineBean.CdWorkflowWithArtifact { workflow := pipelineBean.CdWorkflowWithArtifact{} if wfr.Id > 0 { diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index 3916ec29b2..a617e4635f 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -32,12 +32,12 @@ import ( repository6 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/repository" "github.com/devtron-labs/devtron/pkg/build/pipeline" bean2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" read2 "github.com/devtron-labs/devtron/pkg/chart/read" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/common/read" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" - constants3 "github.com/devtron-labs/devtron/pkg/pipeline/constants" util4 "github.com/devtron-labs/devtron/pkg/pipeline/util" "github.com/devtron-labs/devtron/pkg/plugin" "github.com/devtron-labs/devtron/util/beHelper" @@ -226,9 +226,6 @@ func NewCiCdPipelineOrchestrator( } } -const BEFORE_DOCKER_BUILD string = "BEFORE_DOCKER_BUILD" -const AFTER_DOCKER_BUILD string = "AFTER_DOCKER_BUILD" - func (impl CiCdPipelineOrchestratorImpl) PatchCiMaterialSource(patchRequest *bean.CiMaterialPatchRequest, userId int32) (*bean.CiMaterialPatchRequest, error) { pipeline, err := impl.findUniquePipelineForAppIdAndEnvironmentId(patchRequest.AppId, patchRequest.EnvironmentId) if err != nil { @@ -329,7 +326,7 @@ func (impl CiCdPipelineOrchestratorImpl) validateCiPipelineMaterial(ciPipelineMa func (impl CiCdPipelineOrchestratorImpl) getSkipMessage(ciPipeline *pipelineConfig.CiPipeline) string { switch ciPipeline.PipelineType { - case string(bean2.LINKED_CD): + case string(buildCommonBean.LINKED_CD): return "“Sync with Environment”" default: return "“Linked Build Pipeline”" @@ -1010,7 +1007,7 @@ func (impl CiCdPipelineOrchestratorImpl) CreateCiConf(createRequest *bean.CiConf var pipelineMaterials []*pipelineConfig.CiPipelineMaterial for _, r := range ciPipeline.CiMaterial { - if ciPipeline.PipelineType == bean2.LINKED_CD { + if ciPipeline.PipelineType == buildCommonBean.LINKED_CD { continue } material := &pipelineConfig.CiPipelineMaterial{ @@ -2505,7 +2502,7 @@ func (impl *CiCdPipelineOrchestratorImpl) GetWorkflowCacheConfig(appType helper. if appType == helper.Job { return util4.GetWorkflowCacheConfig(pipelineWorkflowCacheConfig, impl.workflowCacheConfig.IgnoreJob) } else { - if pipelineType == string(constants3.CI_JOB) { + if pipelineType == string(buildCommonBean.CI_JOB) { return util4.GetWorkflowCacheConfigWithBackwardCompatibility(pipelineWorkflowCacheConfig, impl.ciConfig.WorkflowCacheConfig, impl.workflowCacheConfig.IgnoreCIJob, impl.ciConfig.SkipCiJobBuildCachePushPull) } else { return util4.GetWorkflowCacheConfigWithBackwardCompatibility(pipelineWorkflowCacheConfig, impl.ciConfig.WorkflowCacheConfig, impl.workflowCacheConfig.IgnoreCI, impl.ciConfig.IgnoreDockerCacheForCI) diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 5f05b28ef6..ef7f36a198 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -17,86 +17,62 @@ package pipeline import ( - "bufio" "context" "errors" "fmt" "github.com/devtron-labs/common-lib/utils" "github.com/devtron-labs/common-lib/utils/workFlow" - "github.com/devtron-labs/devtron/internal/sql/constants" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" - bean6 "github.com/devtron-labs/devtron/pkg/auth/user/bean" - "github.com/devtron-labs/devtron/pkg/bean/common" + cdWorkflowBean "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/pkg/build/artifacts/imageTagging" - bean4 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" - "github.com/devtron-labs/devtron/pkg/cluster/adapter" - bean5 "github.com/devtron-labs/devtron/pkg/cluster/bean" - "github.com/devtron-labs/devtron/pkg/cluster/environment" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" - constants2 "github.com/devtron-labs/devtron/pkg/pipeline/constants" - util3 "github.com/devtron-labs/devtron/pkg/pipeline/util" + eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/constants" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" - "io/ioutil" - "net/http" - "os" - "path/filepath" "regexp" + "slices" "strconv" "strings" "time" - blob_storage "github.com/devtron-labs/common-lib/blob-storage" - "github.com/devtron-labs/common-lib/utils/k8s" + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" bean2 "github.com/devtron-labs/devtron/api/bean" + client "github.com/devtron-labs/devtron/client/events" "github.com/devtron-labs/devtron/client/gitSensor" + "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/auth/user" - "github.com/devtron-labs/devtron/pkg/cluster" - k8s2 "github.com/devtron-labs/devtron/pkg/k8s" - bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + k8sPkg "github.com/devtron-labs/devtron/pkg/k8s" + pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/types" - resourceGroup "github.com/devtron-labs/devtron/pkg/resourceGroup" + "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/util/rbac" - errors2 "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/client-go/rest" - - "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" - client "github.com/devtron-labs/devtron/client/events" - "github.com/devtron-labs/devtron/internal/sql/repository" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/internal/util" - "github.com/devtron-labs/devtron/pkg/bean" - util2 "github.com/devtron-labs/devtron/util/event" "github.com/go-pg/pg" "go.uber.org/zap" ) type CiHandler interface { - HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) - HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) - CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error - FetchMaterialsByPipelineId(pipelineId int, showAll bool) ([]pipelineConfig.CiPipelineMaterialResponse, error) - FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId int, gitMaterialId int, showAll bool) ([]pipelineConfig.CiPipelineMaterialResponse, error) + // HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) + //HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) + //CheckAndReTriggerCI(workflowStatus eventProcessorBean.CiCdStatus) error + FetchMaterialsByPipelineId(pipelineId int, showAll bool) ([]buildBean.CiPipelineMaterialResponse, error) + FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId int, gitMaterialId int, showAll bool) ([]buildBean.CiPipelineMaterialResponse, error) FetchWorkflowDetails(appId int, pipelineId int, buildId int) (types.WorkflowResponse, error) FetchArtifactsForCiJob(buildId int) (*types.ArtifactsForCiJob, error) - //FetchBuildById(appId int, pipelineId int) (WorkflowResponse, error) - CancelBuild(workflowId int, forceAbort bool) (int, error) - - GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, func() error, error) - GetHistoricBuildLogs(workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) - //SyncWorkflows() error GetBuildHistory(pipelineId int, appId int, offset int, size int) ([]types.WorkflowResponse, error) - DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) - UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, error) + UpdateWorkflow(workflowStatus eventProcessorBean.CiCdStatus) (int, bool, error) FetchCiStatusForTriggerView(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) FetchCiStatusForTriggerViewV1(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) RefreshMaterialByCiPipelineMaterialId(gitMaterialId int) (refreshRes *gitSensor.RefreshGitMaterialResponse, err error) FetchMaterialInfoByArtifactId(ciArtifactId int, envId int) (*types.GitTriggerInfoResponse, error) - UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error + //UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error FetchCiStatusForTriggerViewForEnvironment(request resourceGroup.ResourceGroupingRequest, token string) ([]*pipelineConfig.CiWorkflowStatus, error) + CiHandlerEnt } type CiHandlerImpl struct { @@ -105,15 +81,12 @@ type CiHandlerImpl struct { ciService CiService gitSensorClient gitSensor.Client ciWorkflowRepository pipelineConfig.CiWorkflowRepository - workflowService WorkflowService - ciLogService CiLogService ciArtifactRepository repository.CiArtifactRepository userService user.UserService eventClient client.EventClient eventFactory client.EventFactory ciPipelineRepository pipelineConfig.CiPipelineRepository appListingRepository repository.AppListingRepository - K8sUtil *k8s.K8sServiceImpl cdPipelineRepository pipelineConfig.PipelineRepository enforcerUtil rbac.EnforcerUtil resourceGroupService resourceGroup.ResourceGroupService @@ -122,34 +95,28 @@ type CiHandlerImpl struct { customTagService CustomTagService appWorkflowRepository appWorkflow.AppWorkflowRepository config *types.CiConfig - k8sCommonService k8s2.K8sCommonService - clusterService cluster.ClusterService - blobConfigStorageService BlobStorageConfigService - envService environment.EnvironmentService + k8sCommonService k8sPkg.K8sCommonService workFlowStageStatusService workflowStatus.WorkFlowStageStatusService } -func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, gitSensorClient gitSensor.Client, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, workflowService WorkflowService, - ciLogService CiLogService, ciArtifactRepository repository.CiArtifactRepository, userService user.UserService, eventClient client.EventClient, eventFactory client.EventFactory, ciPipelineRepository pipelineConfig.CiPipelineRepository, - appListingRepository repository.AppListingRepository, K8sUtil *k8s.K8sServiceImpl, cdPipelineRepository pipelineConfig.PipelineRepository, enforcerUtil rbac.EnforcerUtil, resourceGroupService resourceGroup.ResourceGroupService, envRepository repository2.EnvironmentRepository, - imageTaggingService imageTagging.ImageTaggingService, k8sCommonService k8s2.K8sCommonService, clusterService cluster.ClusterService, blobConfigStorageService BlobStorageConfigService, appWorkflowRepository appWorkflow.AppWorkflowRepository, customTagService CustomTagService, - envService environment.EnvironmentService, - workFlowStageStatusService workflowStatus.WorkFlowStageStatusService) *CiHandlerImpl { +func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, gitSensorClient gitSensor.Client, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, + ciArtifactRepository repository.CiArtifactRepository, userService user.UserService, eventClient client.EventClient, eventFactory client.EventFactory, ciPipelineRepository pipelineConfig.CiPipelineRepository, + appListingRepository repository.AppListingRepository, cdPipelineRepository pipelineConfig.PipelineRepository, enforcerUtil rbac.EnforcerUtil, resourceGroupService resourceGroup.ResourceGroupService, envRepository repository2.EnvironmentRepository, + imageTaggingService imageTagging.ImageTaggingService, k8sCommonService k8sPkg.K8sCommonService, appWorkflowRepository appWorkflow.AppWorkflowRepository, customTagService CustomTagService, + workFlowStageStatusService workflowStatus.WorkFlowStageStatusService, +) *CiHandlerImpl { cih := &CiHandlerImpl{ Logger: Logger, ciService: ciService, ciPipelineMaterialRepository: ciPipelineMaterialRepository, gitSensorClient: gitSensorClient, ciWorkflowRepository: ciWorkflowRepository, - workflowService: workflowService, - ciLogService: ciLogService, ciArtifactRepository: ciArtifactRepository, userService: userService, eventClient: eventClient, eventFactory: eventFactory, ciPipelineRepository: ciPipelineRepository, appListingRepository: appListingRepository, - K8sUtil: K8sUtil, cdPipelineRepository: cdPipelineRepository, enforcerUtil: enforcerUtil, resourceGroupService: resourceGroupService, @@ -158,9 +125,6 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline customTagService: customTagService, appWorkflowRepository: appWorkflowRepository, k8sCommonService: k8sCommonService, - clusterService: clusterService, - blobConfigStorageService: blobConfigStorageService, - envService: envService, workFlowStageStatusService: workFlowStageStatusService, } config, err := types.GetCiConfig() @@ -172,173 +136,6 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline return cih } -func (impl *CiHandlerImpl) CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error { - - //return if re-trigger feature is disabled - if !impl.config.WorkflowRetriesEnabled() { - impl.Logger.Debug("CI re-trigger is disabled") - return nil - } - - status, message, ciWorkFlow, err := impl.extractPodStatusAndWorkflow(workflowStatus) - if err != nil { - impl.Logger.Errorw("error in extractPodStatusAndWorkflow", "err", err) - return err - } - - if !executors.CheckIfReTriggerRequired(status, message, ciWorkFlow.Status) { - impl.Logger.Debugw("not re-triggering ci", "status", status, "message", message, "ciWorkflowStatus", ciWorkFlow.Status) - return nil - } - - impl.Logger.Debugw("re-triggering ci", "status", status, "message", message, "ciWorkflowStatus", ciWorkFlow.Status, "ciWorkFlowId", ciWorkFlow.Id) - - retryCount, refCiWorkflow, err := impl.getRefWorkflowAndCiRetryCount(ciWorkFlow) - if err != nil { - impl.Logger.Errorw("error while getting retry count value for a ciWorkflow", "ciWorkFlowId", ciWorkFlow.Id) - return err - } - - err = impl.reTriggerCi(retryCount, refCiWorkflow) - if err != nil { - impl.Logger.Errorw("error in reTriggerCi", "err", err, "status", status, "message", message, "retryCount", retryCount, "ciWorkFlowId", ciWorkFlow.Id) - } - return err -} - -func (impl *CiHandlerImpl) reTriggerCi(retryCount int, refCiWorkflow *pipelineConfig.CiWorkflow) error { - if retryCount >= impl.config.MaxCiWorkflowRetries { - impl.Logger.Infow("maximum retries exhausted for this ciWorkflow", "ciWorkflowId", refCiWorkflow.Id, "retries", retryCount, "configuredRetries", impl.config.MaxCiWorkflowRetries) - return nil - } - impl.Logger.Infow("re-triggering ci for a ci workflow", "ReferenceCiWorkflowId", refCiWorkflow.Id) - ciPipelineMaterialIds := make([]int, 0, len(refCiWorkflow.GitTriggers)) - for id, _ := range refCiWorkflow.GitTriggers { - ciPipelineMaterialIds = append(ciPipelineMaterialIds, id) - } - ciMaterials, err := impl.ciPipelineMaterialRepository.GetByIdsIncludeDeleted(ciPipelineMaterialIds) - if err != nil { - impl.Logger.Errorw("error in getting ci Pipeline Materials using ciPipeline Material Ids", "ciPipelineMaterialIds", ciPipelineMaterialIds, "err", err) - return err - } - - trigger := types.Trigger{} - trigger.BuildTriggerObject(refCiWorkflow, ciMaterials, bean6.SYSTEM_USER_ID, true, nil, "") - _, err = impl.ciService.TriggerCiPipeline(trigger) - - if err != nil { - impl.Logger.Errorw("error occurred in re-triggering ciWorkflow", "triggerDetails", trigger, "err", err) - return err - } - return nil -} - -func (impl *CiHandlerImpl) HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) { - impl.Logger.Debugw("HandleCIManual for pipeline ", "PipelineId", ciTriggerRequest.PipelineId) - commitHashes, runtimeParams, err := impl.buildManualTriggerCommitHashes(ciTriggerRequest) - if err != nil { - return 0, err - } - - ciArtifact, err := impl.ciArtifactRepository.GetLatestArtifactTimeByCiPipelineId(ciTriggerRequest.PipelineId) - if err != nil && err != pg.ErrNoRows { - impl.Logger.Errorw("Error in GetLatestArtifactTimeByCiPipelineId", "err", err, "pipelineId", ciTriggerRequest.PipelineId) - return 0, err - } - - createdOn := time.Time{} - if err != pg.ErrNoRows { - createdOn = ciArtifact.CreatedOn - } - - trigger := types.Trigger{ - PipelineId: ciTriggerRequest.PipelineId, - CommitHashes: commitHashes, - CiMaterials: nil, - TriggeredBy: ciTriggerRequest.TriggeredBy, - InvalidateCache: ciTriggerRequest.InvalidateCache, - RuntimeParameters: runtimeParams, - EnvironmentId: ciTriggerRequest.EnvironmentId, - PipelineType: ciTriggerRequest.PipelineType, - CiArtifactLastFetch: createdOn, - } - id, err := impl.ciService.TriggerCiPipeline(trigger) - - if err != nil { - return 0, err - } - return id, nil -} - -func (impl *CiHandlerImpl) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) { - impl.Logger.Debugw("HandleCIWebhook for material ", "material", gitCiTriggerRequest.CiPipelineMaterial) - ciPipeline, err := impl.GetCiPipeline(gitCiTriggerRequest.CiPipelineMaterial.Id) - if err != nil { - impl.Logger.Errorw("err in getting ci_pipeline by ciPipelineMaterialId", "ciPipelineMaterialId", gitCiTriggerRequest.CiPipelineMaterial.Id, "err", err) - return 0, err - } - if ciPipeline.IsManual || ciPipeline.PipelineType == bean4.LINKED_CD.ToString() { - impl.Logger.Debugw("not handling for manual pipeline or in case of linked cd", "pipelineId", ciPipeline.Id) - return 0, err - } - - ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(ciPipeline.Id) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return 0, err - } - isValidBuildSequence, err := impl.validateBuildSequence(gitCiTriggerRequest, ciPipeline.Id) - if !isValidBuildSequence { - return 0, errors.New("ignoring older build for ciMaterial " + strconv.Itoa(gitCiTriggerRequest.CiPipelineMaterial.Id) + - " commit " + gitCiTriggerRequest.CiPipelineMaterial.GitCommit.Commit) - } - // updating runtime params - runtimeParams := common.NewRuntimeParameters() - for k, v := range gitCiTriggerRequest.ExtraEnvironmentVariables { - runtimeParams = runtimeParams.AddSystemVariable(k, v) - } - commitHashes, err := impl.buildAutomaticTriggerCommitHashes(ciMaterials, gitCiTriggerRequest) - if err != nil { - return 0, err - } - - trigger := types.Trigger{ - PipelineId: ciPipeline.Id, - CommitHashes: commitHashes, - CiMaterials: ciMaterials, - TriggeredBy: gitCiTriggerRequest.TriggeredBy, - RuntimeParameters: runtimeParams, - } - id, err := impl.ciService.TriggerCiPipeline(trigger) - if err != nil { - return 0, err - } - return id, nil -} - -func (impl *CiHandlerImpl) validateBuildSequence(gitCiTriggerRequest bean.GitCiTriggerRequest, pipelineId int) (bool, error) { - isValid := true - lastTriggeredBuild, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflow(pipelineId) - if !(lastTriggeredBuild.Status == string(v1alpha1.NodePending) || lastTriggeredBuild.Status == string(v1alpha1.NodeRunning)) { - return true, nil - } - if err != nil && !util.IsErrNoRows(err) { - impl.Logger.Errorw("cannot get last build for pipeline", "pipelineId", pipelineId) - return false, err - } - - ciPipelineMaterial := gitCiTriggerRequest.CiPipelineMaterial - - if ciPipelineMaterial.Type == string(constants.SOURCE_TYPE_BRANCH_FIXED) { - if ciPipelineMaterial.GitCommit.Date.Before(lastTriggeredBuild.GitTriggers[ciPipelineMaterial.Id].Date) { - impl.Logger.Warnw("older commit cannot be built for pipeline", "pipelineId", pipelineId, "ciMaterial", gitCiTriggerRequest.CiPipelineMaterial.Id) - isValid = false - } - } - - return isValid, nil -} - func (impl *CiHandlerImpl) RefreshMaterialByCiPipelineMaterialId(gitMaterialId int) (refreshRes *gitSensor.RefreshGitMaterialResponse, err error) { impl.Logger.Debugw("refreshing git material", "id", gitMaterialId) refreshRes, err = impl.gitSensorClient.RefreshGitMaterial(context.Background(), @@ -347,12 +144,12 @@ func (impl *CiHandlerImpl) RefreshMaterialByCiPipelineMaterialId(gitMaterialId i return refreshRes, err } -func (impl *CiHandlerImpl) FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId int, gitMaterialId int, showAll bool) ([]pipelineConfig.CiPipelineMaterialResponse, error) { +func (impl *CiHandlerImpl) FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId int, gitMaterialId int, showAll bool) ([]buildBean.CiPipelineMaterialResponse, error) { ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineIdAndGitMaterialId(pipelineId, gitMaterialId) if err != nil { impl.Logger.Errorw("ciMaterials fetch failed", "err", err) } - var ciPipelineMaterialResponses []pipelineConfig.CiPipelineMaterialResponse + var ciPipelineMaterialResponses []buildBean.CiPipelineMaterialResponse var responseMap = make(map[int]bool) ciMaterialHistoryMap := make(map[*pipelineConfig.CiPipelineMaterial]*gitSensor.MaterialChangeResp) @@ -369,13 +166,13 @@ func (impl *CiHandlerImpl) FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId impl.Logger.Debugw("commits for material ", "m", m, "commits: ", changesResp) if apiErr != nil { impl.Logger.Warnw("git sensor FetchChanges failed for material", "id", m.Id) - return []pipelineConfig.CiPipelineMaterialResponse{}, apiErr + return []buildBean.CiPipelineMaterialResponse{}, apiErr } ciMaterialHistoryMap[m] = changesResp } for k, v := range ciMaterialHistoryMap { - r := pipelineConfig.CiPipelineMaterialResponse{ + r := buildBean.CiPipelineMaterialResponse{ Id: k.Id, GitMaterialId: k.GitMaterialId, GitMaterialName: k.GitMaterial.Name[strings.Index(k.GitMaterial.Name, "-")+1:], @@ -398,10 +195,10 @@ func (impl *CiHandlerImpl) FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId regexMaterials, err := impl.ciPipelineMaterialRepository.GetRegexByPipelineId(pipelineId) if err != nil { impl.Logger.Errorw("regex ciMaterials fetch failed", "err", err) - return []pipelineConfig.CiPipelineMaterialResponse{}, err + return []buildBean.CiPipelineMaterialResponse{}, err } for _, k := range regexMaterials { - r := pipelineConfig.CiPipelineMaterialResponse{ + r := buildBean.CiPipelineMaterialResponse{ Id: k.Id, GitMaterialId: k.GitMaterialId, GitMaterialName: k.GitMaterial.Name[strings.Index(k.GitMaterial.Name, "-")+1:], @@ -424,12 +221,12 @@ func (impl *CiHandlerImpl) FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId return ciPipelineMaterialResponses, nil } -func (impl *CiHandlerImpl) FetchMaterialsByPipelineId(pipelineId int, showAll bool) ([]pipelineConfig.CiPipelineMaterialResponse, error) { +func (impl *CiHandlerImpl) FetchMaterialsByPipelineId(pipelineId int, showAll bool) ([]buildBean.CiPipelineMaterialResponse, error) { ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(pipelineId) if err != nil { impl.Logger.Errorw("ciMaterials fetch failed", "err", err) } - var ciPipelineMaterialResponses []pipelineConfig.CiPipelineMaterialResponse + var ciPipelineMaterialResponses []buildBean.CiPipelineMaterialResponse var responseMap = make(map[int]bool) ciMaterialHistoryMap := make(map[*pipelineConfig.CiPipelineMaterial]*gitSensor.MaterialChangeResp) @@ -452,7 +249,7 @@ func (impl *CiHandlerImpl) FetchMaterialsByPipelineId(pipelineId int, showAll bo } for k, v := range ciMaterialHistoryMap { - r := pipelineConfig.CiPipelineMaterialResponse{ + r := buildBean.CiPipelineMaterialResponse{ Id: k.Id, GitMaterialId: k.GitMaterialId, GitMaterialName: k.GitMaterial.Name[strings.Index(k.GitMaterial.Name, "-")+1:], @@ -478,7 +275,7 @@ func (impl *CiHandlerImpl) FetchMaterialsByPipelineId(pipelineId int, showAll bo return nil, err } for _, k := range regexMaterials { - r := pipelineConfig.CiPipelineMaterialResponse{ + r := buildBean.CiPipelineMaterialResponse{ Id: k.Id, GitMaterialId: k.GitMaterialId, GitMaterialName: k.GitMaterial.Name[strings.Index(k.GitMaterial.Name, "-")+1:], @@ -507,9 +304,9 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int if err != nil { impl.Logger.Errorw("ciMaterials fetch failed", "err", err) } - var ciPipelineMaterialResponses []pipelineConfig.CiPipelineMaterialResponse + var ciPipelineMaterialResponses []buildBean.CiPipelineMaterialResponse for _, m := range ciMaterials { - r := pipelineConfig.CiPipelineMaterialResponse{ + r := buildBean.CiPipelineMaterialResponse{ Id: m.Id, GitMaterialId: m.GitMaterialId, Type: string(m.Type), @@ -520,7 +317,7 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int } ciPipelineMaterialResponses = append(ciPipelineMaterialResponses, r) } - //this map contains artifactId -> array of tags of that artifact + // this map contains artifactId -> array of tags of that artifact imageTagsDataMap, err := impl.imageTaggingService.GetTagsDataMapByAppId(appId) if err != nil { impl.Logger.Errorw("error in fetching image tags with appId", "err", err, "appId", appId) @@ -531,6 +328,7 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int impl.Logger.Errorw("err", "err", err) return nil, err } + var workflowIds []int var artifactIds []int for _, w := range workFlows { @@ -544,7 +342,7 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int return nil, err } - //this map contains artifactId -> imageComment of that artifact + // this map contains artifactId -> imageComment of that artifact imageCommetnsDataMap, err := impl.imageTaggingService.GetImageCommentsDataMapByArtifactIds(artifactIds) if err != nil { impl.Logger.Errorw("error in fetching imageCommetnsDataMap", "err", err, "appId", appId, "artifactIds", artifactIds) @@ -586,25 +384,25 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int WorkflowExecutionStage: impl.workFlowStageStatusService.ConvertDBWorkflowStageToMap(allWfStagesDetail, w.Id, w.Status, w.PodStatus, w.Message, bean2.CI_WORKFLOW_TYPE.String(), w.StartedOn, w.FinishedOn), } - if w.Message == bean3.ImageTagUnavailableMessage { - customTag, err := impl.customTagService.GetCustomTagByEntityKeyAndValue(bean3.EntityTypeCiPipelineId, strconv.Itoa(w.CiPipelineId)) + if w.Message == pipelineConfigBean.ImageTagUnavailableMessage { + customTag, err := impl.customTagService.GetCustomTagByEntityKeyAndValue(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(w.CiPipelineId)) if err != nil && err != pg.ErrNoRows { - //err == pg.ErrNoRows should never happen + // err == pg.ErrNoRows should never happen return nil, err } appWorkflows, err := impl.appWorkflowRepository.FindWFCIMappingByCIPipelineId(w.CiPipelineId) if err != nil && err != pg.ErrNoRows { return nil, err } - wfResponse.AppWorkflowId = appWorkflows[0].AppWorkflowId //it is guaranteed there will always be 1 entry (in case of ci_pipeline_id) + wfResponse.AppWorkflowId = appWorkflows[0].AppWorkflowId // it is guaranteed there will always be 1 entry (in case of ci_pipeline_id) wfResponse.CustomTag = &bean2.CustomTagErrorResponse{ TagPattern: customTag.TagPattern, AutoIncreasingNumber: customTag.AutoIncreasingNumber, - Message: bean3.ImageTagUnavailableMessage, + Message: pipelineConfigBean.ImageTagUnavailableMessage, } } if imageTagsDataMap[w.CiArtifactId] != nil { - wfResponse.ImageReleaseTags = imageTagsDataMap[w.CiArtifactId] //if artifact is not yet created,empty list will be sent + wfResponse.ImageReleaseTags = imageTagsDataMap[w.CiArtifactId] // if artifact is not yet created,empty list will be sent } if imageCommetnsDataMap[w.CiArtifactId] != nil { wfResponse.ImageComment = imageCommetnsDataMap[w.CiArtifactId] @@ -614,129 +412,6 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int return ciWorkLowResponses, nil } -func (impl *CiHandlerImpl) CancelBuild(workflowId int, forceAbort bool) (int, error) { - workflow, err := impl.ciWorkflowRepository.FindById(workflowId) - if err != nil { - impl.Logger.Errorw("error in finding ci-workflow by workflow id", "ciWorkflowId", workflowId, "err", err) - return 0, err - } - isExt := workflow.Namespace != constants2.DefaultCiWorkflowNamespace - var env *repository2.Environment - var restConfig *rest.Config - if isExt { - restConfig, err = impl.getRestConfig(workflow) - if err != nil { - return 0, err - } - } - // Terminate workflow - cancelWfDtoRequest := &types.CancelWfRequestDto{ - ExecutorType: workflow.ExecutorType, - WorkflowName: workflow.Name, - Namespace: workflow.Namespace, - RestConfig: restConfig, - IsExt: isExt, - Environment: env, - } - // Terminate workflow - err = impl.workflowService.TerminateWorkflow(cancelWfDtoRequest) - if err != nil && forceAbort { - impl.Logger.Errorw("error in terminating workflow, with force abort flag flag as true", "workflowName", workflow.Name, "err", err) - - cancelWfDtoRequest.WorkflowGenerateName = fmt.Sprintf("%d-%s", workflowId, workflow.Name) - err1 := impl.workflowService.TerminateDanglingWorkflows(cancelWfDtoRequest) - if err1 != nil { - impl.Logger.Errorw("error in terminating dangling workflows", "cancelWfDtoRequest", cancelWfDtoRequest, "err", err) - // ignoring error here in case of force abort, confirmed from product - } - } else if err != nil && strings.Contains(err.Error(), "cannot find workflow") { - return 0, &util.ApiError{Code: "200", HttpStatusCode: http.StatusBadRequest, UserMessage: err.Error()} - } else if err != nil { - impl.Logger.Errorw("cannot terminate wf", "err", err) - return 0, err - } - if forceAbort { - err = impl.handleForceAbortCaseForCi(workflow, forceAbort) - if err != nil { - impl.Logger.Errorw("error in handleForceAbortCaseForCi", "forceAbortFlag", forceAbort, "workflow", workflow, "err", err) - return 0, err - } - return workflow.Id, nil - } - - workflow.Status = cdWorkflow.WorkflowCancel - if workflow.ExecutorType == cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM { - workflow.PodStatus = "Failed" - workflow.Message = constants2.TERMINATE_MESSAGE - } - err = impl.ciService.UpdateCiWorkflowWithStage(workflow) - if err != nil { - impl.Logger.Errorw("cannot update deleted workflow status, but wf deleted", "err", err) - return 0, err - } - imagePathReservationId := workflow.ImagePathReservationId - err = impl.customTagService.DeactivateImagePathReservation(imagePathReservationId) - if err != nil { - impl.Logger.Errorw("error in marking image tag unreserved", "err", err) - return 0, err - } - imagePathReservationIds := workflow.ImagePathReservationIds - if len(imagePathReservationIds) > 0 { - err = impl.customTagService.DeactivateImagePathReservationByImageIds(imagePathReservationIds) - if err != nil { - impl.Logger.Errorw("error in marking image tag unreserved", "err", err) - return 0, err - } - } - return workflow.Id, nil -} - -func (impl *CiHandlerImpl) handleForceAbortCaseForCi(workflow *pipelineConfig.CiWorkflow, forceAbort bool) error { - isWorkflowInNonTerminalStage := workflow.Status == string(v1alpha1.NodePending) || workflow.Status == string(v1alpha1.NodeRunning) - if !isWorkflowInNonTerminalStage { - if forceAbort { - return impl.updateWorkflowForForceAbort(workflow) - } else { - return &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "cannot cancel build, build not in progress"} - } - } - //this arises when someone deletes the workflow in resource browser and wants to force abort a ci - if workflow.Status == string(v1alpha1.NodeRunning) && forceAbort { - return impl.updateWorkflowForForceAbort(workflow) - } - return nil -} - -func (impl *CiHandlerImpl) updateWorkflowForForceAbort(workflow *pipelineConfig.CiWorkflow) error { - workflow.Status = cdWorkflow.WorkflowCancel - workflow.PodStatus = string(bean.Failed) - workflow.Message = constants2.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE - err := impl.ciService.UpdateCiWorkflowWithStage(workflow) - if err != nil { - impl.Logger.Errorw("error in updating workflow status", "err", err) - return err - } - return nil -} - -func (impl *CiHandlerImpl) getRestConfig(workflow *pipelineConfig.CiWorkflow) (*rest.Config, error) { - env, err := impl.envRepository.FindById(workflow.EnvironmentId) - if err != nil { - impl.Logger.Errorw("could not fetch stage env", "err", err) - return nil, err - } - - clusterBean := adapter.GetClusterBean(*env.Cluster) - - clusterConfig := clusterBean.GetClusterConfig() - restConfig, err := impl.K8sUtil.GetRestConfigByCluster(clusterConfig) - if err != nil { - impl.Logger.Errorw("error in getting rest config by cluster id", "err", err) - return nil, err - } - return restConfig, nil -} - func (impl *CiHandlerImpl) FetchWorkflowDetails(appId int, pipelineId int, buildId int) (types.WorkflowResponse, error) { workflow, err := impl.ciWorkflowRepository.FindById(buildId) if err != nil { @@ -766,9 +441,9 @@ func (impl *CiHandlerImpl) FetchWorkflowDetails(appId int, pipelineId int, build return types.WorkflowResponse{}, err } - var ciMaterialsArr []pipelineConfig.CiPipelineMaterialResponse + var ciMaterialsArr []buildBean.CiPipelineMaterialResponse for _, m := range ciMaterials { - res := pipelineConfig.CiPipelineMaterialResponse{ + res := buildBean.CiPipelineMaterialResponse{ Id: m.Id, GitMaterialId: m.GitMaterialId, GitMaterialName: m.GitMaterial.Name[strings.Index(m.GitMaterial.Name, "-")+1:], @@ -794,6 +469,7 @@ func (impl *CiHandlerImpl) FetchWorkflowDetails(appId int, pipelineId int, build impl.ciWorkflowRepository.MigrateIsArtifactUploaded(workflow.Id, ciArtifact.IsArtifactUploaded) isArtifactUploaded = ciArtifact.IsArtifactUploaded } + wfStagesDetail, err := impl.workFlowStageStatusService.GetWorkflowStagesByWorkflowIdsAndWfType([]int{workflow.Id}, bean2.CI_WORKFLOW_TYPE.String()) if err != nil { impl.Logger.Errorw("error in fetching allWfStagesDetail", "err", err, "workflowId", workflow.Id) @@ -811,19 +487,19 @@ func (impl *CiHandlerImpl) FetchWorkflowDetails(appId int, pipelineId int, build CiPipelineId: workflow.CiPipelineId, Namespace: workflow.Namespace, LogLocation: workflow.LogLocation, - BlobStorageEnabled: workflow.BlobStorageEnabled, //TODO default value if value not found in db + BlobStorageEnabled: workflow.BlobStorageEnabled, // TODO default value if value not found in db GitTriggers: workflow.GitTriggers, CiMaterials: ciMaterialsArr, TriggeredBy: workflow.TriggeredBy, TriggeredByEmail: triggeredByUserEmailId, Artifact: ciArtifact.Image, - TargetPlatforms: utils.ConvertTargetPlatformStringToObject(ciArtifact.TargetPlatforms), ArtifactId: ciArtifact.Id, IsArtifactUploaded: isArtifactUploaded, EnvironmentId: workflow.EnvironmentId, EnvironmentName: environmentName, PipelineType: workflow.CiPipeline.PipelineType, PodName: workflow.PodName, + TargetPlatforms: utils.ConvertTargetPlatformStringToObject(ciArtifact.TargetPlatforms), WorkflowExecutionStage: impl.workFlowStageStatusService.ConvertDBWorkflowStageToMap(wfStagesDetail, workflow.Id, workflow.Status, workflow.PodStatus, workflow.Message, bean2.CI_WORKFLOW_TYPE.String(), workflow.StartedOn, workflow.FinishedOn), } return workflowResponse, nil @@ -840,278 +516,8 @@ func (impl *CiHandlerImpl) FetchArtifactsForCiJob(buildId int) (*types.Artifacts } return artifactsResponse, nil } -func (impl *CiHandlerImpl) GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, func() error, error) { - ciWorkflow, err := impl.ciWorkflowRepository.FindById(workflowId) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return nil, nil, err - } - return impl.getWorkflowLogs(ciWorkflow) -} - -func (impl *CiHandlerImpl) getWorkflowLogs(ciWorkflow *pipelineConfig.CiWorkflow) (*bufio.Reader, func() error, error) { - if string(v1alpha1.NodePending) == ciWorkflow.PodStatus { - return bufio.NewReader(strings.NewReader("")), func() error { return nil }, nil - } - ciLogRequest := types.BuildLogRequest{ - PodName: ciWorkflow.PodName, - Namespace: ciWorkflow.Namespace, - } - isExt := false - clusterConfig := &k8s.ClusterConfig{} - if ciWorkflow.EnvironmentId != 0 { - env, err := impl.envRepository.FindById(ciWorkflow.EnvironmentId) - if err != nil { - return nil, nil, err - } - var clusterBean bean5.ClusterBean - if env != nil && env.Cluster != nil { - clusterBean = adapter.GetClusterBean(*env.Cluster) - } - clusterConfig = clusterBean.GetClusterConfig() - isExt = true - } - - logStream, cleanUp, err := impl.ciLogService.FetchRunningWorkflowLogs(ciLogRequest, clusterConfig, isExt) - if logStream == nil || err != nil { - if !ciWorkflow.BlobStorageEnabled { - return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "logs-not-stored-in-repository"} - } else if string(v1alpha1.NodeSucceeded) == ciWorkflow.Status || string(v1alpha1.NodeError) == ciWorkflow.Status || string(v1alpha1.NodeFailed) == ciWorkflow.Status || ciWorkflow.Status == cdWorkflow.WorkflowCancel { - impl.Logger.Debugw("pod is not live", "podName", ciWorkflow.PodName, "err", err) - return impl.getLogsFromRepository(ciWorkflow, clusterConfig, isExt) - } - if err != nil { - impl.Logger.Errorw("err on fetch workflow logs", "err", err) - return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: err.Error()} - } else if logStream == nil { - return nil, cleanUp, fmt.Errorf("no logs found for pod %s", ciWorkflow.PodName) - } - } - logReader := bufio.NewReader(logStream) - return logReader, cleanUp, err -} - -func (impl *CiHandlerImpl) getLogsFromRepository(ciWorkflow *pipelineConfig.CiWorkflow, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { - impl.Logger.Debug("getting historic logs", "ciWorkflowId", ciWorkflow.Id) - ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() - ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion - logsFilePath := impl.config.GetDefaultBuildLogsKeyPrefix() + "/" + ciWorkflow.Name + "/main.log" // this is for backward compatibilty - if strings.Contains(ciWorkflow.LogLocation, "main.log") { - logsFilePath = ciWorkflow.LogLocation - } - ciLogRequest := types.BuildLogRequest{ - PipelineId: ciWorkflow.CiPipelineId, - WorkflowId: ciWorkflow.Id, - PodName: ciWorkflow.PodName, - LogsFilePath: logsFilePath, - CloudProvider: impl.config.CloudProvider, - AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ - Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, - AccountName: impl.config.AzureAccountName, - BlobContainerName: impl.config.AzureBlobContainerCiLog, - AccountKey: impl.config.AzureAccountKey, - }, - AwsS3BaseConfig: &blob_storage.AwsS3BaseConfig{ - AccessKey: impl.config.BlobStorageS3AccessKey, - Passkey: impl.config.BlobStorageS3SecretKey, - EndpointUrl: impl.config.BlobStorageS3Endpoint, - IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - BucketName: ciConfigLogsBucket, - Region: ciConfigCiCacheRegion, - VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, - }, - GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ - BucketName: ciConfigLogsBucket, - CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - }, - } - useExternalBlobStorage := isExternalBlobStorageEnabled(isExt, impl.config.UseBlobStorageConfigInCiWorkflow) - if useExternalBlobStorage { - //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - //from them else return. - cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) - if err != nil { - impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) - return nil, nil, err - } - rq := &ciLogRequest - rq.SetBuildLogRequest(cmConfig, secretConfig) - } - oldLogsStream, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, ciLogRequest) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return nil, nil, err - } - logReader := bufio.NewReader(oldLogsStream) - return logReader, cleanUp, err -} - -func (impl *CiHandlerImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) { - ciWorkflow, err := impl.ciWorkflowRepository.FindById(buildId) - if err != nil { - impl.Logger.Errorw("unable to fetch ciWorkflow", "err", err) - return nil, err - } - useExternalBlobStorage := isExternalBlobStorageEnabled(ciWorkflow.IsExternalRunInJobType(), impl.config.UseBlobStorageConfigInCiWorkflow) - if !ciWorkflow.BlobStorageEnabled { - return nil, errors.New("logs-not-stored-in-repository") - } - - if ciWorkflow.CiPipelineId != pipelineId { - impl.Logger.Error("invalid request, wf not in pipeline") - return nil, errors.New("invalid request, wf not in pipeline") - } - - ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() - item := strconv.Itoa(ciWorkflow.Id) - ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion - azureBlobConfig := &blob_storage.AzureBlobBaseConfig{ - Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, - AccountName: impl.config.AzureAccountName, - BlobContainerName: impl.config.AzureBlobContainerCiLog, - AccountKey: impl.config.AzureAccountKey, - } - awsS3BaseConfig := &blob_storage.AwsS3BaseConfig{ - AccessKey: impl.config.BlobStorageS3AccessKey, - Passkey: impl.config.BlobStorageS3SecretKey, - EndpointUrl: impl.config.BlobStorageS3Endpoint, - IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - BucketName: ciConfigLogsBucket, - Region: ciConfigCiCacheRegion, - VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, - } - gcpBlobBaseConfig := &blob_storage.GcpBlobBaseConfig{ - BucketName: ciConfigLogsBucket, - CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - } - - ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() - key := fmt.Sprintf(ciArtifactLocationFormat, ciWorkflow.Id, ciWorkflow.Id) - if len(ciWorkflow.CiArtifactLocation) != 0 && util3.IsValidUrlSubPath(ciWorkflow.CiArtifactLocation) { - key = ciWorkflow.CiArtifactLocation - } else if util3.IsValidUrlSubPath(key) { - impl.ciWorkflowRepository.MigrateCiArtifactLocation(ciWorkflow.Id, key) - } - baseLogLocationPathConfig := impl.config.BaseLogLocationPath - blobStorageService := blob_storage.NewBlobStorageServiceImpl(nil) - destinationKey := filepath.Clean(filepath.Join(baseLogLocationPathConfig, item)) - request := &blob_storage.BlobStorageRequest{ - StorageType: impl.config.CloudProvider, - SourceKey: key, - DestinationKey: destinationKey, - AzureBlobBaseConfig: azureBlobConfig, - AwsS3BaseConfig: awsS3BaseConfig, - GcpBlobBaseConfig: gcpBlobBaseConfig, - } - if useExternalBlobStorage { - envBean, err := impl.envService.FindById(ciWorkflow.EnvironmentId) - if err != nil { - impl.Logger.Errorw("error in getting envBean by envId", "err", err, "envId", ciWorkflow.EnvironmentId) - return nil, err - } - clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(envBean.ClusterId) - if err != nil { - impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig by clusterId", "err", err, "clusterId", envBean.ClusterId) - return nil, err - } - //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - //from them else return. - cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) - if err != nil { - impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) - return nil, err - } - request = updateRequestWithExtClusterCmAndSecret(request, cmConfig, secretConfig) - } - _, numBytes, err := blobStorageService.Get(request) - if err != nil { - impl.Logger.Errorw("error occurred while downloading file", "request", request, "error", err) - return nil, errors.New("failed to download resource") - } - - file, err := os.Open(destinationKey) - if err != nil { - impl.Logger.Errorw("unable to open file", "file", item, "err", err) - return nil, errors.New("unable to open file") - } - - impl.Logger.Infow("Downloaded ", "filename", file.Name(), "bytes", numBytes) - return file, nil -} - -func (impl *CiHandlerImpl) GetHistoricBuildLogs(workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) { - var err error - if ciWorkflow == nil { - ciWorkflow, err = impl.ciWorkflowRepository.FindById(workflowId) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return nil, err - } - } - ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() - ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion - ciLogRequest := types.BuildLogRequest{ - PipelineId: ciWorkflow.CiPipelineId, - WorkflowId: ciWorkflow.Id, - PodName: ciWorkflow.PodName, - LogsFilePath: ciWorkflow.LogLocation, - CloudProvider: impl.config.CloudProvider, - AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ - Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, - AccountName: impl.config.AzureAccountName, - BlobContainerName: impl.config.AzureBlobContainerCiLog, - AccountKey: impl.config.AzureAccountKey, - }, - AwsS3BaseConfig: &blob_storage.AwsS3BaseConfig{ - AccessKey: impl.config.BlobStorageS3AccessKey, - Passkey: impl.config.BlobStorageS3SecretKey, - EndpointUrl: impl.config.BlobStorageS3Endpoint, - IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - BucketName: ciConfigLogsBucket, - Region: ciConfigCiCacheRegion, - VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, - }, - GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ - BucketName: ciConfigLogsBucket, - CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - }, - } - useExternalBlobStorage := isExternalBlobStorageEnabled(ciWorkflow.IsExternalRunInJobType(), impl.config.UseBlobStorageConfigInCiWorkflow) - if useExternalBlobStorage { - envBean, err := impl.envService.FindById(ciWorkflow.EnvironmentId) - if err != nil { - impl.Logger.Errorw("error in getting envBean by envId", "err", err, "envId", ciWorkflow.EnvironmentId) - return nil, err - } - clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(envBean.ClusterId) - if err != nil { - impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig by clusterId", "err", err, "clusterId", envBean.ClusterId) - return nil, err - } - //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - //from them else return. - cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) - if err != nil { - impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) - return nil, err - } - rq := &ciLogRequest - rq.SetBuildLogRequest(cmConfig, secretConfig) - } - logsFile, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, ciLogRequest) - logs, err := ioutil.ReadFile(logsFile.Name()) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return map[string]string{}, err - } - logStr := string(logs) - resp := make(map[string]string) - resp["logs"] = logStr - defer cleanUp() - return resp, err -} -func ExtractWorkflowStatus(workflowStatus v1alpha1.WorkflowStatus) (string, string, string, string, string, string) { +func ExtractWorkflowStatus(workflowStatus eventProcessorBean.CiCdStatus) (string, string, string, string, string, string) { workflowName := "" status := string(workflowStatus.Phase) podStatus := "" @@ -1119,7 +525,7 @@ func ExtractWorkflowStatus(workflowStatus v1alpha1.WorkflowStatus) (string, stri podName := "" logLocation := "" for k, v := range workflowStatus.Nodes { - if v.TemplateName == bean3.CI_WORKFLOW_NAME { + if v.TemplateName == pipelineConfigBean.CI_WORKFLOW_NAME { if v.BoundaryID == "" { workflowName = k } else { @@ -1141,92 +547,60 @@ func ExtractWorkflowStatus(workflowStatus v1alpha1.WorkflowStatus) (string, stri return workflowName, status, podStatus, message, logLocation, podName } -func (impl *CiHandlerImpl) extractPodStatusAndWorkflow(workflowStatus v1alpha1.WorkflowStatus) (string, string, *pipelineConfig.CiWorkflow, error) { - workflowName, status, _, message, _, _ := ExtractWorkflowStatus(workflowStatus) - if workflowName == "" { - impl.Logger.Errorw("extract workflow status, invalid wf name", "workflowName", workflowName, "status", status, "message", message) - return status, message, nil, errors.New("invalid wf name") - } - workflowId, err := strconv.Atoi(workflowName[:strings.Index(workflowName, "-")]) - if err != nil { - impl.Logger.Errorw("extract workflowId, invalid wf name", "workflowName", workflowName, "err", err) - return status, message, nil, err - } - - savedWorkflow, err := impl.ciWorkflowRepository.FindById(workflowId) - if err != nil { - impl.Logger.Errorw("cannot get saved wf", "workflowId", workflowId, "err", err) - return status, message, nil, err - } - - return status, message, savedWorkflow, err - -} - -func (impl *CiHandlerImpl) getRefWorkflowAndCiRetryCount(savedWorkflow *pipelineConfig.CiWorkflow) (int, *pipelineConfig.CiWorkflow, error) { - var err error - - if savedWorkflow.ReferenceCiWorkflowId != 0 { - savedWorkflow, err = impl.ciWorkflowRepository.FindById(savedWorkflow.ReferenceCiWorkflowId) - } - if err != nil { - impl.Logger.Errorw("cannot get saved wf", "err", err) - return 0, savedWorkflow, err - } - retryCount, err := impl.ciWorkflowRepository.FindRetriedWorkflowCountByReferenceId(savedWorkflow.Id) - return retryCount, savedWorkflow, err -} - -func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, error) { +func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus eventProcessorBean.CiCdStatus) (int, bool, error) { workflowName, status, podStatus, message, _, podName := ExtractWorkflowStatus(workflowStatus) if workflowName == "" { impl.Logger.Errorw("extract workflow status, invalid wf name", "workflowName", workflowName, "status", status, "podStatus", podStatus, "message", message) - return 0, errors.New("invalid wf name") + return 0, false, errors.New("invalid wf name") } workflowId, err := strconv.Atoi(workflowName[:strings.Index(workflowName, "-")]) if err != nil { impl.Logger.Errorw("invalid wf status update req", "err", err) - return 0, err + return 0, false, err } savedWorkflow, err := impl.ciWorkflowRepository.FindById(workflowId) if err != nil { impl.Logger.Errorw("cannot get saved wf", "err", err) - return 0, err + return 0, false, err } - + impl.updateResourceStatusInCache(workflowId, podName, savedWorkflow.Namespace, status) ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() ciArtifactLocation := fmt.Sprintf(ciArtifactLocationFormat, savedWorkflow.Id, savedWorkflow.Id) if impl.stateChanged(status, podStatus, message, workflowStatus.FinishedAt.Time, savedWorkflow) { - if savedWorkflow.Status != cdWorkflow.WorkflowCancel { + if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.PodStatus) { + savedWorkflow.Message = message + if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.Status) { + savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time + } + } else { + impl.Logger.Warnw("cd stage already in terminal state. skipped message and finishedOn from being updated", + "wfId", savedWorkflow.Id, "podStatus", savedWorkflow.PodStatus, "status", savedWorkflow.Status, "message", message, "finishedOn", workflowStatus.FinishedAt.Time) + } + if savedWorkflow.Status != cdWorkflowBean.WorkflowCancel { savedWorkflow.Status = status } savedWorkflow.PodStatus = podStatus - savedWorkflow.Message = message - // NOTE: we are doing this for a quick fix where ci pending message become larger than 250 and in db we had set the charter limit to 250 - if len(message) > 250 { - savedWorkflow.Message = message[:250] - } - if savedWorkflow.ExecutorType == cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM && savedWorkflow.Status == cdWorkflow.WorkflowCancel { + if savedWorkflow.ExecutorType == cdWorkflowBean.WORKFLOW_EXECUTOR_TYPE_SYSTEM && savedWorkflow.Status == cdWorkflowBean.WorkflowCancel { savedWorkflow.PodStatus = "Failed" - savedWorkflow.Message = constants2.TERMINATE_MESSAGE + savedWorkflow.Message = constants.TERMINATE_MESSAGE } - savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time savedWorkflow.Name = workflowName - //savedWorkflow.LogLocation = "/ci-pipeline/" + strconv.Itoa(savedWorkflow.CiPipelineId) + "/workflow/" + strconv.Itoa(savedWorkflow.Id) + "/logs" //TODO need to fetch from workflow object - //savedWorkflow.LogLocation = logLocation // removed because we are saving log location at trigger + // savedWorkflow.LogLocation = "/ci-pipeline/" + strconv.Itoa(savedWorkflow.CiPipelineId) + "/workflow/" + strconv.Itoa(savedWorkflow.Id) + "/logs" //TODO need to fetch from workflow object + // savedWorkflow.LogLocation = logLocation // removed because we are saving log location at trigger savedWorkflow.CiArtifactLocation = ciArtifactLocation savedWorkflow.PodName = podName impl.Logger.Debugw("updating workflow ", "workflow", savedWorkflow) err = impl.ciService.UpdateCiWorkflowWithStage(savedWorkflow) if err != nil { impl.Logger.Error("update wf failed for id " + strconv.Itoa(savedWorkflow.Id)) - return 0, err + return savedWorkflow.Id, true, err } impl.sendCIFailEvent(savedWorkflow, status, message) + return savedWorkflow.Id, true, nil } - return savedWorkflow.Id, nil + return savedWorkflow.Id, false, nil } func (impl *CiHandlerImpl) sendCIFailEvent(savedWorkflow *pipelineConfig.CiWorkflow, status, message string) { @@ -1238,7 +612,7 @@ func (impl *CiHandlerImpl) sendCIFailEvent(savedWorkflow *pipelineConfig.CiWorkf impl.Logger.Warnw("ci failed for workflow: ", "wfId", savedWorkflow.Id) if extractErrorCode(savedWorkflow.Message) != workFlow.CiStageFailErrorCode { - go impl.WriteCIFailEvent(savedWorkflow) + impl.ciService.WriteCIFailEvent(savedWorkflow) } else { impl.Logger.Infof("Step failed notification received for wfID %d with message %s", savedWorkflow.Id, savedWorkflow.Message) } @@ -1257,27 +631,10 @@ func extractErrorCode(msg string) int { return -1 } -func (impl *CiHandlerImpl) WriteCIFailEvent(ciWorkflow *pipelineConfig.CiWorkflow) { - event, _ := impl.eventFactory.Build(util2.Fail, &ciWorkflow.CiPipelineId, ciWorkflow.CiPipeline.AppId, nil, util2.CI) - material := &client.MaterialTriggerInfo{} - material.GitTriggers = ciWorkflow.GitTriggers - event.CiWorkflowRunnerId = ciWorkflow.Id - event.UserId = int(ciWorkflow.TriggeredBy) - event = impl.eventFactory.BuildExtraCIData(event, material) - event.CiArtifactId = 0 - _, evtErr := impl.eventClient.WriteNotificationEvent(event) - if evtErr != nil { - impl.Logger.Errorw("error in writing event", "err", evtErr) - } -} - func (impl *CiHandlerImpl) BuildPayload(ciWorkflow *pipelineConfig.CiWorkflow) *client.Payload { payload := &client.Payload{} payload.AppName = ciWorkflow.CiPipeline.App.AppName payload.PipelineName = ciWorkflow.CiPipeline.Name - //payload["buildName"] = ciWorkflow.Name - //payload["podStatus"] = ciWorkflow.PodStatus - //payload["message"] = ciWorkflow.Message return payload } @@ -1286,191 +643,6 @@ func (impl *CiHandlerImpl) stateChanged(status string, podStatus string, msg str return savedWorkflow.Status != status || savedWorkflow.PodStatus != podStatus || savedWorkflow.Message != msg || savedWorkflow.FinishedOn != finishedAt } -func (impl *CiHandlerImpl) GetCiPipeline(ciMaterialId int) (*pipelineConfig.CiPipeline, error) { - ciMaterial, err := impl.ciPipelineMaterialRepository.GetById(ciMaterialId) - if err != nil { - return nil, err - } - ciPipeline := ciMaterial.CiPipeline - return ciPipeline, nil -} - -func (impl *CiHandlerImpl) buildAutomaticTriggerCommitHashes(ciMaterials []*pipelineConfig.CiPipelineMaterial, request bean.GitCiTriggerRequest) (map[int]pipelineConfig.GitCommit, error) { - commitHashes := map[int]pipelineConfig.GitCommit{} - for _, ciMaterial := range ciMaterials { - if ciMaterial.Id == request.CiPipelineMaterial.Id || len(ciMaterials) == 1 { - request.CiPipelineMaterial.GitCommit = SetGitCommitValuesForBuildingCommitHash(ciMaterial, request.CiPipelineMaterial.GitCommit) - commitHashes[ciMaterial.Id] = request.CiPipelineMaterial.GitCommit - } else { - // this is possible in case of non Webhook, as there would be only one pipeline material per git material in case of PR - lastCommit, err := impl.getLastSeenCommit(ciMaterial.Id) - if err != nil { - return map[int]pipelineConfig.GitCommit{}, err - } - lastCommit = SetGitCommitValuesForBuildingCommitHash(ciMaterial, lastCommit) - commitHashes[ciMaterial.Id] = lastCommit - } - } - return commitHashes, nil -} - -func SetGitCommitValuesForBuildingCommitHash(ciMaterial *pipelineConfig.CiPipelineMaterial, oldGitCommit pipelineConfig.GitCommit) pipelineConfig.GitCommit { - newGitCommit := oldGitCommit - newGitCommit.CiConfigureSourceType = ciMaterial.Type - newGitCommit.CiConfigureSourceValue = ciMaterial.Value - newGitCommit.GitRepoUrl = ciMaterial.GitMaterial.Url - newGitCommit.GitRepoName = ciMaterial.GitMaterial.Name[strings.Index(ciMaterial.GitMaterial.Name, "-")+1:] - return newGitCommit -} - -func (impl *CiHandlerImpl) buildManualTriggerCommitHashes(ciTriggerRequest bean.CiTriggerRequest) (map[int]pipelineConfig.GitCommit, *common.RuntimeParameters, error) { - commitHashes := map[int]pipelineConfig.GitCommit{} - runtimeParams := common.NewRuntimeParameters() - for _, ciPipelineMaterial := range ciTriggerRequest.CiPipelineMaterial { - - pipeLineMaterialFromDb, err := impl.ciPipelineMaterialRepository.GetById(ciPipelineMaterial.Id) - if err != nil { - impl.Logger.Errorw("err in fetching pipeline material by id", "err", err) - return map[int]pipelineConfig.GitCommit{}, nil, err - } - - pipelineType := pipeLineMaterialFromDb.Type - if pipelineType == constants.SOURCE_TYPE_BRANCH_FIXED { - gitCommit, err := impl.BuildManualTriggerCommitHashesForSourceTypeBranchFix(ciPipelineMaterial, pipeLineMaterialFromDb) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return map[int]pipelineConfig.GitCommit{}, nil, err - } - commitHashes[ciPipelineMaterial.Id] = gitCommit - - } else if pipelineType == constants.SOURCE_TYPE_WEBHOOK { - gitCommit, extraEnvVariables, err := impl.BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial, pipeLineMaterialFromDb) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return map[int]pipelineConfig.GitCommit{}, nil, err - } - commitHashes[ciPipelineMaterial.Id] = gitCommit - for key, value := range extraEnvVariables { - runtimeParams = runtimeParams.AddSystemVariable(key, value) - } - } - } - return commitHashes, runtimeParams, nil -} - -func (impl *CiHandlerImpl) BuildManualTriggerCommitHashesForSourceTypeBranchFix(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (pipelineConfig.GitCommit, error) { - commitMetadataRequest := &gitSensor.CommitMetadataRequest{ - PipelineMaterialId: ciPipelineMaterial.Id, - GitHash: ciPipelineMaterial.GitCommit.Commit, - GitTag: ciPipelineMaterial.GitTag, - } - gitCommitResponse, err := impl.gitSensorClient.GetCommitMetadataForPipelineMaterial(context.Background(), commitMetadataRequest) - if err != nil { - impl.Logger.Errorw("err in fetching commit metadata", "commitMetadataRequest", commitMetadataRequest, "err", err) - return pipelineConfig.GitCommit{}, err - } - if gitCommitResponse == nil { - return pipelineConfig.GitCommit{}, errors.New("commit not found") - } - - gitCommit := pipelineConfig.GitCommit{ - Commit: gitCommitResponse.Commit, - Author: gitCommitResponse.Author, - Date: gitCommitResponse.Date, - Message: gitCommitResponse.Message, - Changes: gitCommitResponse.Changes, - GitRepoName: pipeLineMaterialFromDb.GitMaterial.Name[strings.Index(pipeLineMaterialFromDb.GitMaterial.Name, "-")+1:], - GitRepoUrl: pipeLineMaterialFromDb.GitMaterial.Url, - CiConfigureSourceValue: pipeLineMaterialFromDb.Value, - CiConfigureSourceType: pipeLineMaterialFromDb.Type, - } - - return gitCommit, nil -} - -func (impl *CiHandlerImpl) BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (pipelineConfig.GitCommit, map[string]string, error) { - webhookDataInput := ciPipelineMaterial.GitCommit.WebhookData - - // fetch webhook data on the basis of Id - webhookDataRequest := &gitSensor.WebhookDataRequest{ - Id: webhookDataInput.Id, - CiPipelineMaterialId: ciPipelineMaterial.Id, - } - - webhookAndCiData, err := impl.gitSensorClient.GetWebhookData(context.Background(), webhookDataRequest) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return pipelineConfig.GitCommit{}, nil, err - } - webhookData := webhookAndCiData.WebhookData - - // if webhook event is of merged type, then fetch latest commit for target branch - if webhookData.EventActionType == bean.WEBHOOK_EVENT_MERGED_ACTION_TYPE { - - // get target branch name from webhook - targetBranchName := webhookData.Data[bean.WEBHOOK_SELECTOR_TARGET_BRANCH_NAME_NAME] - if targetBranchName == "" { - impl.Logger.Error("target branch not found from webhook data") - return pipelineConfig.GitCommit{}, nil, err - } - - // get latest commit hash for target branch - latestCommitMetadataRequest := &gitSensor.CommitMetadataRequest{ - PipelineMaterialId: ciPipelineMaterial.Id, - BranchName: targetBranchName, - } - - latestCommit, err := impl.gitSensorClient.GetCommitMetadata(context.Background(), latestCommitMetadataRequest) - - if err != nil { - impl.Logger.Errorw("err", "err", err) - return pipelineConfig.GitCommit{}, nil, err - } - - // update webhookData (local) with target latest hash - webhookData.Data[bean.WEBHOOK_SELECTOR_TARGET_CHECKOUT_NAME] = latestCommit.Commit - - } - - // build git commit - gitCommit := pipelineConfig.GitCommit{ - GitRepoName: pipeLineMaterialFromDb.GitMaterial.Name[strings.Index(pipeLineMaterialFromDb.GitMaterial.Name, "-")+1:], - GitRepoUrl: pipeLineMaterialFromDb.GitMaterial.Url, - CiConfigureSourceValue: pipeLineMaterialFromDb.Value, - CiConfigureSourceType: pipeLineMaterialFromDb.Type, - WebhookData: pipelineConfig.WebhookData{ - Id: int(webhookData.Id), - EventActionType: webhookData.EventActionType, - Data: webhookData.Data, - }, - } - - return gitCommit, webhookAndCiData.ExtraEnvironmentVariables, nil -} - -func (impl *CiHandlerImpl) getLastSeenCommit(ciMaterialId int) (pipelineConfig.GitCommit, error) { - var materialIds []int - materialIds = append(materialIds, ciMaterialId) - headReq := &gitSensor.HeadRequest{ - MaterialIds: materialIds, - } - res, err := impl.gitSensorClient.GetHeadForPipelineMaterials(context.Background(), headReq) - if err != nil { - return pipelineConfig.GitCommit{}, err - } - if len(res) == 0 { - return pipelineConfig.GitCommit{}, errors.New("received empty response") - } - gitCommit := pipelineConfig.GitCommit{ - Commit: res[0].GitCommit.Commit, - Author: res[0].GitCommit.Author, - Date: res[0].GitCommit.Date, - Message: res[0].GitCommit.Message, - Changes: res[0].GitCommit.Changes, - } - return gitCommit, nil -} - func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewV1(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) { ciWorkflowStatuses, err := impl.ciWorkflowRepository.FIndCiWorkflowStatusesByAppId(appId) if err != nil && !util.IsErrNoRows(err) { @@ -1490,12 +662,7 @@ func (impl *CiHandlerImpl) FetchCiStatusForTriggerView(appId int) ([]*pipelineCo return ciWorkflowStatuses, err } for _, pipeline := range pipelines { - pipelineId := 0 - if pipeline.ParentCiPipeline == 0 { - pipelineId = pipeline.Id - } else { - pipelineId = pipeline.ParentCiPipeline - } + pipelineId := impl.getPipelineIdForTriggerView(pipeline) workflow, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflow(pipelineId) if err != nil && !util.IsErrNoRows(err) { impl.Logger.Errorw("err", "pipelineId", pipelineId, "err", err) @@ -1540,7 +707,7 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId return &types.GitTriggerInfoResponse{}, err } - ciMaterialsArr := make([]pipelineConfig.CiPipelineMaterialResponse, 0) + ciMaterialsArr := make([]buildBean.CiPipelineMaterialResponse, 0) var triggeredByUserEmailId string //check workflow data only for non external builds if !ciPipeline.IsExternal { @@ -1596,7 +763,7 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId history = append(history, _gitCommit) - res := pipelineConfig.CiPipelineMaterialResponse{ + res := buildBean.CiPipelineMaterialResponse{ Id: m.Id, GitMaterialId: m.GitMaterialId, GitMaterialName: m.GitMaterial.Name[strings.Index(m.GitMaterial.Name, "-")+1:], @@ -1618,6 +785,7 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId //GitTriggers: workflow.GitTriggers, CiMaterials: ciMaterialsArr, TriggeredByEmail: triggeredByUserEmailId, + CiPipelineId: ciPipeline.Id, AppId: ciPipeline.AppId, AppName: deployDetail.AppName, EnvironmentId: deployDetail.EnvironmentId, @@ -1631,127 +799,6 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId return gitTriggerInfoResponse, nil } -func (impl *CiHandlerImpl) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error { - ciWorkflows, err := impl.ciWorkflowRepository.FindByStatusesIn([]string{constants2.Starting, constants2.Running}) - if err != nil { - impl.Logger.Errorw("error on fetching ci workflows", "err", err) - return err - } - client, err := impl.K8sUtil.GetClientForInCluster() - if err != nil { - impl.Logger.Errorw("error while fetching k8s client", "error", err) - return err - } - - for _, ciWorkflow := range ciWorkflows { - var isExt bool - var env *repository2.Environment - var restConfig *rest.Config - if ciWorkflow.Namespace != constants2.DefaultCiWorkflowNamespace { - isExt = true - env, err = impl.envRepository.FindById(ciWorkflow.EnvironmentId) - if err != nil { - impl.Logger.Errorw("could not fetch stage env", "err", err) - return err - } - restConfig, err = impl.getRestConfig(ciWorkflow) - if err != nil { - return err - } - } - - isEligibleToMarkFailed := false - isPodDeleted := false - if time.Since(ciWorkflow.StartedOn) > (time.Minute * time.Duration(timeoutForFailureCiBuild)) { - - //check weather pod is exists or not, if exits check its status - wf, err := impl.workflowService.GetWorkflowStatus(ciWorkflow.ExecutorType, ciWorkflow.Name, ciWorkflow.Namespace, restConfig) - if err != nil { - impl.Logger.Warnw("unable to fetch ci workflow", "err", err) - statusError, ok := err.(*errors2.StatusError) - if ok && statusError.Status().Code == http.StatusNotFound { - impl.Logger.Warnw("ci workflow not found", "err", err) - isEligibleToMarkFailed = true - } else { - continue - // skip this and process for next ci workflow - } - } - - //if ci workflow is exists, check its pod - if !isEligibleToMarkFailed { - ns := constants2.DefaultCiWorkflowNamespace - if isExt { - _, client, err = impl.k8sCommonService.GetCoreClientByClusterId(env.ClusterId) - if err != nil { - impl.Logger.Warnw("error in getting core v1 client using GetCoreClientByClusterId", "err", err, "clusterId", env.Cluster.Id) - continue - } - ns = env.Namespace - } - _, err := impl.K8sUtil.GetPodByName(ns, ciWorkflow.PodName, client) - if err != nil { - impl.Logger.Warnw("unable to fetch ci workflow - pod", "err", err) - statusError, ok := err.(*errors2.StatusError) - if ok && statusError.Status().Code == http.StatusNotFound { - impl.Logger.Warnw("pod not found", "err", err) - isEligibleToMarkFailed = true - } else { - continue - // skip this and process for next ci workflow - } - } - if ciWorkflow.ExecutorType == cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM { - if wf.Status == string(v1alpha1.WorkflowFailed) { - isPodDeleted = true - } - } else { - //check workflow status,get the status - if wf.Status == string(v1alpha1.WorkflowFailed) && wf.Message == constants2.POD_DELETED_MESSAGE { - isPodDeleted = true - } - } - } - } - if isEligibleToMarkFailed { - ciWorkflow.Status = "Failed" - ciWorkflow.PodStatus = "Failed" - if isPodDeleted { - ciWorkflow.Message = cdWorkflow.POD_DELETED_MESSAGE - //error logging handled inside handlePodDeleted - impl.handlePodDeleted(ciWorkflow) - } else { - ciWorkflow.Message = "marked failed by job" - } - err := impl.ciService.UpdateCiWorkflowWithStage(ciWorkflow) - if err != nil { - impl.Logger.Errorw("unable to update ci workflow, its eligible to mark failed", "err", err) - // skip this and process for next ci workflow - } - err = impl.customTagService.DeactivateImagePathReservation(ciWorkflow.ImagePathReservationId) - if err != nil { - impl.Logger.Errorw("unable to update ci workflow, its eligible to mark failed", "err", err) - } - } - } - return nil -} - -func (impl *CiHandlerImpl) handlePodDeleted(ciWorkflow *pipelineConfig.CiWorkflow) { - if !impl.config.WorkflowRetriesEnabled() { - impl.Logger.Debug("ci workflow retry feature disabled") - return - } - retryCount, refCiWorkflow, err := impl.getRefWorkflowAndCiRetryCount(ciWorkflow) - if err != nil { - impl.Logger.Errorw("error in getRefWorkflowAndCiRetryCount", "ciWorkflowId", ciWorkflow.Id, "err", err) - } - impl.Logger.Infow("re-triggering ci by UpdateCiWorkflowStatusFailedCron", "refCiWorkflowId", refCiWorkflow.Id, "ciWorkflow.Status", ciWorkflow.Status, "ciWorkflow.Message", ciWorkflow.Message, "retryCount", retryCount) - err = impl.reTriggerCi(retryCount, refCiWorkflow) - if err != nil { - impl.Logger.Errorw("error in reTriggerCi", "ciWorkflowId", refCiWorkflow.Id, "workflowStatus", ciWorkflow.Status, "ciWorkflowMessage", "ciWorkflow.Message", "retryCount", retryCount, "err", err) - } -} func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewForEnvironment(request resourceGroup.ResourceGroupingRequest, token string) ([]*pipelineConfig.CiWorkflowStatus, error) { ciWorkflowStatuses := make([]*pipelineConfig.CiWorkflowStatus, 0) var cdPipelines []*pipelineConfig.Pipeline @@ -1761,7 +808,7 @@ func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewForEnvironment(request res if err != nil { return nil, err } - //override appIds if already provided app group id in request. + // override appIds if already provided app group id in request. request.ResourceIds = appIds } if len(request.ResourceIds) > 0 { @@ -1794,7 +841,7 @@ func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewForEnvironment(request res if len(ciPipelineIds) == 0 { return ciWorkflowStatuses, nil } - //authorization block starts here + // authorization block starts here var appObjectArr []string objects := impl.enforcerUtil.GetAppObjectByCiPipelineIds(ciPipelineIds) ciPipelineIds = []int{} @@ -1803,17 +850,12 @@ func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewForEnvironment(request res } appResults, _ := request.CheckAuthBatch(token, appObjectArr, []string{}) for _, ciPipeline := range ciPipelines { - appObject := objects[ciPipeline.Id] //here only app permission have to check + appObject := objects[ciPipeline.Id] // here only app permission have to check if !appResults[appObject] { - //if user unauthorized, skip items + // if user unauthorized, skip items continue } - ciPipelineId := 0 - if ciPipeline.ParentCiPipeline == 0 { - ciPipelineId = ciPipeline.Id - } else { - ciPipelineId = ciPipeline.ParentCiPipeline - } + ciPipelineId := impl.getPipelineIdForTriggerView(ciPipeline) ciPipelineIds = append(ciPipelineIds, ciPipelineId) } if len(ciPipelineIds) == 0 { diff --git a/pkg/pipeline/CiLogService.go b/pkg/pipeline/CiLogService.go index 6ad0752f5c..d554917831 100644 --- a/pkg/pipeline/CiLogService.go +++ b/pkg/pipeline/CiLogService.go @@ -36,12 +36,11 @@ type CiLogService interface { type CiLogServiceImpl struct { logger *zap.SugaredLogger - ciService CiService kubeClient *kubernetes.Clientset k8sUtil *k8s.K8sServiceImpl } -func NewCiLogServiceImpl(logger *zap.SugaredLogger, ciService CiService, k8sUtil *k8s.K8sServiceImpl) (*CiLogServiceImpl, error) { +func NewCiLogServiceImpl(logger *zap.SugaredLogger, k8sUtil *k8s.K8sServiceImpl) (*CiLogServiceImpl, error) { _, _, clientSet, err := k8sUtil.GetK8sInClusterConfigAndClients() if err != nil { logger.Errorw("error in getting k8s in cluster client set", "err", err) @@ -49,7 +48,6 @@ func NewCiLogServiceImpl(logger *zap.SugaredLogger, ciService CiService, k8sUtil } return &CiLogServiceImpl{ logger: logger, - ciService: ciService, kubeClient: clientSet, k8sUtil: k8sUtil, }, nil diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 3eb0faf6ee..c88ae5ca11 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -17,152 +17,50 @@ package pipeline import ( - "encoding/json" - "errors" - "fmt" "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" - "github.com/caarlos0/env" - "github.com/devtron-labs/common-lib/utils" - bean3 "github.com/devtron-labs/common-lib/utils/bean" - commonBean "github.com/devtron-labs/common-lib/workflow" bean5 "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/internal/sql/constants" + client "github.com/devtron-labs/devtron/client/events" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" - "github.com/devtron-labs/devtron/pkg/attributes" - bean4 "github.com/devtron-labs/devtron/pkg/attributes/bean" - "github.com/devtron-labs/devtron/pkg/bean/common" - "github.com/devtron-labs/devtron/pkg/build/pipeline" - bean6 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" - repository6 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" - "github.com/devtron-labs/devtron/pkg/pipeline/adapter" - pipelineConst "github.com/devtron-labs/devtron/pkg/pipeline/constants" - "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" - bean2 "github.com/devtron-labs/devtron/pkg/plugin/bean" "github.com/devtron-labs/devtron/pkg/sql" util3 "github.com/devtron-labs/devtron/util" - "github.com/devtron-labs/devtron/util/sliceUtil" - "path" - "path/filepath" - "slices" - "strconv" - "strings" - "time" - - repository5 "github.com/devtron-labs/devtron/internal/sql/repository" - appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" - repository3 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" - "github.com/devtron-labs/devtron/internal/sql/repository/helper" - "github.com/devtron-labs/devtron/pkg/app" - "github.com/devtron-labs/devtron/pkg/auth/user" - pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" - "github.com/devtron-labs/devtron/pkg/pipeline/repository" - "github.com/devtron-labs/devtron/pkg/pipeline/types" - "github.com/devtron-labs/devtron/pkg/plugin" - repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository" - "github.com/devtron-labs/devtron/pkg/resourceQualifiers" - "github.com/devtron-labs/devtron/pkg/variables" - repository4 "github.com/devtron-labs/devtron/pkg/variables/repository" - "github.com/go-pg/pg" - "net/http" - - "github.com/devtron-labs/common-lib/blob-storage" - client "github.com/devtron-labs/devtron/client/events" - "github.com/devtron-labs/devtron/internal/middleware" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/internal/util" - "github.com/devtron-labs/devtron/pkg/bean" util2 "github.com/devtron-labs/devtron/util/event" "go.uber.org/zap" ) type CiService interface { - TriggerCiPipeline(trigger types.Trigger) (int, error) - GetCiMaterials(pipelineId int, ciMaterials []*pipelineConfig.CiPipelineMaterial) ([]*pipelineConfig.CiPipelineMaterial, error) + WriteCITriggerEvent(trigger types.Trigger, pipeline *pipelineConfig.CiPipeline, workflowRequest *types.WorkflowRequest) + WriteCIFailEvent(ciWorkflow *pipelineConfig.CiWorkflow) SaveCiWorkflowWithStage(wf *pipelineConfig.CiWorkflow) error UpdateCiWorkflowWithStage(wf *pipelineConfig.CiWorkflow) error } -type BuildxCacheFlags struct { - BuildxCacheModeMin bool `env:"BUILDX_CACHE_MODE_MIN" envDefault:"false" description:"To set build cache mode to minimum in buildx" ` - AsyncBuildxCacheExport bool `env:"ASYNC_BUILDX_CACHE_EXPORT" envDefault:"false" description:"To enable async container image cache export"` -} type CiServiceImpl struct { - Logger *zap.SugaredLogger - workflowService WorkflowService - ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository - workflowStageStatusService workflowStatus.WorkFlowStageStatusService - eventClient client.EventClient - eventFactory client.EventFactory - ciPipelineRepository pipelineConfig.CiPipelineRepository - ciArtifactRepository repository5.CiArtifactRepository - pipelineStageService PipelineStageService - userService user.UserService - ciTemplateService pipeline.CiTemplateReadService - appCrudOperationService app.AppCrudOperationService - envRepository repository6.EnvironmentRepository - appRepository appRepository.AppRepository - customTagService CustomTagService - config *types.CiConfig - scopedVariableManager variables.ScopedVariableManager - pluginInputVariableParser PluginInputVariableParser - globalPluginService plugin.GlobalPluginService - infraProvider infraProviders.InfraProvider - ciCdPipelineOrchestrator CiCdPipelineOrchestrator - buildxCacheFlags *BuildxCacheFlags - attributeService attributes.AttributesService - ciWorkflowRepository pipelineConfig.CiWorkflowRepository - transactionManager sql.TransactionWrapper + Logger *zap.SugaredLogger + workflowStageStatusService workflowStatus.WorkFlowStageStatusService + eventClient client.EventClient + eventFactory client.EventFactory + config *types.CiConfig + ciWorkflowRepository pipelineConfig.CiWorkflowRepository + transactionManager sql.TransactionWrapper } -func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService, - ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, +func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowStageStatusService workflowStatus.WorkFlowStageStatusService, eventClient client.EventClient, eventFactory client.EventFactory, - ciPipelineRepository pipelineConfig.CiPipelineRepository, - ciArtifactRepository repository5.CiArtifactRepository, - pipelineStageService PipelineStageService, - userService user.UserService, - ciTemplateService pipeline.CiTemplateReadService, appCrudOperationService app.AppCrudOperationService, envRepository repository6.EnvironmentRepository, appRepository appRepository.AppRepository, - scopedVariableManager variables.ScopedVariableManager, - customTagService CustomTagService, - pluginInputVariableParser PluginInputVariableParser, - globalPluginService plugin.GlobalPluginService, - infraProvider infraProviders.InfraProvider, - ciCdPipelineOrchestrator CiCdPipelineOrchestrator, attributeService attributes.AttributesService, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, transactionManager sql.TransactionWrapper, ) *CiServiceImpl { - buildxCacheFlags := &BuildxCacheFlags{} - err := env.Parse(buildxCacheFlags) - if err != nil { - Logger.Infow("error occurred while parsing BuildxCacheFlags env,so setting BuildxCacheModeMin and AsyncBuildxCacheExport to default value", "err", err) - } cis := &CiServiceImpl{ - Logger: Logger, - workflowService: workflowService, - ciPipelineMaterialRepository: ciPipelineMaterialRepository, - workflowStageStatusService: workflowStageStatusService, - eventClient: eventClient, - eventFactory: eventFactory, - ciPipelineRepository: ciPipelineRepository, - ciArtifactRepository: ciArtifactRepository, - pipelineStageService: pipelineStageService, - userService: userService, - ciTemplateService: ciTemplateService, - appCrudOperationService: appCrudOperationService, - envRepository: envRepository, - appRepository: appRepository, - scopedVariableManager: scopedVariableManager, - customTagService: customTagService, - pluginInputVariableParser: pluginInputVariableParser, - globalPluginService: globalPluginService, - infraProvider: infraProvider, - ciCdPipelineOrchestrator: ciCdPipelineOrchestrator, - buildxCacheFlags: buildxCacheFlags, - attributeService: attributeService, - ciWorkflowRepository: ciWorkflowRepository, - transactionManager: transactionManager, + Logger: Logger, + workflowStageStatusService: workflowStageStatusService, + eventClient: eventClient, + eventFactory: eventFactory, + ciWorkflowRepository: ciWorkflowRepository, + transactionManager: transactionManager, } config, err := types.GetCiConfig() if err != nil { @@ -172,325 +70,9 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService return cis } -func (impl *CiServiceImpl) GetCiMaterials(pipelineId int, ciMaterials []*pipelineConfig.CiPipelineMaterial) ([]*pipelineConfig.CiPipelineMaterial, error) { - if !(len(ciMaterials) == 0) { - return ciMaterials, nil - } else { - ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(pipelineId) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return nil, err - } - impl.Logger.Debug("ciMaterials for pipeline trigger ", ciMaterials) - return ciMaterials, nil - } -} - -func (impl *CiServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, workflowRequest *types.WorkflowRequest) error { - // externalCi artifact is meant only for CI_JOB - if trigger.PipelineType != string(bean6.CI_JOB) { - return nil - } - - // checking if user has given run time parameters for externalCiArtifact, if given then sending git material to Ci-Runner - externalCiArtifact, exists := trigger.RuntimeParameters.GetGlobalRuntimeVariables()[pipelineConst.ExtraEnvVarExternalCiArtifactKey] - // validate externalCiArtifact as docker image - if exists { - externalCiArtifact = strings.TrimSpace(externalCiArtifact) - if !strings.Contains(externalCiArtifact, ":") { - if utils.IsValidDockerTagName(externalCiArtifact) { - fullImageUrl, err := utils.BuildDockerImagePath(bean3.DockerRegistryInfo{ - DockerImageTag: externalCiArtifact, - DockerRegistryId: workflowRequest.DockerRegistryId, - DockerRegistryType: workflowRequest.DockerRegistryType, - DockerRegistryURL: workflowRequest.DockerRegistryURL, - DockerRepository: workflowRequest.DockerRepository, - }) - if err != nil { - impl.Logger.Errorw("Error in building docker image", "err", err) - return err - } - externalCiArtifact = fullImageUrl - } else { - impl.Logger.Errorw("validation error", "externalCiArtifact", externalCiArtifact) - return fmt.Errorf("invalid image name or url given in externalCiArtifact") - } - - } - // This will overwrite the existing runtime parameters value for constants.externalCiArtifact - trigger.RuntimeParameters = trigger.RuntimeParameters.AddRuntimeGlobalVariable(pipelineConst.ExtraEnvVarExternalCiArtifactKey, externalCiArtifact) - var artifactExists bool - var err error - - imageDigest, ok := trigger.RuntimeParameters.GetGlobalRuntimeVariables()[pipelineConst.ExtraEnvVarImageDigestKey] - if !ok || len(imageDigest) == 0 { - artifactExists, err = impl.ciArtifactRepository.IfArtifactExistByImage(externalCiArtifact, trigger.PipelineId) - if err != nil { - impl.Logger.Errorw("error in fetching ci artifact", "err", err) - return err - } - if artifactExists { - impl.Logger.Errorw("ci artifact already exists with same image name", "artifact", externalCiArtifact) - return fmt.Errorf("ci artifact already exists with same image name") - } - } else { - artifactExists, err = impl.ciArtifactRepository.IfArtifactExistByImageDigest(imageDigest, externalCiArtifact, trigger.PipelineId) - if err != nil { - impl.Logger.Errorw("error in fetching ci artifact", "err", err, "imageDigest", imageDigest) - return err - } - if artifactExists { - impl.Logger.Errorw("ci artifact already exists with same digest", "artifact", externalCiArtifact) - return fmt.Errorf("ci artifact already exists with same digest") - } - - } - - } - if trigger.PipelineType == string(bean6.CI_JOB) && len(ciMaterials) != 0 && !exists && externalCiArtifact == "" { - ciMaterials[0].GitMaterial = nil - ciMaterials[0].GitMaterialId = 0 - } - return nil -} - -func (impl *CiServiceImpl) markCurrentCiWorkflowFailed(savedCiWf *pipelineConfig.CiWorkflow, validationErr error) error { - // currently such requirement is not there - if savedCiWf == nil { - return nil - } - if savedCiWf.Id != 0 && slices.Contains(cdWorkflow.WfrTerminalStatusList, savedCiWf.Status) { - impl.Logger.Debug("workflow is already in terminal state", "status", savedCiWf.Status, "workflowId", savedCiWf.Id, "message", savedCiWf.Message) - return nil - } - - savedCiWf.Status = cdWorkflow.WorkflowFailed - savedCiWf.Message = validationErr.Error() - savedCiWf.FinishedOn = time.Now() - - var dbErr error - if savedCiWf.Id == 0 { - dbErr = impl.SaveCiWorkflowWithStage(savedCiWf) - } else { - dbErr = impl.UpdateCiWorkflowWithStage(savedCiWf) - } - - if dbErr != nil { - impl.Logger.Errorw("save/update workflow error", "err", dbErr) - return dbErr - } - return nil -} - -func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { - impl.Logger.Debug("ci pipeline manual trigger") - ciMaterials, err := impl.GetCiMaterials(trigger.PipelineId, trigger.CiMaterials) - if err != nil { - return 0, err - } - - ciPipelineScripts, err := impl.ciPipelineRepository.FindCiScriptsByCiPipelineId(trigger.PipelineId) - if err != nil && !util.IsErrNoRows(err) { - return 0, err - } - - var pipeline *pipelineConfig.CiPipeline - for _, m := range ciMaterials { - pipeline = m.CiPipeline - break - } - - scope := resourceQualifiers.Scope{ - AppId: pipeline.App.Id, - } - ciWorkflowConfigNamespace := impl.config.GetDefaultNamespace() - envModal, isJob, err := impl.getEnvironmentForJob(pipeline, trigger) - if err != nil { - return 0, err - } - if isJob && envModal != nil { - ciWorkflowConfigNamespace = envModal.Namespace - - // This will be populated for jobs running in selected environment - scope.EnvId = envModal.Id - scope.ClusterId = envModal.ClusterId - - scope.SystemMetadata = &resourceQualifiers.SystemMetadata{ - EnvironmentName: envModal.Name, - ClusterName: envModal.Cluster.ClusterName, - Namespace: envModal.Namespace, - } - } - if scope.SystemMetadata == nil { - scope.SystemMetadata = &resourceQualifiers.SystemMetadata{ - Namespace: ciWorkflowConfigNamespace, - AppName: pipeline.App.AppName, - } - } - - savedCiWf, err := impl.saveNewWorkflow(pipeline, ciWorkflowConfigNamespace, trigger.CommitHashes, trigger.TriggeredBy, trigger.EnvironmentId, isJob, trigger.ReferenceCiWorkflowId) - if err != nil { - impl.Logger.Errorw("could not save new workflow", "err", err) - return 0, err - } - - // preCiSteps, postCiSteps, refPluginsData, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(pipeline.Id, ciEvent) - request := pipelineConfigBean.NewBuildPrePostStepDataReq(pipeline.Id, pipelineConfigBean.CiStage, scope) - prePostAndRefPluginResponse, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(request) - if err != nil { - impl.Logger.Errorw("error in getting pre steps data for wf request", "err", err, "ciPipelineId", pipeline.Id) - dbErr := impl.markCurrentCiWorkflowFailed(savedCiWf, err) - if dbErr != nil { - impl.Logger.Errorw("saving workflow error", "err", dbErr) - } - return 0, err - } - preCiSteps := prePostAndRefPluginResponse.PreStageSteps - postCiSteps := prePostAndRefPluginResponse.PostStageSteps - refPluginsData := prePostAndRefPluginResponse.RefPluginData - variableSnapshot := prePostAndRefPluginResponse.VariableSnapshot - - if len(preCiSteps) == 0 && isJob { - errMsg := fmt.Sprintf("No tasks are configured in this job pipeline") - validationErr := util.NewApiError(http.StatusNotFound, errMsg, errMsg) - - return 0, validationErr - } - - // get env variables of git trigger data and add it in the extraEnvVariables - gitTriggerEnvVariables, _, err := impl.ciCdPipelineOrchestrator.GetGitCommitEnvVarDataForCICDStage(savedCiWf.GitTriggers) - if err != nil { - impl.Logger.Errorw("error in getting gitTrigger env data for stage", "gitTriggers", savedCiWf.GitTriggers, "err", err) - return 0, err - } - - for k, v := range gitTriggerEnvVariables { - trigger.RuntimeParameters = trigger.RuntimeParameters.AddSystemVariable(k, v) - } - - workflowRequest, err := impl.buildWfRequestForCiPipeline(pipeline, trigger, ciMaterials, savedCiWf, ciWorkflowConfigNamespace, ciPipelineScripts, preCiSteps, postCiSteps, refPluginsData, isJob) - if err != nil { - impl.Logger.Errorw("make workflow req", "err", err) - return 0, err - } - err = impl.handleRuntimeParamsValidations(trigger, ciMaterials, workflowRequest) - if err != nil { - savedCiWf.Status = cdWorkflow.WorkflowAborted - savedCiWf.Message = err.Error() - err1 := impl.UpdateCiWorkflowWithStage(savedCiWf) - if err1 != nil { - impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag") - } - return 0, err - } - - workflowRequest.Scope = scope - workflowRequest.AppId = pipeline.AppId - workflowRequest.BuildxCacheModeMin = impl.buildxCacheFlags.BuildxCacheModeMin - workflowRequest.AsyncBuildxCacheExport = impl.buildxCacheFlags.AsyncBuildxCacheExport - if impl.config != nil && impl.config.BuildxK8sDriverOptions != "" { - err = impl.setBuildxK8sDriverData(workflowRequest) - if err != nil { - impl.Logger.Errorw("error in setBuildxK8sDriverData", "BUILDX_K8S_DRIVER_OPTIONS", impl.config.BuildxK8sDriverOptions, "err", err) - return 0, err - } - } - - // savedCiWf.LogLocation = impl.ciCdConfig.CiDefaultBuildLogsKeyPrefix + "/" + workflowRequest.WorkflowNamePrefix + "/main.log" - savedCiWf.LogLocation = fmt.Sprintf("%s/%s/main.log", impl.config.GetDefaultBuildLogsKeyPrefix(), workflowRequest.WorkflowNamePrefix) - err = impl.updateCiWorkflow(workflowRequest, savedCiWf) - - appLabels, err := impl.appCrudOperationService.GetLabelsByAppId(pipeline.AppId) - if err != nil { - return 0, err - } - workflowRequest.AppLabels = appLabels - workflowRequest.Env = envModal - if isJob { - workflowRequest.Type = pipelineConfigBean.JOB_WORKFLOW_PIPELINE_TYPE - } else { - workflowRequest.Type = pipelineConfigBean.CI_WORKFLOW_PIPELINE_TYPE - } - - workflowRequest.CiPipelineType = trigger.PipelineType - err = impl.executeCiPipeline(workflowRequest) - if err != nil { - impl.Logger.Errorw("error in executing ci pipeline", "err", err) - dbErr := impl.markCurrentCiWorkflowFailed(savedCiWf, err) - if dbErr != nil { - impl.Logger.Errorw("update ci workflow error", "err", dbErr) - } - return 0, err - } - impl.Logger.Debugw("ci triggered", " pipeline ", trigger.PipelineId) - - var variableSnapshotHistories = sliceUtil.GetBeansPtr( - repository4.GetSnapshotBean(savedCiWf.Id, repository4.HistoryReferenceTypeCIWORKFLOW, variableSnapshot)) - if len(variableSnapshotHistories) > 0 { - err = impl.scopedVariableManager.SaveVariableHistoriesForTrigger(variableSnapshotHistories, trigger.TriggeredBy) - if err != nil { - impl.Logger.Errorf("Not able to save variable snapshot for CI trigger %s", err) - } - } - - middleware.CiTriggerCounter.WithLabelValues(pipeline.App.AppName, pipeline.Name).Inc() - go impl.WriteCITriggerEvent(trigger, pipeline, workflowRequest) - return savedCiWf.Id, err -} - -func (impl *CiServiceImpl) setBuildxK8sDriverData(workflowRequest *types.WorkflowRequest) error { - ciBuildConfig := workflowRequest.CiBuildConfig - if ciBuildConfig != nil { - if dockerBuildConfig := ciBuildConfig.DockerBuildConfig; dockerBuildConfig != nil { - k8sDriverOptions, err := impl.getK8sDriverOptions() - if err != nil { - errMsg := "error in parsing BUILDX_K8S_DRIVER_OPTIONS from the devtron-cm, " - err = errors.New(errMsg + "error : " + err.Error()) - impl.Logger.Errorw(errMsg, "err", err) - } - dockerBuildConfig.BuildxK8sDriverOptions = k8sDriverOptions - - } - } - return nil -} - -func (impl *CiServiceImpl) getK8sDriverOptions() ([]map[string]string, error) { - buildxK8sDriverOptions := make([]map[string]string, 0) - err := json.Unmarshal([]byte(impl.config.BuildxK8sDriverOptions), &buildxK8sDriverOptions) - if err != nil { - return nil, err - } else { - return buildxK8sDriverOptions, nil - } -} - -func (impl *CiServiceImpl) getEnvironmentForJob(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger) (*repository6.Environment, bool, error) { - app, err := impl.appRepository.FindById(pipeline.AppId) - if err != nil { - impl.Logger.Errorw("could not find app", "err", err) - return nil, false, err - } - - var env *repository6.Environment - isJob := false - if app.AppType == helper.Job { - isJob = true - if trigger.EnvironmentId != 0 { - env, err = impl.envRepository.FindById(trigger.EnvironmentId) - if err != nil { - impl.Logger.Errorw("could not find environment", "err", err) - return nil, isJob, err - } - return env, isJob, nil - } - } - return nil, isJob, nil -} - func (impl *CiServiceImpl) WriteCITriggerEvent(trigger types.Trigger, pipeline *pipelineConfig.CiPipeline, workflowRequest *types.WorkflowRequest) { event, _ := impl.eventFactory.Build(util2.Trigger, &pipeline.Id, pipeline.AppId, nil, util2.CI) - material := &client.MaterialTriggerInfo{} + material := &buildBean.MaterialTriggerInfo{} material.GitTriggers = trigger.CommitHashes @@ -503,635 +85,17 @@ func (impl *CiServiceImpl) WriteCITriggerEvent(trigger types.Trigger, pipeline * } } -// TODO: Send all trigger data -func (impl *CiServiceImpl) BuildPayload(trigger types.Trigger, pipeline *pipelineConfig.CiPipeline) *client.Payload { - payload := &client.Payload{} - payload.AppName = pipeline.App.AppName - payload.PipelineName = pipeline.Name - return payload -} - -func (impl *CiServiceImpl) saveNewWorkflow(pipeline *pipelineConfig.CiPipeline, ciWorkflowConfigNamespace string, - commitHashes map[int]pipelineConfig.GitCommit, userId int32, EnvironmentId int, isJob bool, refCiWorkflowId int) (wf *pipelineConfig.CiWorkflow, error error) { - - ciWorkflow := &pipelineConfig.CiWorkflow{ - Name: pipeline.Name + "-" + strconv.Itoa(pipeline.Id), - Status: cdWorkflow.WorkflowStarting, // starting CIStage - Message: "", - StartedOn: time.Now(), - CiPipelineId: pipeline.Id, - Namespace: impl.config.GetDefaultNamespace(), - BlobStorageEnabled: impl.config.BlobStorageEnabled, - GitTriggers: commitHashes, - LogLocation: "", - TriggeredBy: userId, - ReferenceCiWorkflowId: refCiWorkflowId, - ExecutorType: impl.config.GetWorkflowExecutorType(), - } - if isJob { - ciWorkflow.Namespace = ciWorkflowConfigNamespace - ciWorkflow.EnvironmentId = EnvironmentId - } - err := impl.SaveCiWorkflowWithStage(ciWorkflow) - if err != nil { - impl.Logger.Errorw("saving workflow error", "err", err) - return &pipelineConfig.CiWorkflow{}, err - } - impl.Logger.Debugw("workflow saved ", "id", ciWorkflow.Id) - return ciWorkflow, nil -} - -func (impl *CiServiceImpl) executeCiPipeline(workflowRequest *types.WorkflowRequest) error { - _, _, err := impl.workflowService.SubmitWorkflow(workflowRequest) - if err != nil { - impl.Logger.Errorw("workflow error", "err", err) - return err - } - return nil -} - -func (impl *CiServiceImpl) buildS3ArtifactLocation(ciWorkflowConfigLogsBucket string, savedWf *pipelineConfig.CiWorkflow) (string, string, string) { - ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() - ArtifactLocation := fmt.Sprintf("s3://"+path.Join(ciWorkflowConfigLogsBucket, ciArtifactLocationFormat), savedWf.Id, savedWf.Id) - artifactFileName := fmt.Sprintf(ciArtifactLocationFormat, savedWf.Id, savedWf.Id) - return ArtifactLocation, ciWorkflowConfigLogsBucket, artifactFileName -} - -func (impl *CiServiceImpl) buildDefaultArtifactLocation(savedWf *pipelineConfig.CiWorkflow) string { - ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() - ArtifactLocation := fmt.Sprintf(ciArtifactLocationFormat, savedWf.Id, savedWf.Id) - return ArtifactLocation -} - -func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, savedWf *pipelineConfig.CiWorkflow, ciWorkflowConfigNamespace string, ciPipelineScripts []*pipelineConfig.CiPipelineScript, preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, refPluginsData []*pipelineConfigBean.RefPluginObject, isJob bool) (*types.WorkflowRequest, error) { - var ciProjectDetails []pipelineConfigBean.CiProjectDetails - commitHashes := trigger.CommitHashes - for _, ciMaterial := range ciMaterials { - // ignore those materials which have inactive git material - if ciMaterial == nil || ciMaterial.GitMaterial == nil || !ciMaterial.GitMaterial.Active { - continue - } - commitHashForPipelineId := commitHashes[ciMaterial.Id] - ciProjectDetail := pipelineConfigBean.CiProjectDetails{ - GitRepository: ciMaterial.GitMaterial.Url, - MaterialName: ciMaterial.GitMaterial.Name, - CheckoutPath: ciMaterial.GitMaterial.CheckoutPath, - FetchSubmodules: ciMaterial.GitMaterial.FetchSubmodules, - CommitHash: commitHashForPipelineId.Commit, - Author: commitHashForPipelineId.Author, - SourceType: ciMaterial.Type, - SourceValue: ciMaterial.Value, - GitTag: ciMaterial.GitTag, - Message: commitHashForPipelineId.Message, - Type: string(ciMaterial.Type), - CommitTime: commitHashForPipelineId.Date.Format(bean.LayoutRFC3339), - GitOptions: pipelineConfigBean.GitOptions{ - UserName: ciMaterial.GitMaterial.GitProvider.UserName, - Password: ciMaterial.GitMaterial.GitProvider.Password, - SshPrivateKey: ciMaterial.GitMaterial.GitProvider.SshPrivateKey, - AccessToken: ciMaterial.GitMaterial.GitProvider.AccessToken, - AuthMode: ciMaterial.GitMaterial.GitProvider.AuthMode, - EnableTLSVerification: ciMaterial.GitMaterial.GitProvider.EnableTLSVerification, - TlsKey: ciMaterial.GitMaterial.GitProvider.TlsKey, - TlsCert: ciMaterial.GitMaterial.GitProvider.TlsCert, - CaCert: ciMaterial.GitMaterial.GitProvider.CaCert, - }, - } - - if ciMaterial.Type == constants.SOURCE_TYPE_WEBHOOK { - webhookData := commitHashForPipelineId.WebhookData - ciProjectDetail.WebhookData = pipelineConfig.WebhookData{ - Id: webhookData.Id, - EventActionType: webhookData.EventActionType, - Data: webhookData.Data, - } - } - - ciProjectDetails = append(ciProjectDetails, ciProjectDetail) - } - - var beforeDockerBuildScripts []*bean.CiScript - var afterDockerBuildScripts []*bean.CiScript - for _, ciPipelineScript := range ciPipelineScripts { - ciTask := &bean.CiScript{ - Id: ciPipelineScript.Id, - Index: ciPipelineScript.Index, - Name: ciPipelineScript.Name, - Script: ciPipelineScript.Script, - OutputLocation: ciPipelineScript.OutputLocation, - } - - if ciPipelineScript.Stage == BEFORE_DOCKER_BUILD { - beforeDockerBuildScripts = append(beforeDockerBuildScripts, ciTask) - } else if ciPipelineScript.Stage == AFTER_DOCKER_BUILD { - afterDockerBuildScripts = append(afterDockerBuildScripts, ciTask) - } - } - - if !(len(beforeDockerBuildScripts) == 0 && len(afterDockerBuildScripts) == 0) { - // found beforeDockerBuildScripts/afterDockerBuildScripts - // building preCiSteps & postCiSteps from them, refPluginsData not needed - preCiSteps = buildCiStepsDataFromDockerBuildScripts(beforeDockerBuildScripts) - postCiSteps = buildCiStepsDataFromDockerBuildScripts(afterDockerBuildScripts) - refPluginsData = []*pipelineConfigBean.RefPluginObject{} - } - host, err := impl.attributeService.GetByKey(bean4.HostUrlKey) - if err != nil { - impl.Logger.Errorw("error in getting host url", "err", err, "hostUrl", host.Value) - return nil, err - } - ciWorkflowConfigCiCacheBucket := impl.config.DefaultCacheBucket - - ciWorkflowConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion - - ciWorkflowConfigCiImage := impl.config.GetDefaultImage() - ciTemplate := pipeline.CiTemplate - ciLevelArgs := pipeline.DockerArgs - - if ciLevelArgs == "" { - ciLevelArgs = "{}" - } - - if pipeline.CiTemplate.DockerBuildOptions == "" { - pipeline.CiTemplate.DockerBuildOptions = "{}" - } - userEmailId, err := impl.userService.GetActiveEmailById(trigger.TriggeredBy) - if err != nil { - impl.Logger.Errorw("unable to find user email by id", "err", err, "id", trigger.TriggeredBy) - return nil, err - } - var dockerfilePath string - var dockerRepository string - var checkoutPath string - var ciBuildConfigBean *bean6.CiBuildConfigBean - dockerRegistry := &repository3.DockerArtifactStore{} - ciBaseBuildConfigEntity := ciTemplate.CiBuildConfig - ciBaseBuildConfigBean, err := adapter.ConvertDbBuildConfigToBean(ciBaseBuildConfigEntity) - if err != nil { - impl.Logger.Errorw("error occurred while converting buildconfig dbEntity to configBean", "ciBuildConfigEntity", ciBaseBuildConfigEntity, "err", err) - return nil, errors.New("error while parsing ci build config") - } - if !pipeline.IsExternal && pipeline.IsDockerConfigOverridden { - templateOverrideBean, err := impl.ciTemplateService.FindTemplateOverrideByCiPipelineId(pipeline.Id) - if err != nil { - return nil, err - } - ciBuildConfigBean = templateOverrideBean.CiBuildConfig - // updating args coming from ciBaseBuildConfigEntity because it is not part of Ci override - if ciBuildConfigBean != nil && ciBuildConfigBean.DockerBuildConfig != nil && ciBaseBuildConfigBean != nil && ciBaseBuildConfigBean.DockerBuildConfig != nil { - ciBuildConfigBean.DockerBuildConfig.Args = ciBaseBuildConfigBean.DockerBuildConfig.Args - } - templateOverride := templateOverrideBean.CiTemplateOverride - checkoutPath = templateOverride.GitMaterial.CheckoutPath - dockerfilePath = templateOverride.DockerfilePath - dockerRepository = templateOverride.DockerRepository - dockerRegistry = templateOverride.DockerRegistry - } else { - checkoutPath = ciTemplate.GitMaterial.CheckoutPath - dockerfilePath = ciTemplate.DockerfilePath - dockerRegistry = ciTemplate.DockerRegistry - dockerRepository = ciTemplate.DockerRepository - ciBuildConfigBean = ciBaseBuildConfigBean - if ciBuildConfigBean != nil { - ciBuildConfigBean.BuildContextGitMaterialId = ciTemplate.BuildContextGitMaterialId - } - - } - if checkoutPath == "" { - checkoutPath = "./" - } - var dockerImageTag string - customTag, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id)) - if err != nil && err != pg.ErrNoRows { - return nil, err - } - if customTag.Id != 0 && customTag.Enabled == true { - imagePathReservation, err := impl.customTagService.GenerateImagePath(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id), dockerRegistry.RegistryURL, dockerRepository) - if err != nil { - if errors.Is(err, pipelineConfigBean.ErrImagePathInUse) { - errMsg := pipelineConfigBean.ImageTagUnavailableMessage - validationErr := util.NewApiError(http.StatusConflict, errMsg, errMsg) - dbErr := impl.markCurrentCiWorkflowFailed(savedWf, validationErr) - if dbErr != nil { - impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag", "err", dbErr, "savedWf", savedWf.Id) - } - return nil, err - } - return nil, err - } - savedWf.ImagePathReservationIds = []int{imagePathReservation.Id} - // imagePath = docker.io/avd0/dashboard:fd23414b - imagePathSplit := strings.Split(imagePathReservation.ImagePath, ":") - if len(imagePathSplit) >= 1 { - dockerImageTag = imagePathSplit[len(imagePathSplit)-1] - } - } else { - dockerImageTag = impl.buildImageTag(commitHashes, pipeline.Id, savedWf.Id) - } - - // copyContainerImage plugin specific logic - var registryCredentialMap map[string]bean2.RegistryCredentials - var pluginArtifactStage string - var imageReservationIds []int - var registryDestinationImageMap map[string][]string - if !isJob { - registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imageReservationIds, err = impl.GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps, postCiSteps, dockerImageTag, customTag.Id, - fmt.Sprintf(pipelineConfigBean.ImagePathPattern, - dockerRegistry.RegistryURL, - dockerRepository, - dockerImageTag), - dockerRegistry.Id) - if err != nil { - impl.Logger.Errorw("error in getting env variables for copyContainerImage plugin") - dbErr := impl.markCurrentCiWorkflowFailed(savedWf, err) - if dbErr != nil { - impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag", "err", dbErr, "savedWf", savedWf.Id) - } - return nil, err - } - - savedWf.ImagePathReservationIds = append(savedWf.ImagePathReservationIds, imageReservationIds...) - } - // mergedArgs := string(merged) - oldArgs := ciTemplate.Args - ciBuildConfigBean, err = adapter.OverrideCiBuildConfig(dockerfilePath, oldArgs, ciLevelArgs, ciTemplate.DockerBuildOptions, ciTemplate.TargetPlatform, ciBuildConfigBean) - if err != nil { - impl.Logger.Errorw("error occurred while overriding ci build config", "oldArgs", oldArgs, "ciLevelArgs", ciLevelArgs, "error", err) - return nil, errors.New("error while parsing ci build config") - } - buildContextCheckoutPath, err := impl.ciPipelineMaterialRepository.GetCheckoutPath(ciBuildConfigBean.BuildContextGitMaterialId) - if err != nil && err != pg.ErrNoRows { - impl.Logger.Errorw("error occurred while getting checkout path from git material", "gitMaterialId", ciBuildConfigBean.BuildContextGitMaterialId, "error", err) - return nil, err - } - if buildContextCheckoutPath == "" { - buildContextCheckoutPath = checkoutPath - } - if ciBuildConfigBean.UseRootBuildContext { - // use root build context i.e '.' - buildContextCheckoutPath = "." - } - - ciBuildConfigBean.PipelineType = trigger.PipelineType - - if ciBuildConfigBean.CiBuildType == bean6.SELF_DOCKERFILE_BUILD_TYPE || ciBuildConfigBean.CiBuildType == bean6.MANAGED_DOCKERFILE_BUILD_TYPE { - ciBuildConfigBean.DockerBuildConfig.BuildContext = filepath.Join(buildContextCheckoutPath, ciBuildConfigBean.DockerBuildConfig.BuildContext) - dockerBuildConfig := ciBuildConfigBean.DockerBuildConfig - dockerfilePath = filepath.Join(checkoutPath, dockerBuildConfig.DockerfilePath) - dockerBuildConfig.DockerfilePath = dockerfilePath - checkoutPath = dockerfilePath[:strings.LastIndex(dockerfilePath, "/")+1] - } else if ciBuildConfigBean.CiBuildType == bean6.BUILDPACK_BUILD_TYPE { - buildPackConfig := ciBuildConfigBean.BuildPackConfig - checkoutPath = filepath.Join(checkoutPath, buildPackConfig.ProjectPath) - } - - defaultTargetPlatform := impl.config.DefaultTargetPlatform - useBuildx := impl.config.UseBuildx - - if ciBuildConfigBean.DockerBuildConfig != nil { - if ciBuildConfigBean.DockerBuildConfig.TargetPlatform == "" && useBuildx { - ciBuildConfigBean.DockerBuildConfig.TargetPlatform = defaultTargetPlatform - ciBuildConfigBean.DockerBuildConfig.UseBuildx = useBuildx - } - ciBuildConfigBean.DockerBuildConfig.BuildxProvenanceMode = impl.config.BuildxProvenanceMode - } - - workflowRequest := &types.WorkflowRequest{ - WorkflowNamePrefix: strconv.Itoa(savedWf.Id) + "-" + savedWf.Name, - PipelineName: pipeline.Name, - PipelineId: pipeline.Id, - CiCacheFileName: pipeline.Name + "-" + strconv.Itoa(pipeline.Id) + ".tar.gz", - CiProjectDetails: ciProjectDetails, - Namespace: ciWorkflowConfigNamespace, - BlobStorageConfigured: savedWf.BlobStorageEnabled, - CiImage: ciWorkflowConfigCiImage, - WorkflowId: savedWf.Id, - TriggeredBy: savedWf.TriggeredBy, - CacheLimit: impl.config.CacheLimit, - ScanEnabled: pipeline.ScanEnabled, - CloudProvider: impl.config.CloudProvider, - DefaultAddressPoolBaseCidr: impl.config.GetDefaultAddressPoolBaseCidr(), - DefaultAddressPoolSize: impl.config.GetDefaultAddressPoolSize(), - PreCiSteps: preCiSteps, - PostCiSteps: postCiSteps, - RefPlugins: refPluginsData, - AppName: pipeline.App.AppName, - TriggerByAuthor: userEmailId, - CiBuildConfig: ciBuildConfigBean, - CiBuildDockerMtuValue: impl.config.CiRunnerDockerMTUValue, - CacheInvalidate: trigger.InvalidateCache, - SystemEnvironmentVariables: trigger.RuntimeParameters.GetSystemVariables(), - EnableBuildContext: impl.config.EnableBuildContext, - OrchestratorHost: impl.config.OrchestratorHost, - HostUrl: host.Value, - OrchestratorToken: impl.config.OrchestratorToken, - ImageRetryCount: impl.config.ImageRetryCount, - ImageRetryInterval: impl.config.ImageRetryInterval, - WorkflowExecutor: impl.config.GetWorkflowExecutorType(), - Type: pipelineConfigBean.CI_WORKFLOW_PIPELINE_TYPE, - CiArtifactLastFetch: trigger.CiArtifactLastFetch, - RegistryDestinationImageMap: registryDestinationImageMap, - RegistryCredentialMap: registryCredentialMap, - PluginArtifactStage: pluginArtifactStage, - ImageScanMaxRetries: impl.config.ImageScanMaxRetries, - ImageScanRetryDelay: impl.config.ImageScanRetryDelay, - UseDockerApiToGetDigest: impl.config.UseDockerApiToGetDigest, - } - workflowRequest.SetAwsInspectorConfig("") - //in oss, there is no pipeline level workflow cache config, so we pass inherit to get the app level config - workflowCacheConfig := impl.ciCdPipelineOrchestrator.GetWorkflowCacheConfig(pipeline.App.AppType, trigger.PipelineType, common.WorkflowCacheConfigInherit) - workflowRequest.IgnoreDockerCachePush = !workflowCacheConfig.Value - workflowRequest.IgnoreDockerCachePull = !workflowCacheConfig.Value - impl.Logger.Debugw("Ignore Cache values", "IgnoreDockerCachePush", workflowRequest.IgnoreDockerCachePush, "IgnoreDockerCachePull", workflowRequest.IgnoreDockerCachePull) - if pipeline.App.AppType == helper.Job { - workflowRequest.AppName = pipeline.App.DisplayName - } - if pipeline.ScanEnabled { - scanToolMetadata, scanVia, err := impl.fetchImageScanExecutionMedium() - if err != nil { - impl.Logger.Errorw("error occurred getting scanned via", "err", err) - return nil, err - } - workflowRequest.SetExecuteImageScanningVia(scanVia) - if scanVia.IsScanMediumExternal() { - imageScanExecutionSteps, refPlugins, err := impl.fetchImageScanExecutionStepsForWfRequest(scanToolMetadata) - if err != nil { - impl.Logger.Errorw("error occurred, fetchImageScanExecutionStepsForWfRequest", "scanToolMetadata", scanToolMetadata, "err", err) - return nil, err - } - workflowRequest.SetImageScanningSteps(imageScanExecutionSteps) - workflowRequest.RefPlugins = append(workflowRequest.RefPlugins, refPlugins...) - } - } - - if dockerRegistry != nil { - workflowRequest.DockerRegistryId = dockerRegistry.Id - workflowRequest.DockerRegistryType = string(dockerRegistry.RegistryType) - workflowRequest.DockerImageTag = dockerImageTag - workflowRequest.DockerRegistryURL = dockerRegistry.RegistryURL - workflowRequest.DockerRepository = dockerRepository - workflowRequest.CheckoutPath = checkoutPath - workflowRequest.DockerUsername = dockerRegistry.Username - workflowRequest.DockerPassword = dockerRegistry.Password - workflowRequest.AwsRegion = dockerRegistry.AWSRegion - workflowRequest.AccessKey = dockerRegistry.AWSAccessKeyId - workflowRequest.SecretKey = dockerRegistry.AWSSecretAccessKey - workflowRequest.DockerConnection = dockerRegistry.Connection - workflowRequest.DockerCert = dockerRegistry.Cert - - } - ciWorkflowConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() - - switch workflowRequest.CloudProvider { - case types.BLOB_STORAGE_S3: - // No AccessKey is used for uploading artifacts, instead IAM based auth is used - workflowRequest.CiCacheRegion = ciWorkflowConfigCiCacheRegion - workflowRequest.CiCacheLocation = ciWorkflowConfigCiCacheBucket - workflowRequest.CiArtifactLocation, workflowRequest.CiArtifactBucket, workflowRequest.CiArtifactFileName = impl.buildS3ArtifactLocation(ciWorkflowConfigLogsBucket, savedWf) - workflowRequest.BlobStorageS3Config = &blob_storage.BlobStorageS3Config{ - AccessKey: impl.config.BlobStorageS3AccessKey, - Passkey: impl.config.BlobStorageS3SecretKey, - EndpointUrl: impl.config.BlobStorageS3Endpoint, - IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - CiCacheBucketName: ciWorkflowConfigCiCacheBucket, - CiCacheRegion: ciWorkflowConfigCiCacheRegion, - CiCacheBucketVersioning: impl.config.BlobStorageS3BucketVersioned, - CiArtifactBucketName: workflowRequest.CiArtifactBucket, - CiArtifactRegion: impl.config.GetDefaultCdLogsBucketRegion(), - CiArtifactBucketVersioning: impl.config.BlobStorageS3BucketVersioned, - CiLogBucketName: impl.config.GetDefaultBuildLogsBucket(), - CiLogRegion: impl.config.GetDefaultCdLogsBucketRegion(), - CiLogBucketVersioning: impl.config.BlobStorageS3BucketVersioned, - } - case types.BLOB_STORAGE_GCP: - workflowRequest.GcpBlobConfig = &blob_storage.GcpBlobConfig{ - CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - CacheBucketName: ciWorkflowConfigCiCacheBucket, - LogBucketName: ciWorkflowConfigLogsBucket, - ArtifactBucketName: ciWorkflowConfigLogsBucket, - } - workflowRequest.CiArtifactLocation = impl.buildDefaultArtifactLocation(savedWf) - workflowRequest.CiArtifactFileName = workflowRequest.CiArtifactLocation - case types.BLOB_STORAGE_AZURE: - workflowRequest.AzureBlobConfig = &blob_storage.AzureBlobConfig{ - Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, - AccountName: impl.config.AzureAccountName, - BlobContainerCiCache: impl.config.AzureBlobContainerCiCache, - AccountKey: impl.config.AzureAccountKey, - BlobContainerCiLog: impl.config.AzureBlobContainerCiLog, - BlobContainerArtifact: impl.config.AzureBlobContainerCiLog, - } - workflowRequest.BlobStorageS3Config = &blob_storage.BlobStorageS3Config{ - EndpointUrl: impl.config.AzureGatewayUrl, - IsInSecure: impl.config.AzureGatewayConnectionInsecure, - CiLogBucketName: impl.config.AzureBlobContainerCiLog, - CiLogRegion: impl.config.DefaultCacheBucketRegion, - CiLogBucketVersioning: impl.config.BlobStorageS3BucketVersioned, - AccessKey: impl.config.AzureAccountName, - } - workflowRequest.CiArtifactLocation = impl.buildDefaultArtifactLocation(savedWf) - workflowRequest.CiArtifactFileName = workflowRequest.CiArtifactLocation - default: - if impl.config.BlobStorageEnabled { - return nil, fmt.Errorf("blob storage %s not supported", workflowRequest.CloudProvider) - } - } - return workflowRequest, nil -} - -func (impl *CiServiceImpl) GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, customTag string, customTagId int, buildImagePath string, buildImagedockerRegistryId string) (map[string][]string, map[string]bean2.RegistryCredentials, string, []int, error) { - - copyContainerImagePluginDetail, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(COPY_CONTAINER_IMAGE) - if err != nil && err != pg.ErrNoRows { - impl.Logger.Errorw("error in getting copyContainerImage plugin id", "err", err) - return nil, nil, "", nil, err - } - - pluginIdToVersionMap := make(map[int]string) - for _, p := range copyContainerImagePluginDetail { - pluginIdToVersionMap[p.Id] = p.Version - } - - for _, step := range preCiSteps { - if _, ok := pluginIdToVersionMap[step.RefPluginId]; ok { - // for copyContainerImage plugin parse destination images and save its data in image path reservation table - return nil, nil, "", nil, errors.New("copyContainerImage plugin not allowed in pre-ci step, please remove it and try again") - } - } - - registryCredentialMap := make(map[string]bean2.RegistryCredentials) - registryDestinationImageMap := make(map[string][]string) - var allDestinationImages []string //saving all images to be reserved in this array - - for _, step := range postCiSteps { - if version, ok := pluginIdToVersionMap[step.RefPluginId]; ok { - destinationImageMap, credentialMap, err := impl.pluginInputVariableParser.HandleCopyContainerImagePluginInputVariables(step.InputVars, customTag, buildImagePath, buildImagedockerRegistryId) - if err != nil { - impl.Logger.Errorw("error in parsing copyContainerImage input variable", "err", err) - return nil, nil, "", nil, err - } - if version == COPY_CONTAINER_IMAGE_VERSION_V1 { - // this is needed in ci runner only for v1 - registryDestinationImageMap = destinationImageMap - } - for _, images := range destinationImageMap { - allDestinationImages = append(allDestinationImages, images...) - } - for k, v := range credentialMap { - registryCredentialMap[k] = v - } - } - } - - pluginArtifactStage := repository5.POST_CI - for _, image := range allDestinationImages { - if image == buildImagePath { - return nil, registryCredentialMap, pluginArtifactStage, nil, - pipelineConfigBean.ErrImagePathInUse - } - } - savedCIArtifacts, err := impl.ciArtifactRepository.FindCiArtifactByImagePaths(allDestinationImages) - if err != nil { - impl.Logger.Errorw("error in fetching artifacts by image path", "err", err) - return nil, nil, pluginArtifactStage, nil, err - } - if len(savedCIArtifacts) > 0 { - // if already present in ci artifact, return "image path already in use error" - return nil, nil, pluginArtifactStage, nil, pipelineConfigBean.ErrImagePathInUse - } - imagePathReservationIds, err := impl.ReserveImagesGeneratedAtPlugin(customTagId, allDestinationImages) - if err != nil { - return nil, nil, pluginArtifactStage, imagePathReservationIds, err - } - return registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imagePathReservationIds, nil -} - -func (impl *CiServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, destinationImages []string) ([]int, error) { - var imagePathReservationIds []int - for _, image := range destinationImages { - imagePathReservationData, err := impl.customTagService.ReserveImagePath(image, customTagId) - if err != nil { - impl.Logger.Errorw("Error in marking custom tag reserved", "err", err) - return imagePathReservationIds, err - } - imagePathReservationIds = append(imagePathReservationIds, imagePathReservationData.Id) - } - return imagePathReservationIds, nil -} - -func buildCiStepsDataFromDockerBuildScripts(dockerBuildScripts []*bean.CiScript) []*pipelineConfigBean.StepObject { - // before plugin support, few variables were set as env vars in ci-runner - // these variables are now moved to global vars in plugin steps, but to avoid error in old scripts adding those variables in payload - inputVars := []*commonBean.VariableObject{ - { - Name: "DOCKER_IMAGE_TAG", - Format: "STRING", - VariableType: commonBean.VariableTypeRefGlobal, - ReferenceVariableName: "DOCKER_IMAGE_TAG", - }, - { - Name: "DOCKER_REPOSITORY", - Format: "STRING", - VariableType: commonBean.VariableTypeRefGlobal, - ReferenceVariableName: "DOCKER_REPOSITORY", - }, - { - Name: "DOCKER_REGISTRY_URL", - Format: "STRING", - VariableType: commonBean.VariableTypeRefGlobal, - ReferenceVariableName: "DOCKER_REGISTRY_URL", - }, - { - Name: "DOCKER_IMAGE", - Format: "STRING", - VariableType: commonBean.VariableTypeRefGlobal, - ReferenceVariableName: "DOCKER_IMAGE", - }, - } - var ciSteps []*pipelineConfigBean.StepObject - for _, dockerBuildScript := range dockerBuildScripts { - ciStep := &pipelineConfigBean.StepObject{ - Name: dockerBuildScript.Name, - Index: dockerBuildScript.Index, - Script: dockerBuildScript.Script, - ArtifactPaths: []string{dockerBuildScript.OutputLocation}, - StepType: string(repository.PIPELINE_STEP_TYPE_INLINE), - ExecutorType: string(repository2.SCRIPT_TYPE_SHELL), - InputVars: inputVars, - } - ciSteps = append(ciSteps, ciStep) - } - return ciSteps -} - -func (impl *CiServiceImpl) buildImageTag(commitHashes map[int]pipelineConfig.GitCommit, id int, wfId int) string { - dockerImageTag := "" - toAppendDevtronParamInTag := true - for _, v := range commitHashes { - if v.WebhookData.Id == 0 { - if v.Commit == "" { - continue - } - dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _getTruncatedImageTag(v.Commit)) - } else { - _targetCheckout := v.WebhookData.Data[bean.WEBHOOK_SELECTOR_TARGET_CHECKOUT_NAME] - if _targetCheckout == "" { - continue - } - // if not PR based then meaning tag based - isPRBasedEvent := v.WebhookData.EventActionType == bean.WEBHOOK_EVENT_MERGED_ACTION_TYPE - if !isPRBasedEvent && impl.config.CiCdConfig.UseImageTagFromGitProviderForTagBasedBuild { - dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _targetCheckout) - } else { - dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _getTruncatedImageTag(_targetCheckout)) - } - if isPRBasedEvent { - _sourceCheckout := v.WebhookData.Data[bean.WEBHOOK_SELECTOR_SOURCE_CHECKOUT_NAME] - dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _getTruncatedImageTag(_sourceCheckout)) - } else { - toAppendDevtronParamInTag = !impl.config.CiCdConfig.UseImageTagFromGitProviderForTagBasedBuild - } - } - } - toAppendDevtronParamInTag = toAppendDevtronParamInTag && dockerImageTag != "" - if toAppendDevtronParamInTag { - dockerImageTag = fmt.Sprintf("%s-%d-%d", dockerImageTag, id, wfId) - } - // replace / with underscore, as docker image tag doesn't support slash. it gives error - dockerImageTag = strings.ReplaceAll(dockerImageTag, "/", "_") - return dockerImageTag -} - -func getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, commitOrCheckOutData string) string { - if dockerImageTag == "" { - dockerImageTag = commitOrCheckOutData - } else { - if commitOrCheckOutData != "" { - dockerImageTag = fmt.Sprintf("%s-%s", dockerImageTag, commitOrCheckOutData) - } - } - return dockerImageTag -} - -func (impl *CiServiceImpl) updateCiWorkflow(request *types.WorkflowRequest, savedWf *pipelineConfig.CiWorkflow) error { - ciBuildConfig := request.CiBuildConfig - ciBuildType := string(ciBuildConfig.CiBuildType) - savedWf.CiBuildType = ciBuildType - return impl.UpdateCiWorkflowWithStage(savedWf) -} - -func _getTruncatedImageTag(imageTag string) string { - _length := len(imageTag) - if _length == 0 { - return imageTag - } - - _truncatedLength := 8 - - if _length < _truncatedLength { - return imageTag - } else { - return imageTag[:_truncatedLength] +func (impl *CiServiceImpl) WriteCIFailEvent(ciWorkflow *pipelineConfig.CiWorkflow) { + event, _ := impl.eventFactory.Build(util2.Fail, &ciWorkflow.CiPipelineId, ciWorkflow.CiPipeline.AppId, nil, util2.CI) + material := &buildBean.MaterialTriggerInfo{} + material.GitTriggers = ciWorkflow.GitTriggers + event.CiWorkflowRunnerId = ciWorkflow.Id + event.UserId = int(ciWorkflow.TriggeredBy) + event = impl.eventFactory.BuildExtraCIData(event, material) + event.CiArtifactId = 0 + _, evtErr := impl.eventClient.WriteNotificationEvent(event) + if evtErr != nil { + impl.Logger.Errorw("error in writing event", "err", evtErr) } } diff --git a/pkg/pipeline/CiService_ent.go b/pkg/pipeline/CiService_ent.go deleted file mode 100644 index 342429d052..0000000000 --- a/pkg/pipeline/CiService_ent.go +++ /dev/null @@ -1,16 +0,0 @@ -package pipeline - -import ( - "github.com/devtron-labs/common-lib/imageScan/bean" - bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" - "github.com/devtron-labs/devtron/pkg/pipeline/types" - "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" -) - -func (impl *CiServiceImpl) fetchImageScanExecutionMedium() (*repository.ScanToolMetadata, bean.ScanExecutionMedium, error) { - return &repository.ScanToolMetadata{}, "", nil -} - -func (impl *CiServiceImpl) fetchImageScanExecutionStepsForWfRequest(scanToolMetadata *repository.ScanToolMetadata) ([]*types.ImageScanningSteps, []*bean2.RefPluginObject, error) { - return nil, nil, nil -} diff --git a/pkg/pipeline/PipelineStageService.go b/pkg/pipeline/PipelineStageService.go index 2ff4123f0f..2a538b7282 100644 --- a/pkg/pipeline/PipelineStageService.go +++ b/pkg/pipeline/PipelineStageService.go @@ -74,6 +74,11 @@ func NewPipelineStageService(logger *zap.SugaredLogger, } } +const ( + preCdStage = "preCD" + postCdStage = "postCD" +) + type PipelineStageServiceImpl struct { logger *zap.SugaredLogger pipelineStageRepository repository.PipelineStageRepository diff --git a/pkg/pipeline/adapter/adapter.go b/pkg/pipeline/adapter/adapter.go index badb5c0f3e..d480731a2a 100644 --- a/pkg/pipeline/adapter/adapter.go +++ b/pkg/pipeline/adapter/adapter.go @@ -23,6 +23,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/ciPipeline" "github.com/devtron-labs/devtron/pkg/bean" bean2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" bean3 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" @@ -190,7 +191,7 @@ func mergeMap(oldDockerArgs map[string]string, ciLevelDockerArgs map[string]stri // IsLinkedCD will return if the pipelineConfig.CiPipeline is a Linked CD func IsLinkedCD(ci pipelineConfig.CiPipeline) bool { - return ci.ParentCiPipeline != 0 && ci.PipelineType == string(bean2.LINKED_CD) + return ci.ParentCiPipeline != 0 && ci.PipelineType == string(common.LINKED_CD) } // IsLinkedCI will return if the pipelineConfig.CiPipeline is a Linked CI @@ -199,7 +200,7 @@ func IsLinkedCI(ci *pipelineConfig.CiPipeline) bool { return false } return ci.ParentCiPipeline != 0 && - ci.PipelineType == string(bean2.LINKED) + ci.PipelineType == string(common.LINKED) } // IsCIJob will return if the pipelineConfig.CiPipeline is a CI JOB @@ -207,7 +208,7 @@ func IsCIJob(ci *pipelineConfig.CiPipeline) bool { if ci == nil { return false } - return ci.PipelineType == string(bean2.CI_JOB) + return ci.PipelineType == string(common.CI_JOB) } // GetSourceCiDownStreamResponse will take the models []bean.LinkedCIDetails and []pipelineConfig.CdWorkflowRunner (for last deployment status) and generate the []CiPipeline.SourceCiDownStreamResponse diff --git a/pkg/pipeline/bean/CdHandlerBean.go b/pkg/pipeline/bean/CdHandlerBean.go index 4b6480e365..5dc408c226 100644 --- a/pkg/pipeline/bean/CdHandlerBean.go +++ b/pkg/pipeline/bean/CdHandlerBean.go @@ -4,37 +4,38 @@ import ( "github.com/devtron-labs/common-lib/utils/bean" "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" bean2 "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus/bean" "time" ) type CdWorkflowWithArtifact struct { - Id int `json:"id"` - CdWorkflowId int `json:"cd_workflow_id"` - Name string `json:"name"` - Status string `json:"status"` - PodStatus string `json:"pod_status"` - Message string `json:"message"` - StartedOn time.Time `json:"started_on"` - FinishedOn time.Time `json:"finished_on"` - PipelineId int `json:"pipeline_id"` - Namespace string `json:"namespace"` - LogFilePath string `json:"log_file_path"` - TriggeredBy int32 `json:"triggered_by"` - EmailId string `json:"email_id"` - Image string `json:"image"` - TargetPlatforms []*bean.TargetPlatform `json:"targetPlatforms"` - MaterialInfo string `json:"material_info,omitempty"` - DataSource string `json:"data_source,omitempty"` - CiArtifactId int `json:"ci_artifact_id,omitempty"` - IsArtifactUploaded bool `json:"isArtifactUploaded"` - WorkflowType string `json:"workflow_type,omitempty"` - ExecutorType string `json:"executor_type,omitempty"` - BlobStorageEnabled bool `json:"blobStorageEnabled"` - GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` - CiMaterials []pipelineConfig.CiPipelineMaterialResponse `json:"ciMaterials"` - ImageReleaseTags []*repository.ImageTag `json:"imageReleaseTags"` - ImageComment *repository.ImageComment `json:"imageComment"` - RefCdWorkflowRunnerId int `json:"referenceCdWorkflowRunnerId"` - WorkflowExecutionStage map[string][]*bean2.WorkflowStageDto `json:"workflowExecutionStages"` + Id int `json:"id"` + CdWorkflowId int `json:"cd_workflow_id"` + Name string `json:"name"` + Status string `json:"status"` + PodStatus string `json:"pod_status"` + Message string `json:"message"` + StartedOn time.Time `json:"started_on"` + FinishedOn time.Time `json:"finished_on"` + PipelineId int `json:"pipeline_id"` + Namespace string `json:"namespace"` + LogFilePath string `json:"log_file_path"` + TriggeredBy int32 `json:"triggered_by"` + EmailId string `json:"email_id"` + Image string `json:"image"` + TargetPlatforms []*bean.TargetPlatform `json:"targetPlatforms"` + MaterialInfo string `json:"material_info,omitempty"` + DataSource string `json:"data_source,omitempty"` + CiArtifactId int `json:"ci_artifact_id,omitempty"` + IsArtifactUploaded bool `json:"isArtifactUploaded"` + WorkflowType string `json:"workflow_type,omitempty"` + ExecutorType string `json:"executor_type,omitempty"` + BlobStorageEnabled bool `json:"blobStorageEnabled"` + GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` + CiMaterials []buildBean.CiPipelineMaterialResponse `json:"ciMaterials"` + ImageReleaseTags []*repository.ImageTag `json:"imageReleaseTags"` + ImageComment *repository.ImageComment `json:"imageComment"` + RefCdWorkflowRunnerId int `json:"referenceCdWorkflowRunnerId"` + WorkflowExecutionStage map[string][]*bean2.WorkflowStageDto `json:"workflowExecutionStages"` } diff --git a/pkg/pipeline/bean/WorkflowTemplate.go b/pkg/pipeline/bean/WorkflowTemplate.go index 55e91e565c..dcab419b5e 100644 --- a/pkg/pipeline/bean/WorkflowTemplate.go +++ b/pkg/pipeline/bean/WorkflowTemplate.go @@ -18,6 +18,7 @@ package bean import ( blob_storage "github.com/devtron-labs/common-lib/blob-storage" + informerBean "github.com/devtron-labs/common-lib/informer" "github.com/devtron-labs/devtron/api/bean" v1 "k8s.io/api/core/v1" v12 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -47,17 +48,20 @@ type WorkflowTemplate struct { RefPlugins []*RefPluginObject TerminationGracePeriod int WorkflowType string + DevtronInstanceUID string } const ( - CI_WORKFLOW_NAME = "ci" - CI_WORKFLOW_WITH_STAGES = "ci-stages-with-env" - CiStage = "CI" - JobStage = "JOB" - CdStage = "CD" - CD_WORKFLOW_NAME = "cd" - CD_WORKFLOW_WITH_STAGES = "cd-stages-with-env" - WorkflowGenerateNamePrefix = "devtron.ai/generate-name-prefix" + CI_WORKFLOW_NAME = "ci" + CI_WORKFLOW_WITH_STAGES = "ci-stages-with-env" + CiStage = "CI" + JobStage = "JOB" + CdStage = "CD" + CD_WORKFLOW_NAME = "cd" + CD_WORKFLOW_WITH_STAGES = "cd-stages-with-env" + WorkflowGenerateNamePrefix = "devtron.ai/generate-name-prefix" + DevtronLabelPurposeKey = "devtron.ai/purpose" + DevtronLabelPurposeWorkflow = "workflow" ) func (workflowTemplate *WorkflowTemplate) GetEntrypoint() string { @@ -76,8 +80,11 @@ func (workflowTemplate *WorkflowTemplate) SetActiveDeadlineSeconds(timeout int64 } func (workflowTemplate *WorkflowTemplate) CreateObjectMetadata() *v12.ObjectMeta { - - workflowLabels := map[string]string{WorkflowGenerateNamePrefix: workflowTemplate.WorkflowNamePrefix} + workflowLabels := map[string]string{ + WorkflowGenerateNamePrefix: workflowTemplate.WorkflowNamePrefix, + informerBean.WorkflowTypeLabelKey: workflowTemplate.WorkflowType, + informerBean.DevtronOwnerInstanceLabelKey: workflowTemplate.DevtronInstanceUID, + } switch workflowTemplate.WorkflowType { case CI_WORKFLOW_NAME: workflowLabels["devtron.ai/workflow-purpose"] = "ci" diff --git a/pkg/pipeline/constants/constants.go b/pkg/pipeline/constants/constants.go index 0d5cd5bcab..0028c37d45 100644 --- a/pkg/pipeline/constants/constants.go +++ b/pkg/pipeline/constants/constants.go @@ -19,19 +19,6 @@ package constants const CDPipelineNotFoundErr = "cd pipeline not found" const CiPipelineNotFoundErr = "ci pipeline not found" -type PipelineType string - -// default PipelineType -const DefaultPipelineType = CI_BUILD - -const ( - CI_BUILD PipelineType = "CI_BUILD" - LINKED PipelineType = "LINKED" - EXTERNAL PipelineType = "EXTERNAL" - CI_JOB PipelineType = "CI_JOB" - LINKED_CD PipelineType = "LINKED_CD" -) - type PatchPipelineActionResponse string const ( @@ -43,32 +30,13 @@ const ( PATCH_PIPELINE_ACTION_NOT_APPLICABLE PatchPipelineActionResponse = "N/A" ) -func (r PipelineType) ToString() string { - return string(r) -} - -func (pType PipelineType) IsValidPipelineType() bool { - switch pType { - case CI_BUILD, LINKED, EXTERNAL, CI_JOB, LINKED_CD: - return true - default: - return false - } -} - const ( UNIQUE_DEPLOYMENT_APP_NAME = "unique_deployment_app_name" ) -const ( - ExtraEnvVarExternalCiArtifactKey = "externalCiArtifact" - ExtraEnvVarImageDigestKey = "imageDigest" -) - const DefaultCiWorkflowNamespace = "devtron-ci" const Running = "Running" const Starting = "Starting" -const POD_DELETED_MESSAGE = "pod deleted" const TERMINATE_MESSAGE = "workflow shutdown with strategy: Terminate" const FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE = "workflow shutdown with strategy: Force Abort" const POD_TIMEOUT_MESSAGE = "Pod was active on the node longer than the specified deadline" diff --git a/pkg/pipeline/executors/WorkflowUtils.go b/pkg/pipeline/executors/WorkflowUtils.go index 0c9f18e911..bbf7439660 100644 --- a/pkg/pipeline/executors/WorkflowUtils.go +++ b/pkg/pipeline/executors/WorkflowUtils.go @@ -21,6 +21,7 @@ import ( "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned" v1alpha12 "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned/typed/workflow/v1alpha1" + informerBean "github.com/devtron-labs/common-lib/informer" "github.com/devtron-labs/common-lib/utils" bean2 "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository" @@ -287,16 +288,20 @@ func GetClientInstance(config *rest.Config, namespace string) (v1alpha12.Workflo } func CheckIfReTriggerRequired(status, message, workflowRunnerStatus string) bool { - return ((status == string(v1alpha1.NodeError) || status == string(v1alpha1.NodeFailed)) && - message == cdWorkflow.POD_DELETED_MESSAGE) && (workflowRunnerStatus != cdWorkflow.WorkflowCancel && workflowRunnerStatus != cdWorkflow.WorkflowAborted) + return (status == string(v1alpha1.NodeError) || status == string(v1alpha1.NodeFailed)) && + message == cdWorkflow.POD_DELETED_MESSAGE && + workflowRunnerStatus != cdWorkflow.WorkflowCancel && + workflowRunnerStatus != cdWorkflow.WorkflowAborted && + workflowRunnerStatus != cdWorkflow.WorkflowSucceeded } func GetWorkflowLabelsForSystemExecutor(workflowTemplate bean.WorkflowTemplate) map[string]string { return map[string]string{ - DEVTRON_WORKFLOW_LABEL_KEY: DEVTRON_WORKFLOW_LABEL_VALUE, - "devtron.ai/purpose": "workflow", - "workflowType": workflowTemplate.WorkflowType, - bean.WorkflowGenerateNamePrefix: workflowTemplate.WorkflowNamePrefix, + DEVTRON_WORKFLOW_LABEL_KEY: DEVTRON_WORKFLOW_LABEL_VALUE, + bean.DevtronLabelPurposeKey: bean.DevtronLabelPurposeWorkflow, + informerBean.WorkflowTypeLabelKey: workflowTemplate.WorkflowType, + bean.WorkflowGenerateNamePrefix: workflowTemplate.WorkflowNamePrefix, + informerBean.DevtronOwnerInstanceLabelKey: workflowTemplate.DevtronInstanceUID, } } diff --git a/pkg/pipeline/pipelineStageVariableParser.go b/pkg/pipeline/pipelineStageVariableParser.go index b97517d30b..bd0d3bce77 100644 --- a/pkg/pipeline/pipelineStageVariableParser.go +++ b/pkg/pipeline/pipelineStageVariableParser.go @@ -30,13 +30,9 @@ import ( ) type copyContainerImagePluginInputVariable = string -type RefPluginName = string const ( - COPY_CONTAINER_IMAGE RefPluginName = "Copy container image" - COPY_CONTAINER_IMAGE_VERSION_V1 = "1.0.0" - COPY_CONTAINER_IMAGE_VERSION_V2 = "2.0.0" - EMPTY_STRING = " " + EMPTY_STRING = " " ) const ( diff --git a/pkg/pipeline/types/CiCdConfig.go b/pkg/pipeline/types/CiCdConfig.go index 5d50162141..246a0ec801 100644 --- a/pkg/pipeline/types/CiCdConfig.go +++ b/pkg/pipeline/types/CiCdConfig.go @@ -27,6 +27,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/pkg/bean/common" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/pipeline/bean" v12 "k8s.io/api/core/v1" @@ -550,17 +551,18 @@ type ArtifactsForCiJob struct { } type GitTriggerInfoResponse struct { - CiMaterials []pipelineConfig.CiPipelineMaterialResponse `json:"ciMaterials"` - TriggeredByEmail string `json:"triggeredByEmail"` - LastDeployedTime string `json:"lastDeployedTime,omitempty"` - AppId int `json:"appId"` - AppName string `json:"appName"` - EnvironmentId int `json:"environmentId"` - EnvironmentName string `json:"environmentName"` - Default bool `json:"default,omitempty"` - ImageTaggingData ImageTaggingResponseDTO `json:"imageTaggingData"` - Image string `json:"image"` - TargetPlatforms []*bean2.TargetPlatform `json:"targetPlatforms"` + CiMaterials []buildBean.CiPipelineMaterialResponse `json:"ciMaterials"` + TriggeredByEmail string `json:"triggeredByEmail"` + LastDeployedTime string `json:"lastDeployedTime,omitempty"` + CiPipelineId int `json:"ciPipelineId"` + AppId int `json:"appId"` + AppName string `json:"appName"` + EnvironmentId int `json:"environmentId"` + EnvironmentName string `json:"environmentName"` + Default bool `json:"default,omitempty"` + ImageTaggingData ImageTaggingResponseDTO `json:"imageTaggingData"` + Image string `json:"image"` + TargetPlatforms []*bean2.TargetPlatform `json:"targetPlatforms"` } type Trigger struct { diff --git a/pkg/pipeline/types/Workflow.go b/pkg/pipeline/types/Workflow.go index dca45ab8fc..dceb9a03b0 100644 --- a/pkg/pipeline/types/Workflow.go +++ b/pkg/pipeline/types/Workflow.go @@ -31,6 +31,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" bean2 "github.com/devtron-labs/devtron/pkg/bean" bean5 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" repository4 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" infraBean "github.com/devtron-labs/devtron/pkg/infraConfig/bean/v1" "github.com/devtron-labs/devtron/pkg/pipeline/bean" @@ -649,37 +650,37 @@ type ChildCdMetadata struct { } type WorkflowResponse struct { - Id int `json:"id"` - Name string `json:"name"` - Status string `json:"status"` - PodStatus string `json:"podStatus"` - Message string `json:"message"` - StartedOn time.Time `json:"startedOn"` - FinishedOn time.Time `json:"finishedOn"` - CiPipelineId int `json:"ciPipelineId"` - Namespace string `json:"namespace"` - LogLocation string `json:"logLocation"` - BlobStorageEnabled bool `json:"blobStorageEnabled"` - GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` - CiMaterials []pipelineConfig.CiPipelineMaterialResponse `json:"ciMaterials"` - TriggeredBy int32 `json:"triggeredBy"` - Artifact string `json:"artifact"` - TriggeredByEmail string `json:"triggeredByEmail"` - Stage string `json:"stage"` - ArtifactId int `json:"artifactId"` - IsArtifactUploaded bool `json:"isArtifactUploaded"` - IsVirtualEnvironment bool `json:"isVirtualEnvironment"` - PodName string `json:"podName"` - EnvironmentId int `json:"environmentId"` - EnvironmentName string `json:"environmentName"` - ImageReleaseTags []*repository3.ImageTag `json:"imageReleaseTags"` - ImageComment *repository3.ImageComment `json:"imageComment"` - AppWorkflowId int `json:"appWorkflowId"` - CustomTag *bean3.CustomTagErrorResponse `json:"customTag,omitempty"` - PipelineType string `json:"pipelineType"` - ReferenceWorkflowId int `json:"referenceWorkflowId"` - TargetPlatforms []*bean7.TargetPlatform `json:"targetPlatforms"` - WorkflowExecutionStage map[string][]*bean6.WorkflowStageDto `json:"workflowExecutionStages"` + Id int `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + PodStatus string `json:"podStatus"` + Message string `json:"message"` + StartedOn time.Time `json:"startedOn"` + FinishedOn time.Time `json:"finishedOn"` + CiPipelineId int `json:"ciPipelineId"` + Namespace string `json:"namespace"` + LogLocation string `json:"logLocation"` + BlobStorageEnabled bool `json:"blobStorageEnabled"` + GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` + CiMaterials []buildBean.CiPipelineMaterialResponse `json:"ciMaterials"` + TriggeredBy int32 `json:"triggeredBy"` + Artifact string `json:"artifact"` + TriggeredByEmail string `json:"triggeredByEmail"` + Stage string `json:"stage"` + ArtifactId int `json:"artifactId"` + IsArtifactUploaded bool `json:"isArtifactUploaded"` + IsVirtualEnvironment bool `json:"isVirtualEnvironment"` + PodName string `json:"podName"` + EnvironmentId int `json:"environmentId"` + EnvironmentName string `json:"environmentName"` + ImageReleaseTags []*repository3.ImageTag `json:"imageReleaseTags"` + ImageComment *repository3.ImageComment `json:"imageComment"` + AppWorkflowId int `json:"appWorkflowId"` + CustomTag *bean3.CustomTagErrorResponse `json:"customTag,omitempty"` + PipelineType string `json:"pipelineType"` + ReferenceWorkflowId int `json:"referenceWorkflowId"` + TargetPlatforms []*bean7.TargetPlatform `json:"targetPlatforms"` + WorkflowExecutionStage map[string][]*bean6.WorkflowStageDto `json:"workflowExecutionStages"` } type ConfigMapSecretDto struct { diff --git a/pkg/pipeline/types/Workflow_ent.go b/pkg/pipeline/types/Workflow_ent.go index 2a5c03e78f..ecf78b9e1a 100644 --- a/pkg/pipeline/types/Workflow_ent.go +++ b/pkg/pipeline/types/Workflow_ent.go @@ -30,6 +30,6 @@ func (workflowRequest *WorkflowRequest) SetImageScanningSteps(imageScanningSteps return } -func (workflowRequest *WorkflowRequest) SetAwsInspectorConfig(awsInspectorConfig string) { +func (workflowRequest *WorkflowRequest) SetEntOnlyFields(trigger Trigger, ciConfig *CiConfig) { return } diff --git a/pkg/ucid/bean.go b/pkg/ucid/bean.go new file mode 100644 index 0000000000..acd595ad57 --- /dev/null +++ b/pkg/ucid/bean.go @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ucid + +const ( + DevtronUniqueClientIdConfigMap = "devtron-ucid" + DevtronUniqueClientIdConfigMapKey = "UCID" + UIEventKey = "uiEventKey" + InstallEventKey = "installEvent" +) diff --git a/pkg/ucid/ucid.go b/pkg/ucid/ucid.go new file mode 100644 index 0000000000..65c03e8887 --- /dev/null +++ b/pkg/ucid/ucid.go @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ucid + +import ( + "fmt" + "github.com/devtron-labs/common-lib/utils/k8s" + "github.com/devtron-labs/devtron/pkg/util" + globalUtil "github.com/devtron-labs/devtron/util" + goCache "github.com/patrickmn/go-cache" + "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type Service interface { + GetUCIDWithCache(cache *goCache.Cache) (string, bool, error) + GetUCIDWithOutCache() (string, bool, error) +} + +type ServiceImpl struct { + logger *zap.SugaredLogger + K8sUtil *k8s.K8sServiceImpl + aCDAuthConfig *util.ACDAuthConfig +} + +func NewServiceImpl( + logger *zap.SugaredLogger, + K8sUtil *k8s.K8sServiceImpl, + aCDAuthConfig *util.ACDAuthConfig, +) *ServiceImpl { + return &ServiceImpl{ + logger: logger, + K8sUtil: K8sUtil, + aCDAuthConfig: aCDAuthConfig, + } +} + +func (impl *ServiceImpl) GetUCIDWithOutCache() (string, bool, error) { + return impl.GetUCIDWithCache(nil) +} + +func (impl *ServiceImpl) GetUCIDWithCache(cache *goCache.Cache) (string, bool, error) { + if cache != nil { + ucid, found := cache.Get(DevtronUniqueClientIdConfigMapKey) + if found { + return ucid.(string), true, nil + } + } + client, err := impl.K8sUtil.GetClientForInCluster() + if err != nil { + impl.logger.Errorw("exception while getting unique client id", "error", err) + return "", false, err + } + cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, DevtronUniqueClientIdConfigMap, client) + if errStatus, ok := status.FromError(err); !ok || errStatus.Code() == codes.NotFound || errStatus.Code() == codes.Unknown { + // if not found, create new cm + cm = &v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: DevtronUniqueClientIdConfigMap}} + data := map[string]string{} + data[DevtronUniqueClientIdConfigMapKey] = globalUtil.Generate(16) // generate unique random number + data[InstallEventKey] = "1" // used in operator to detect event is install or upgrade + data[UIEventKey] = "1" + cm.Data = data + _, err = impl.K8sUtil.CreateConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, cm, client) + if err != nil { + impl.logger.Errorw("exception while getting unique client id", "error", err) + return "", false, err + } + } + if cm == nil { + impl.logger.Errorw("configmap not found while getting unique client id") + return "", false, fmt.Errorf("configmap %q not found while getting unique client id", DevtronUniqueClientIdConfigMap) + } + dataMap := cm.Data + ucid := dataMap[DevtronUniqueClientIdConfigMapKey] + if cache != nil { + cache.Set(DevtronUniqueClientIdConfigMapKey, ucid, goCache.DefaultExpiration) + } + return ucid, false, nil +} diff --git a/pkg/ucid/wire_ucid.go b/pkg/ucid/wire_ucid.go new file mode 100644 index 0000000000..f8f4402af2 --- /dev/null +++ b/pkg/ucid/wire_ucid.go @@ -0,0 +1,8 @@ +package ucid + +import "github.com/google/wire" + +var WireSet = wire.NewSet( + NewServiceImpl, + wire.Bind(new(Service), new(*ServiceImpl)), +) diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index d87f6bfd97..a53e90dbd6 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -24,6 +24,7 @@ import ( "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/devtron-labs/common-lib/async" "github.com/devtron-labs/common-lib/utils" + "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/common-lib/utils/workFlow" bean6 "github.com/devtron-labs/devtron/api/helm-app/bean" client2 "github.com/devtron-labs/devtron/api/helm-app/service" @@ -38,6 +39,10 @@ import ( bean7 "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/build/artifacts" bean5 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" + "github.com/devtron-labs/devtron/pkg/build/trigger" + "github.com/devtron-labs/devtron/pkg/cluster/adapter" + repository5 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/manifest" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" @@ -45,6 +50,8 @@ import ( triggerBean "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/userDeploymentRequest/service" eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" + "github.com/devtron-labs/devtron/pkg/executor" + k8sPkg "github.com/devtron-labs/devtron/pkg/k8s" "github.com/devtron-labs/devtron/pkg/pipeline" constants2 "github.com/devtron-labs/devtron/pkg/pipeline/constants" repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository" @@ -58,6 +65,9 @@ import ( "github.com/devtron-labs/devtron/pkg/workflow/dag/helper" error2 "github.com/devtron-labs/devtron/util/error" util2 "github.com/devtron-labs/devtron/util/event" + errors2 "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/rest" + "net/http" "strings" "sync" "time" @@ -83,6 +93,7 @@ import ( ) type WorkflowDagExecutor interface { + UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error UpdateCiWorkflowForCiSuccess(request *bean2.CiArtifactWebhookRequest) (err error) HandleCiSuccessEvent(triggerContext triggerBean.TriggerContext, ciPipelineId int, request *bean2.CiArtifactWebhookRequest, imagePushedAt time.Time) (id int, err error) HandlePreStageSuccessEvent(triggerContext triggerBean.TriggerContext, cdStageCompleteEvent eventProcessorBean.CdStageCompleteEvent) error @@ -125,7 +136,7 @@ type WorkflowDagExecutorImpl struct { helmAppService client2.HelmAppService cdWorkflowCommonService cd.CdWorkflowCommonService - cdTriggerService devtronApps.TriggerService + cdHandlerService devtronApps.HandlerService userDeploymentRequestService service.UserDeploymentRequestService manifestCreationService manifest.ManifestCreationService @@ -134,6 +145,12 @@ type WorkflowDagExecutorImpl struct { asyncRunnable *async.Runnable scanHistoryRepository repository3.ImageScanHistoryRepository imageScanService imageScanning.ImageScanService + + K8sUtil *k8s.K8sServiceImpl + envRepository repository5.EnvironmentRepository + k8sCommonService k8sPkg.K8sCommonService + workflowService executor.WorkflowService + ciHandlerService trigger.HandlerService } func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, @@ -154,7 +171,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi ciService pipeline.CiService, helmAppService client2.HelmAppService, cdWorkflowCommonService cd.CdWorkflowCommonService, - cdTriggerService devtronApps.TriggerService, + cdHandlerService devtronApps.HandlerService, userDeploymentRequestService service.UserDeploymentRequestService, manifestCreationService manifest.ManifestCreationService, commonArtifactService artifacts.CommonArtifactService, @@ -162,6 +179,11 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi asyncRunnable *async.Runnable, scanHistoryRepository repository3.ImageScanHistoryRepository, imageScanService imageScanning.ImageScanService, + K8sUtil *k8s.K8sServiceImpl, + envRepository repository5.EnvironmentRepository, + k8sCommonService k8sPkg.K8sCommonService, + workflowService executor.WorkflowService, + ciHandlerService trigger.HandlerService, ) *WorkflowDagExecutorImpl { wde := &WorkflowDagExecutorImpl{logger: Logger, pipelineRepository: pipelineRepository, @@ -180,7 +202,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi pipelineStatusTimelineService: pipelineStatusTimelineService, helmAppService: helmAppService, cdWorkflowCommonService: cdWorkflowCommonService, - cdTriggerService: cdTriggerService, + cdHandlerService: cdHandlerService, userDeploymentRequestService: userDeploymentRequestService, manifestCreationService: manifestCreationService, commonArtifactService: commonArtifactService, @@ -190,6 +212,11 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi imageScanService: imageScanService, cdWorkflowRunnerService: cdWorkflowRunnerService, ciService: ciService, + K8sUtil: K8sUtil, + envRepository: envRepository, + k8sCommonService: k8sCommonService, + workflowService: workflowService, + ciHandlerService: ciHandlerService, } config, err := types.GetCdConfig() if err != nil { @@ -209,6 +236,130 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi return wde } +func (impl *WorkflowDagExecutorImpl) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error { + ciWorkflows, err := impl.ciWorkflowRepository.FindByStatusesIn([]string{constants2.Starting, constants2.Running}) + if err != nil { + impl.logger.Errorw("error on fetching ci workflows", "err", err) + return err + } + client, err := impl.K8sUtil.GetClientForInCluster() + if err != nil { + impl.logger.Errorw("error while fetching k8s client", "error", err) + return err + } + + for _, ciWorkflow := range ciWorkflows { + var isExt bool + var env *repository5.Environment + var restConfig *rest.Config + if ciWorkflow.Namespace != constants2.DefaultCiWorkflowNamespace { + isExt = true + env, err = impl.envRepository.FindById(ciWorkflow.EnvironmentId) + if err != nil { + impl.logger.Errorw("could not fetch stage env", "err", err) + return err + } + restConfig, err = impl.getRestConfig(ciWorkflow) + if err != nil { + return err + } + } + + isEligibleToMarkFailed := false + isPodDeleted := false + if time.Since(ciWorkflow.StartedOn) > (time.Minute * time.Duration(timeoutForFailureCiBuild)) { + + //check weather pod is exists or not, if exits check its status + wf, err := impl.workflowService.GetWorkflowStatus(ciWorkflow.ExecutorType, ciWorkflow.Name, ciWorkflow.Namespace, restConfig) + if err != nil { + impl.logger.Warnw("unable to fetch ci workflow", "err", err) + statusError, ok := err.(*errors2.StatusError) + if ok && statusError.Status().Code == http.StatusNotFound { + impl.logger.Warnw("ci workflow not found", "err", err) + isEligibleToMarkFailed = true + } else { + continue + // skip this and process for next ci workflow + } + } + + //if ci workflow is exists, check its pod + if !isEligibleToMarkFailed { + ns := constants2.DefaultCiWorkflowNamespace + if isExt { + _, client, err = impl.k8sCommonService.GetCoreClientByClusterId(env.ClusterId) + if err != nil { + impl.logger.Warnw("error in getting core v1 client using GetCoreClientByClusterId", "err", err, "clusterId", env.Cluster.Id) + continue + } + ns = env.Namespace + } + _, err := impl.K8sUtil.GetPodByName(ns, ciWorkflow.PodName, client) + if err != nil { + impl.logger.Warnw("unable to fetch ci workflow - pod", "err", err) + statusError, ok := err.(*errors2.StatusError) + if ok && statusError.Status().Code == http.StatusNotFound { + impl.logger.Warnw("pod not found", "err", err) + isEligibleToMarkFailed = true + } else { + continue + // skip this and process for next ci workflow + } + } + if ciWorkflow.ExecutorType == cdWorkflow2.WORKFLOW_EXECUTOR_TYPE_SYSTEM { + if wf.Status == string(v1alpha1.WorkflowFailed) { + isPodDeleted = true + } + } else { + // check workflow status,get the status + if wf.Status == string(v1alpha1.WorkflowFailed) && wf.Message == cdWorkflow2.POD_DELETED_MESSAGE { + isPodDeleted = true + } + } + } + } + if isEligibleToMarkFailed { + ciWorkflow.Status = "Failed" + ciWorkflow.PodStatus = "Failed" + if isPodDeleted { + ciWorkflow.Message = cdWorkflow2.POD_DELETED_MESSAGE + // error logging handled inside handlePodDeleted + impl.ciHandlerService.HandlePodDeleted(ciWorkflow) + } else { + ciWorkflow.Message = "marked failed by job" + } + err := impl.ciService.UpdateCiWorkflowWithStage(ciWorkflow) + if err != nil { + impl.logger.Errorw("unable to update ci workflow, its eligible to mark failed", "err", err) + // skip this and process for next ci workflow + } + err = impl.customTagService.DeactivateImagePathReservation(ciWorkflow.ImagePathReservationId) + if err != nil { + impl.logger.Errorw("unable to update ci workflow, its eligible to mark failed", "err", err) + } + } + } + return nil +} + +func (impl *WorkflowDagExecutorImpl) getRestConfig(workflow *pipelineConfig.CiWorkflow) (*rest.Config, error) { + env, err := impl.envRepository.FindById(workflow.EnvironmentId) + if err != nil { + impl.logger.Errorw("could not fetch stage env", "err", err) + return nil, err + } + + clusterBean := adapter.GetClusterBean(*env.Cluster) + + clusterConfig := clusterBean.GetClusterConfig() + restConfig, err := impl.K8sUtil.GetRestConfigByCluster(clusterConfig) + if err != nil { + impl.logger.Errorw("error in getting rest config by cluster id", "err", err) + return nil, err + } + return restConfig, nil +} + func (impl *WorkflowDagExecutorImpl) HandleCdStageReTrigger(runner *pipelineConfig.CdWorkflowRunner) error { // do not re-trigger if retries = 0 if !impl.config.WorkflowRetriesEnabled() { @@ -250,13 +401,13 @@ func (impl *WorkflowDagExecutorImpl) HandleCdStageReTrigger(runner *pipelineConf } if runner.WorkflowType == bean.CD_WORKFLOW_TYPE_PRE { - _, err = impl.cdTriggerService.TriggerPreStage(triggerRequest) + _, err = impl.cdHandlerService.TriggerPreStage(triggerRequest) if err != nil { impl.logger.Errorw("error in TriggerPreStage ", "err", err, "cdWorkflowRunnerId", runner.Id) return err } } else if runner.WorkflowType == bean.CD_WORKFLOW_TYPE_POST { - _, err = impl.cdTriggerService.TriggerPostStage(triggerRequest) + _, err = impl.cdHandlerService.TriggerPostStage(triggerRequest) if err != nil { impl.logger.Errorw("error in TriggerPostStage ", "err", err, "cdWorkflowRunnerId", runner.Id) return err @@ -409,7 +560,7 @@ func (impl *WorkflowDagExecutorImpl) ProcessDevtronAsyncInstallRequest(cdAsyncIn impl.logger.Errorw("error in getting deployment config by appId and envId", "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId, "err", err) return err } - releaseId, _, releaseErr := impl.cdTriggerService.TriggerRelease(newCtx, overrideRequest, envDeploymentConfig, cdAsyncInstallReq.TriggeredAt, cdAsyncInstallReq.TriggeredBy) + releaseId, _, releaseErr := impl.cdHandlerService.TriggerRelease(newCtx, overrideRequest, envDeploymentConfig, cdAsyncInstallReq.TriggeredAt, cdAsyncInstallReq.TriggeredBy) if releaseErr != nil { impl.logger.Errorw("error encountered in ProcessDevtronAsyncInstallRequest", "err", releaseErr, "cdWfrId", cdWfr.Id) impl.handleAsyncTriggerReleaseError(newCtx, releaseErr, cdWfr, overrideRequest) @@ -539,13 +690,13 @@ func (impl *WorkflowDagExecutorImpl) triggerIfAutoStageCdPipeline(request trigge // pre stage exists if request.Pipeline.PreTriggerType == pipelineConfig.TRIGGER_TYPE_AUTOMATIC { impl.logger.Debugw("trigger pre stage for pipeline", "artifactId", request.Artifact.Id, "pipelineId", request.Pipeline.Id) - _, err = impl.cdTriggerService.TriggerPreStage(request) // TODO handle error here + _, err = impl.cdHandlerService.TriggerPreStage(request) // TODO handle error here return err } } else if request.Pipeline.TriggerType == pipelineConfig.TRIGGER_TYPE_AUTOMATIC { // trigger deployment impl.logger.Debugw("trigger cd for pipeline", "artifactId", request.Artifact.Id, "pipelineId", request.Pipeline.Id) - err = impl.cdTriggerService.TriggerAutomaticDeployment(request) + err = impl.cdHandlerService.TriggerAutomaticDeployment(request) return err } return nil @@ -615,7 +766,7 @@ func (impl *WorkflowDagExecutorImpl) HandlePreStageSuccessEvent(triggerContext t } else { ciArtifactId = cdStageCompleteEvent.CiArtifactDTO.Id } - err = impl.cdTriggerService.TriggerAutoCDOnPreStageSuccess(triggerContext, cdStageCompleteEvent.CdPipelineId, ciArtifactId, cdStageCompleteEvent.WorkflowId) + err = impl.cdHandlerService.TriggerAutoCDOnPreStageSuccess(triggerContext, cdStageCompleteEvent.CdPipelineId, ciArtifactId, cdStageCompleteEvent.WorkflowId) if err != nil { impl.logger.Errorw("error in triggering cd on pre cd succcess", "err", err) return err @@ -659,7 +810,7 @@ func (impl *WorkflowDagExecutorImpl) HandleDeploymentSuccessEvent(triggerContext RefCdWorkflowRunnerId: 0, } triggerRequest.TriggerContext.Context = context.Background() - _, err = impl.cdTriggerService.TriggerPostStage(triggerRequest) + _, err = impl.cdHandlerService.TriggerPostStage(triggerRequest) if err != nil { impl.logger.Errorw("error in triggering post stage after successful deployment event", "err", err, "cdWorkflow", cdWorkflow) return err @@ -806,7 +957,7 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext trigger // image scanning plugin can only be applied in Post-ci, scanning in pre-ci doesn't make sense pipelineStage := repository4.PIPELINE_STAGE_TYPE_POST_CI - if pipelineModal.PipelineType == constants2.CI_JOB.ToString() { + if pipelineModal.PipelineType == buildCommonBean.CI_JOB.ToString() { pipelineStage = repository4.PIPELINE_STAGE_TYPE_PRE_CI } // this flag comes from ci-runner when scanning is enabled from ciPipeline modal @@ -830,7 +981,7 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext trigger var pluginArtifacts []*repository.CiArtifact for registry, artifacts := range request.PluginRegistryArtifactDetails { for _, image := range artifacts { - if pipelineModal.PipelineType == string(bean5.CI_JOB) && image == "" { + if pipelineModal.PipelineType == string(buildCommonBean.CI_JOB) && image == "" { continue } pluginArtifact := &repository.CiArtifact{ @@ -994,7 +1145,7 @@ func (impl *WorkflowDagExecutorImpl) HandleCiStepFailedEvent(ciPipelineId int, r func (impl *WorkflowDagExecutorImpl) WriteCiStepFailedEvent(pipeline *pipelineConfig.CiPipeline, request *bean2.CiArtifactWebhookRequest, ciWorkflow *pipelineConfig.CiWorkflow) { event, _ := impl.eventFactory.Build(util2.Fail, &pipeline.Id, pipeline.AppId, nil, util2.CI) - material := &client.MaterialTriggerInfo{} + material := &bean5.MaterialTriggerInfo{} material.GitTriggers = ciWorkflow.GitTriggers event.CiWorkflowRunnerId = ciWorkflow.Id event.UserId = int(ciWorkflow.TriggeredBy) diff --git a/scripts/sql/32803400_workflow_message.down.sql b/scripts/sql/32803400_workflow_message.down.sql new file mode 100644 index 0000000000..7b794d76d6 --- /dev/null +++ b/scripts/sql/32803400_workflow_message.down.sql @@ -0,0 +1 @@ +-- Migration down is not applicable here as cd_workflow_runner.message has moved to higher DataType \ No newline at end of file diff --git a/scripts/sql/32803400_workflow_message.up.sql b/scripts/sql/32803400_workflow_message.up.sql new file mode 100644 index 0000000000..bc3083ee77 --- /dev/null +++ b/scripts/sql/32803400_workflow_message.up.sql @@ -0,0 +1,11 @@ +BEGIN; + +-- cd_workflow_runner.message has a limit of 1000 characters. migrating to +ALTER TABLE "public"."cd_workflow_runner" + ALTER COLUMN "message" TYPE TEXT; + +-- ci_workflow.message has a limit of 250 characters. migrating to +ALTER TABLE "public"."ci_workflow" + ALTER COLUMN "message" TYPE TEXT; + +COMMIT; \ No newline at end of file diff --git a/util/helper.go b/util/helper.go index c8d3e00c27..50fbbf91ab 100644 --- a/util/helper.go +++ b/util/helper.go @@ -25,8 +25,8 @@ import ( "fmt" "github.com/devtron-labs/devtron/internal/middleware" "github.com/juju/errors" + "go.opentelemetry.io/otel" "io" - "io/ioutil" "math/rand" "net/http" "os" @@ -139,8 +139,10 @@ func GenerateNewWorkflowName(name string) string { return fmt.Sprintf("%s-clone-%s", name, Generate(4)) } -func HttpRequest(url string) (map[string]interface{}, error) { - req, err := http.NewRequest(http.MethodGet, url, nil) +func HttpRequest(ctx context.Context, url string) (map[string]interface{}, error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "util.HttpRequest") + defer span.End() + req, err := http.NewRequestWithContext(newCtx, http.MethodGet, url, nil) if err != nil { return nil, err } @@ -151,7 +153,7 @@ func HttpRequest(url string) (map[string]interface{}, error) { return nil, err } if res.StatusCode >= 200 && res.StatusCode <= 299 { - resBody, err := ioutil.ReadAll(res.Body) + resBody, err := io.ReadAll(res.Body) if err != nil { return nil, err } diff --git a/vendor/github.com/devtron-labs/common-lib/informer/bean.go b/vendor/github.com/devtron-labs/common-lib/informer/bean.go new file mode 100644 index 0000000000..38aed85afc --- /dev/null +++ b/vendor/github.com/devtron-labs/common-lib/informer/bean.go @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package informer + +const ( + ClusterModifyEventSecretType = "cluster.request/modify" + ClusterActionAdd = "add" + ClusterActionUpdate = "update" + ClusterActionDelete = "delete" + SecretFieldAction = "action" + SecretFieldClusterId = "cluster_id" +) + +const ( + WorkflowTypeLabelKey = "workflowType" + CiWorkflowName = "ci" + CdWorkflowName = "cd" +) + +const ( + DevtronOwnerInstanceLabelKey = "devtron.ai/owner-instance" +) + +const ( + PodDeletedMessage = "pod deleted" +) diff --git a/client/telemetry/PosthogClient.go b/vendor/github.com/devtron-labs/common-lib/telemetry/PosthogClient.go similarity index 70% rename from client/telemetry/PosthogClient.go rename to vendor/github.com/devtron-labs/common-lib/telemetry/PosthogClient.go index 565470dc5c..79a8c72f88 100644 --- a/client/telemetry/PosthogClient.go +++ b/vendor/github.com/devtron-labs/common-lib/telemetry/PosthogClient.go @@ -17,11 +17,13 @@ package telemetry import ( + "context" "encoding/base64" "encoding/json" + "go.opentelemetry.io/otel" "time" - "github.com/devtron-labs/devtron/util" + "github.com/devtron-labs/common-lib/utils/http" "github.com/patrickmn/go-cache" "github.com/posthog/posthog-go" "go.uber.org/zap" @@ -32,26 +34,13 @@ type PosthogClient struct { cache *cache.Cache } -var ( - PosthogApiKey string = "" - PosthogEndpoint string = "https://app.posthog.com" - SummaryCronExpr string = "0 0 * * *" // Run once a day, midnight - HeartbeatCronExpr string = "0 0/6 * * *" - CacheExpiry int = 1440 - PosthogEncodedApiKey string = "" - IsOptOut bool = false -) - -const ( - TelemetryApiKeyEndpoint string = "aHR0cHM6Ly90ZWxlbWV0cnkuZGV2dHJvbi5haS9kZXZ0cm9uL3RlbGVtZXRyeS9wb3N0aG9nSW5mbw==" - TelemetryOptOutApiBaseUrl string = "aHR0cHM6Ly90ZWxlbWV0cnkuZGV2dHJvbi5haS9kZXZ0cm9uL3RlbGVtZXRyeS9vcHQtb3V0" - ResponseApiKey string = "PosthogApiKey" - ResponseUrlKey string = "PosthogEndpoint" -) +func (p *PosthogClient) GetCache() *cache.Cache { + return p.cache +} func NewPosthogClient(logger *zap.SugaredLogger) (*PosthogClient, error) { if PosthogApiKey == "" { - encodedApiKey, apiKey, posthogUrl, err := getPosthogApiKey(TelemetryApiKeyEndpoint, logger) + encodedApiKey, apiKey, posthogUrl, err := getPosthogApiKey(context.Background(), TelemetryApiKeyEndpoint, logger) if err != nil { logger.Errorw("exception caught while getting api key", "err", err) } else { @@ -75,14 +64,16 @@ func NewPosthogClient(logger *zap.SugaredLogger) (*PosthogClient, error) { return pgClient, nil } -func getPosthogApiKey(encodedPosthogApiKeyUrl string, logger *zap.SugaredLogger) (string, string, string, error) { +func getPosthogApiKey(ctx context.Context, encodedPosthogApiKeyUrl string, logger *zap.SugaredLogger) (string, string, string, error) { + newCtx, span := otel.Tracer("common").Start(ctx, "telemetry.getPosthogApiKey") + defer span.End() decodedPosthogApiKeyUrl, err := base64.StdEncoding.DecodeString(encodedPosthogApiKeyUrl) if err != nil { logger.Errorw("error fetching posthog api key, decode error", "err", err) return "", "", "", err } apiKeyUrl := string(decodedPosthogApiKeyUrl) - response, err := util.HttpRequest(apiKeyUrl) + response, err := http.HttpRequest(newCtx, apiKeyUrl) if err != nil { logger.Errorw("error fetching posthog api key, http call", "err", err) return "", "", "", err @@ -94,7 +85,7 @@ func getPosthogApiKey(encodedPosthogApiKeyUrl string, logger *zap.SugaredLogger) return "", "", "", err } var datamap map[string]string - if err := json.Unmarshal(posthogInfoByte, &datamap); err != nil { + if err = json.Unmarshal(posthogInfoByte, &datamap); err != nil { logger.Errorw("error while unmarshal data", "err", err) return "", "", "", err } diff --git a/vendor/github.com/devtron-labs/common-lib/telemetry/bean.go b/vendor/github.com/devtron-labs/common-lib/telemetry/bean.go new file mode 100644 index 0000000000..b85621d0f3 --- /dev/null +++ b/vendor/github.com/devtron-labs/common-lib/telemetry/bean.go @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package telemetry + +var ( + PosthogApiKey string = "" + PosthogEndpoint string = "https://app.posthog.com" + SummaryCronExpr string = "0 0 * * *" // Run once a day, midnight + HeartbeatCronExpr string = "0 0/6 * * *" + CacheExpiry int = 1440 + PosthogEncodedApiKey string = "" + IsOptOut bool = false +) + +const ( + TelemetryApiKeyEndpoint string = "aHR0cHM6Ly90ZWxlbWV0cnkuZGV2dHJvbi5haS9kZXZ0cm9uL3RlbGVtZXRyeS9wb3N0aG9nSW5mbw==" + TelemetryOptOutApiBaseUrl string = "aHR0cHM6Ly90ZWxlbWV0cnkuZGV2dHJvbi5haS9kZXZ0cm9uL3RlbGVtZXRyeS9vcHQtb3V0" + ResponseApiKey string = "PosthogApiKey" + ResponseUrlKey string = "PosthogEndpoint" +) diff --git a/vendor/github.com/devtron-labs/common-lib/utils/http/HttpUtil.go b/vendor/github.com/devtron-labs/common-lib/utils/http/HttpUtil.go index 933b97f4f9..9f8524e489 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/http/HttpUtil.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/http/HttpUtil.go @@ -17,11 +17,13 @@ package http import ( + "context" "crypto/tls" "crypto/x509" "encoding/json" "github.com/pkg/errors" - "io/ioutil" + "go.opentelemetry.io/otel" + "io" "net/http" "os" ) @@ -83,8 +85,10 @@ func CertPoolFromFile(filename string) (*x509.CertPool, error) { return cp, nil } -func HttpRequest(url string) (map[string]interface{}, error) { - req, err := http.NewRequest(http.MethodGet, url, nil) +func HttpRequest(ctx context.Context, url string) (map[string]interface{}, error) { + newCtx, span := otel.Tracer("common").Start(ctx, "http.HttpRequest") + defer span.End() + req, err := http.NewRequestWithContext(newCtx, http.MethodGet, url, nil) if err != nil { return nil, err } @@ -95,7 +99,7 @@ func HttpRequest(url string) (map[string]interface{}, error) { return nil, err } if res.StatusCode >= 200 && res.StatusCode <= 299 { - resBody, err := ioutil.ReadAll(res.Body) + resBody, err := io.ReadAll(res.Body) if err != nil { return nil, err } diff --git a/vendor/github.com/golang-jwt/jwt/v4/parser.go b/vendor/github.com/golang-jwt/jwt/v4/parser.go index 9dd36e5a5a..0fc510a0aa 100644 --- a/vendor/github.com/golang-jwt/jwt/v4/parser.go +++ b/vendor/github.com/golang-jwt/jwt/v4/parser.go @@ -7,6 +7,8 @@ import ( "strings" ) +const tokenDelimiter = "." + type Parser struct { // If populated, only these methods will be considered valid. // @@ -122,9 +124,10 @@ func (p *Parser) ParseWithClaims(tokenString string, claims Claims, keyFunc Keyf // It's only ever useful in cases where you know the signature is valid (because it has // been checked previously in the stack) and you want to extract values from it. func (p *Parser) ParseUnverified(tokenString string, claims Claims) (token *Token, parts []string, err error) { - parts = strings.Split(tokenString, ".") - if len(parts) != 3 { - return nil, parts, NewValidationError("token contains an invalid number of segments", ValidationErrorMalformed) + var ok bool + parts, ok = splitToken(tokenString) + if !ok { + return nil, nil, NewValidationError("token contains an invalid number of segments", ValidationErrorMalformed) } token = &Token{Raw: tokenString} @@ -174,3 +177,30 @@ func (p *Parser) ParseUnverified(tokenString string, claims Claims) (token *Toke return token, parts, nil } + +// splitToken splits a token string into three parts: header, claims, and signature. It will only +// return true if the token contains exactly two delimiters and three parts. In all other cases, it +// will return nil parts and false. +func splitToken(token string) ([]string, bool) { + parts := make([]string, 3) + header, remain, ok := strings.Cut(token, tokenDelimiter) + if !ok { + return nil, false + } + parts[0] = header + claims, remain, ok := strings.Cut(remain, tokenDelimiter) + if !ok { + return nil, false + } + parts[1] = claims + // One more cut to ensure the signature is the last part of the token and there are no more + // delimiters. This avoids an issue where malicious input could contain additional delimiters + // causing unecessary overhead parsing tokens. + signature, _, unexpected := strings.Cut(remain, tokenDelimiter) + if unexpected { + return nil, false + } + parts[2] = signature + + return parts, true +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 4a1cd3503d..436ab6735d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -472,7 +472,7 @@ github.com/davecgh/go-spew/spew # github.com/deckarep/golang-set v1.8.0 ## explicit; go 1.17 github.com/deckarep/golang-set -# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250421131910-ad3aa9bb920e +# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250422075757-fe1e5dd1b92b ## explicit; go 1.21 github.com/devtron-labs/authenticator/apiToken github.com/devtron-labs/authenticator/client @@ -480,7 +480,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421131910-ad3aa9bb920e +# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250422075757-fe1e5dd1b92b ## explicit; go 1.21 github.com/devtron-labs/common-lib/async github.com/devtron-labs/common-lib/blob-storage @@ -492,10 +492,12 @@ github.com/devtron-labs/common-lib/fetchAllEnv github.com/devtron-labs/common-lib/git-manager github.com/devtron-labs/common-lib/git-manager/util github.com/devtron-labs/common-lib/imageScan/bean +github.com/devtron-labs/common-lib/informer github.com/devtron-labs/common-lib/middlewares github.com/devtron-labs/common-lib/pubsub-lib github.com/devtron-labs/common-lib/pubsub-lib/metrics github.com/devtron-labs/common-lib/pubsub-lib/model +github.com/devtron-labs/common-lib/telemetry github.com/devtron-labs/common-lib/utils github.com/devtron-labs/common-lib/utils/bean github.com/devtron-labs/common-lib/utils/grpc @@ -699,7 +701,7 @@ github.com/gogo/protobuf/jsonpb github.com/gogo/protobuf/proto github.com/gogo/protobuf/sortkeys github.com/gogo/protobuf/types -# github.com/golang-jwt/jwt/v4 v4.5.1 +# github.com/golang-jwt/jwt/v4 v4.5.2 ## explicit; go 1.16 github.com/golang-jwt/jwt/v4 # github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da @@ -2350,8 +2352,8 @@ xorm.io/xorm/log xorm.io/xorm/names xorm.io/xorm/schemas xorm.io/xorm/tags -# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250421131910-ad3aa9bb920e -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421131910-ad3aa9bb920e +# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250422075757-fe1e5dd1b92b +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250422075757-fe1e5dd1b92b # github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127 # github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5 # k8s.io/api => k8s.io/api v0.29.7 diff --git a/wire_gen.go b/wire_gen.go index 78a257534d..60f4d5a649 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run -mod=mod github.com/google/wire/cmd/wire +//go:generate go run github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject @@ -12,6 +12,7 @@ import ( "github.com/devtron-labs/authenticator/middleware" "github.com/devtron-labs/common-lib/cloud-provider-identifier" "github.com/devtron-labs/common-lib/pubsub-lib" + "github.com/devtron-labs/common-lib/telemetry" "github.com/devtron-labs/common-lib/utils/grpc" "github.com/devtron-labs/common-lib/utils/k8s" apiToken2 "github.com/devtron-labs/devtron/api/apiToken" @@ -49,7 +50,7 @@ import ( "github.com/devtron-labs/devtron/api/restHandler/app/pipeline/configure" history2 "github.com/devtron-labs/devtron/api/restHandler/app/pipeline/history" status3 "github.com/devtron-labs/devtron/api/restHandler/app/pipeline/status" - "github.com/devtron-labs/devtron/api/restHandler/app/pipeline/trigger" + trigger2 "github.com/devtron-labs/devtron/api/restHandler/app/pipeline/trigger" "github.com/devtron-labs/devtron/api/restHandler/app/pipeline/webhook" "github.com/devtron-labs/devtron/api/restHandler/app/workflow" "github.com/devtron-labs/devtron/api/restHandler/scopedVariable" @@ -62,7 +63,7 @@ import ( configure2 "github.com/devtron-labs/devtron/api/router/app/pipeline/configure" history3 "github.com/devtron-labs/devtron/api/router/app/pipeline/history" status4 "github.com/devtron-labs/devtron/api/router/app/pipeline/status" - trigger2 "github.com/devtron-labs/devtron/api/router/app/pipeline/trigger" + trigger3 "github.com/devtron-labs/devtron/api/router/app/pipeline/trigger" workflow2 "github.com/devtron-labs/devtron/api/router/app/workflow" server2 "github.com/devtron-labs/devtron/api/server" "github.com/devtron-labs/devtron/api/sse" @@ -90,7 +91,7 @@ import ( "github.com/devtron-labs/devtron/client/grafana" "github.com/devtron-labs/devtron/client/lens" "github.com/devtron-labs/devtron/client/proxy" - "github.com/devtron-labs/devtron/client/telemetry" + telemetry2 "github.com/devtron-labs/devtron/client/telemetry" repository2 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/appStatus" @@ -100,7 +101,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/deploymentConfig" repository9 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/helper" - repository22 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" + repository23 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/resourceGroup" "github.com/devtron-labs/devtron/internal/util" @@ -147,14 +148,15 @@ import ( read21 "github.com/devtron-labs/devtron/pkg/build/git/gitHost/read" repository26 "github.com/devtron-labs/devtron/pkg/build/git/gitHost/repository" read15 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/read" - repository20 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/repository" + repository21 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/repository" "github.com/devtron-labs/devtron/pkg/build/git/gitProvider" read9 "github.com/devtron-labs/devtron/pkg/build/git/gitProvider/read" - repository11 "github.com/devtron-labs/devtron/pkg/build/git/gitProvider/repository" + repository12 "github.com/devtron-labs/devtron/pkg/build/git/gitProvider/repository" "github.com/devtron-labs/devtron/pkg/build/git/gitWebhook" - repository23 "github.com/devtron-labs/devtron/pkg/build/git/gitWebhook/repository" + repository11 "github.com/devtron-labs/devtron/pkg/build/git/gitWebhook/repository" pipeline2 "github.com/devtron-labs/devtron/pkg/build/pipeline" read14 "github.com/devtron-labs/devtron/pkg/build/pipeline/read" + "github.com/devtron-labs/devtron/pkg/build/trigger" service7 "github.com/devtron-labs/devtron/pkg/bulkAction/service" "github.com/devtron-labs/devtron/pkg/chart" "github.com/devtron-labs/devtron/pkg/chart/gitOpsConfig" @@ -184,7 +186,7 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/manifest/configMapAndSecret" read19 "github.com/devtron-labs/devtron/pkg/deployment/manifest/configMapAndSecret/read" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" - repository16 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" + repository17 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" read12 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/read" @@ -199,12 +201,13 @@ import ( "github.com/devtron-labs/devtron/pkg/devtronResource" "github.com/devtron-labs/devtron/pkg/devtronResource/history/deployment/cdPipeline" read10 "github.com/devtron-labs/devtron/pkg/devtronResource/read" - repository13 "github.com/devtron-labs/devtron/pkg/devtronResource/repository" + repository14 "github.com/devtron-labs/devtron/pkg/devtronResource/repository" "github.com/devtron-labs/devtron/pkg/dockerRegistry" "github.com/devtron-labs/devtron/pkg/eventProcessor" "github.com/devtron-labs/devtron/pkg/eventProcessor/celEvaluator" "github.com/devtron-labs/devtron/pkg/eventProcessor/in" "github.com/devtron-labs/devtron/pkg/eventProcessor/out" + "github.com/devtron-labs/devtron/pkg/executor" "github.com/devtron-labs/devtron/pkg/externalLink" "github.com/devtron-labs/devtron/pkg/fluxApplication" "github.com/devtron-labs/devtron/pkg/generateManifest" @@ -213,7 +216,7 @@ import ( "github.com/devtron-labs/devtron/pkg/gitops" "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" config4 "github.com/devtron-labs/devtron/pkg/infraConfig/config" - repository14 "github.com/devtron-labs/devtron/pkg/infraConfig/repository" + repository15 "github.com/devtron-labs/devtron/pkg/infraConfig/repository" "github.com/devtron-labs/devtron/pkg/infraConfig/repository/audit" service2 "github.com/devtron-labs/devtron/pkg/infraConfig/service" audit2 "github.com/devtron-labs/devtron/pkg/infraConfig/service/audit" @@ -233,21 +236,21 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/draftAwareConfigService" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/history" - repository21 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" + repository22 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders" "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders/infraGetters/ci" "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders/infraGetters/job" - repository18 "github.com/devtron-labs/devtron/pkg/pipeline/repository" + repository19 "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" - repository17 "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus/repository" + repository18 "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus/repository" "github.com/devtron-labs/devtron/pkg/plugin" - repository19 "github.com/devtron-labs/devtron/pkg/plugin/repository" + repository20 "github.com/devtron-labs/devtron/pkg/plugin/repository" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning" read18 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/read" repository24 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/repository" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool" - repository15 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" + repository16 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" "github.com/devtron-labs/devtron/pkg/server" @@ -258,11 +261,12 @@ import ( read4 "github.com/devtron-labs/devtron/pkg/team/read" repository8 "github.com/devtron-labs/devtron/pkg/team/repository" "github.com/devtron-labs/devtron/pkg/terminal" + "github.com/devtron-labs/devtron/pkg/ucid" "github.com/devtron-labs/devtron/pkg/userResource" util3 "github.com/devtron-labs/devtron/pkg/util" "github.com/devtron-labs/devtron/pkg/variables" "github.com/devtron-labs/devtron/pkg/variables/parsers" - repository12 "github.com/devtron-labs/devtron/pkg/variables/repository" + repository13 "github.com/devtron-labs/devtron/pkg/variables/repository" "github.com/devtron-labs/devtron/pkg/webhook/helm" "github.com/devtron-labs/devtron/pkg/workflow/cd" read20 "github.com/devtron-labs/devtron/pkg/workflow/cd/read" @@ -487,11 +491,12 @@ func InitializeApp() (*App, error) { clusterRbacServiceImpl := rbac2.NewClusterRbacServiceImpl(environmentServiceImpl, enforcerImpl, enforcerUtilImpl, clusterServiceImplExtended, sugaredLogger, userServiceImpl, clusterReadServiceImpl) clusterRestHandlerImpl := cluster3.NewClusterRestHandlerImpl(clusterServiceImplExtended, genericNoteServiceImpl, clusterDescriptionServiceImpl, sugaredLogger, userServiceImpl, validate, enforcerImpl, deleteServiceExtendedImpl, environmentServiceImpl, clusterRbacServiceImpl) clusterRouterImpl := cluster3.NewClusterRouterImpl(clusterRestHandlerImpl) + gitWebhookRepositoryImpl := repository11.NewGitWebhookRepositoryImpl(db) ciCdConfig, err := types.GetCiCdConfig() if err != nil { return nil, err } - gitProviderRepositoryImpl := repository11.NewGitProviderRepositoryImpl(db) + gitProviderRepositoryImpl := repository12.NewGitProviderRepositoryImpl(db) gitProviderReadServiceImpl := read9.NewGitProviderReadService(sugaredLogger, gitProviderRepositoryImpl) commonBaseServiceImpl := commonService.NewCommonBaseServiceImpl(sugaredLogger, environmentVariables, moduleReadServiceImpl) commonServiceImpl := commonService.NewCommonServiceImpl(sugaredLogger, chartRepositoryImpl, envConfigOverrideRepositoryImpl, dockerArtifactStoreRepositoryImpl, attributesRepositoryImpl, environmentRepositoryImpl, appRepositoryImpl, gitOpsConfigReadServiceImpl, gitProviderReadServiceImpl, envConfigOverrideReadServiceImpl, commonBaseServiceImpl, teamReadServiceImpl) @@ -499,8 +504,8 @@ func InitializeApp() (*App, error) { mergeUtil := util.MergeUtil{ Logger: sugaredLogger, } - scopedVariableRepositoryImpl := repository12.NewScopedVariableRepository(db, sugaredLogger, transactionUtilImpl) - devtronResourceSearchableKeyRepositoryImpl := repository13.NewDevtronResourceSearchableKeyRepositoryImpl(sugaredLogger, db) + scopedVariableRepositoryImpl := repository13.NewScopedVariableRepository(db, sugaredLogger, transactionUtilImpl) + devtronResourceSearchableKeyRepositoryImpl := repository14.NewDevtronResourceSearchableKeyRepositoryImpl(sugaredLogger, db) devtronResourceSearchableKeyServiceImpl, err := read10.NewDevtronResourceSearchableKeyServiceImpl(sugaredLogger, devtronResourceSearchableKeyRepositoryImpl) if err != nil { return nil, err @@ -517,9 +522,9 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - variableEntityMappingRepositoryImpl := repository12.NewVariableEntityMappingRepository(sugaredLogger, db, transactionUtilImpl) + variableEntityMappingRepositoryImpl := repository13.NewVariableEntityMappingRepository(sugaredLogger, db, transactionUtilImpl) variableEntityMappingServiceImpl := variables.NewVariableEntityMappingServiceImpl(variableEntityMappingRepositoryImpl, sugaredLogger) - variableSnapshotHistoryRepositoryImpl := repository12.NewVariableSnapshotHistoryRepository(sugaredLogger, db) + variableSnapshotHistoryRepositoryImpl := repository13.NewVariableSnapshotHistoryRepository(sugaredLogger, db) variableSnapshotHistoryServiceImpl := variables.NewVariableSnapshotHistoryServiceImpl(variableSnapshotHistoryRepositoryImpl, sugaredLogger) variableTemplateParserImpl, err := parsers.NewVariableTemplateParserImpl(sugaredLogger) if err != nil { @@ -540,7 +545,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - infraConfigRepositoryImpl := repository14.NewInfraProfileRepositoryImpl(db, transactionUtilImpl) + infraConfigRepositoryImpl := repository15.NewInfraProfileRepositoryImpl(db, transactionUtilImpl) pipelineOverrideRepositoryImpl := chartConfig.NewPipelineOverrideRepository(db) utilMergeUtil := &util.MergeUtil{ Logger: sugaredLogger, @@ -573,7 +578,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - scanToolMetadataRepositoryImpl := repository15.NewScanToolMetadataRepositoryImpl(db, sugaredLogger) + scanToolMetadataRepositoryImpl := repository16.NewScanToolMetadataRepositoryImpl(db, sugaredLogger) scanToolMetadataServiceImpl := scanTool.NewScanToolMetadataServiceImpl(sugaredLogger, scanToolMetadataRepositoryImpl) moduleServiceImpl := module.NewModuleServiceImpl(sugaredLogger, serverEnvConfigServerEnvConfig, moduleRepositoryImpl, moduleActionAuditLogRepositoryImpl, helmAppServiceImpl, serverDataStoreServerDataStore, serverCacheServiceImpl, moduleCacheServiceImpl, moduleCronServiceImpl, moduleServiceHelperImpl, moduleResourceStatusRepositoryImpl, scanToolMetadataServiceImpl, environmentVariables, moduleEnvConfig) eventRESTClientImpl := client2.NewEventRESTClientImpl(sugaredLogger, httpClient, eventClientConfig, pubSubClientServiceImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, attributesRepositoryImpl, moduleServiceImpl) @@ -607,11 +612,11 @@ func InitializeApp() (*App, error) { ciTemplateOverrideRepositoryImpl := pipelineConfig.NewCiTemplateOverrideRepositoryImpl(db, sugaredLogger) ciPipelineConfigReadServiceImpl := read14.NewCiPipelineConfigReadServiceImpl(sugaredLogger, ciPipelineRepositoryImpl, ciTemplateOverrideRepositoryImpl) dockerRegistryIpsConfigServiceImpl := dockerRegistry.NewDockerRegistryIpsConfigServiceImpl(sugaredLogger, dockerRegistryIpsConfigRepositoryImpl, k8sServiceImpl, dockerArtifactStoreRepositoryImpl, clusterReadServiceImpl, ciPipelineConfigReadServiceImpl) - appLevelMetricsRepositoryImpl := repository16.NewAppLevelMetricsRepositoryImpl(db, sugaredLogger) - envLevelAppMetricsRepositoryImpl := repository16.NewEnvLevelAppMetricsRepositoryImpl(db, sugaredLogger) + appLevelMetricsRepositoryImpl := repository17.NewAppLevelMetricsRepositoryImpl(db, sugaredLogger) + envLevelAppMetricsRepositoryImpl := repository17.NewEnvLevelAppMetricsRepositoryImpl(db, sugaredLogger) deployedAppMetricsServiceImpl := deployedAppMetrics.NewDeployedAppMetricsServiceImpl(sugaredLogger, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, chartRefServiceImpl) appListingServiceImpl := app2.NewAppListingServiceImpl(sugaredLogger, appListingRepositoryImpl, appDetailsReadServiceImpl, appRepositoryImpl, appListingViewBuilderImpl, pipelineRepositoryImpl, linkoutsRepositoryImpl, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, environmentRepositoryImpl, chartRepositoryImpl, ciPipelineRepositoryImpl, dockerRegistryIpsConfigServiceImpl, userRepositoryImpl, deployedAppMetricsServiceImpl, ciArtifactRepositoryImpl, envConfigOverrideReadServiceImpl, ciPipelineConfigReadServiceImpl) - workflowStageRepositoryImpl := repository17.NewWorkflowStageRepositoryImpl(sugaredLogger, db) + workflowStageRepositoryImpl := repository18.NewWorkflowStageRepositoryImpl(sugaredLogger, db) workFlowStageStatusServiceImpl := workflowStatus.NewWorkflowStageFlowStatusServiceImpl(sugaredLogger, workflowStageRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, transactionUtilImpl) cdWorkflowRunnerServiceImpl := cd.NewCdWorkflowRunnerServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl, workFlowStageStatusServiceImpl, transactionUtilImpl) appServiceImpl := app2.NewAppService(pipelineOverrideRepositoryImpl, utilMergeUtil, sugaredLogger, pipelineRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, appRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, chartTemplateServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, appStatusServiceImpl, installedAppReadServiceImpl, installedAppVersionHistoryRepositoryImpl, scopedVariableCMCSManagerImpl, acdConfig, gitOpsConfigReadServiceImpl, gitOperationServiceImpl, deploymentTemplateServiceImpl, appListingServiceImpl, deploymentConfigServiceImpl, envConfigOverrideReadServiceImpl, cdWorkflowRunnerServiceImpl) @@ -626,12 +631,13 @@ func InitializeApp() (*App, error) { } ciInfraGetter := ci.NewCiInfraGetter(sugaredLogger, infraConfigServiceImpl, infraConfigAuditServiceImpl) infraProviderImpl := infraProviders.NewInfraProviderImpl(sugaredLogger, infraGetter, ciInfraGetter) - workflowServiceImpl, err := pipeline.NewWorkflowServiceImpl(sugaredLogger, environmentRepositoryImpl, ciCdConfig, configReadServiceImpl, globalCMCSServiceImpl, argoWorkflowExecutorImpl, k8sServiceImpl, systemWorkflowExecutorImpl, k8sCommonServiceImpl, infraProviderImpl) + serviceImpl := ucid.NewServiceImpl(sugaredLogger, k8sServiceImpl, acdAuthConfig) + workflowServiceImpl, err := executor.NewWorkflowServiceImpl(sugaredLogger, environmentRepositoryImpl, ciCdConfig, configReadServiceImpl, globalCMCSServiceImpl, argoWorkflowExecutorImpl, systemWorkflowExecutorImpl, k8sCommonServiceImpl, infraProviderImpl, serviceImpl, k8sServiceImpl) if err != nil { return nil, err } - pipelineStageRepositoryImpl := repository18.NewPipelineStageRepository(sugaredLogger, db) - globalPluginRepositoryImpl := repository19.NewGlobalPluginRepository(sugaredLogger, db) + pipelineStageRepositoryImpl := repository19.NewPipelineStageRepository(sugaredLogger, db) + globalPluginRepositoryImpl := repository20.NewGlobalPluginRepository(sugaredLogger, db) globalPluginServiceImpl := plugin.NewGlobalPluginService(sugaredLogger, globalPluginRepositoryImpl, pipelineStageRepositoryImpl, userServiceImpl) pipelineStageServiceImpl := pipeline.NewPipelineStageService(sugaredLogger, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, pipelineRepositoryImpl, scopedVariableManagerImpl, globalPluginServiceImpl) ciTemplateRepositoryImpl := pipelineConfig.NewCiTemplateRepositoryImpl(db, sugaredLogger) @@ -641,12 +647,11 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - materialRepositoryImpl := repository20.NewMaterialRepositoryImpl(db) + materialRepositoryImpl := repository21.NewMaterialRepositoryImpl(db) gitMaterialReadServiceImpl := read15.NewGitMaterialReadServiceImpl(sugaredLogger, materialRepositoryImpl) appCrudOperationServiceImpl := app2.NewAppCrudOperationServiceImpl(appLabelRepositoryImpl, sugaredLogger, appRepositoryImpl, userRepositoryImpl, installedAppRepositoryImpl, genericNoteServiceImpl, installedAppDBServiceImpl, crudOperationServiceConfig, dbMigrationServiceImpl, gitMaterialReadServiceImpl) imageTagRepositoryImpl := repository2.NewImageTagRepository(db, sugaredLogger) customTagServiceImpl := pipeline.NewCustomTagService(sugaredLogger, imageTagRepositoryImpl) - pluginInputVariableParserImpl := pipeline.NewPluginInputVariableParserImpl(sugaredLogger, dockerRegistryConfigImpl, customTagServiceImpl) clientConfig, err := gitSensor.GetConfig() if err != nil { return nil, err @@ -655,54 +660,54 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - prePostCdScriptHistoryRepositoryImpl := repository21.NewPrePostCdScriptHistoryRepositoryImpl(sugaredLogger, db) - configMapHistoryRepositoryImpl := repository21.NewConfigMapHistoryRepositoryImpl(sugaredLogger, db, transactionUtilImpl) + prePostCdScriptHistoryRepositoryImpl := repository22.NewPrePostCdScriptHistoryRepositoryImpl(sugaredLogger, db) + configMapHistoryRepositoryImpl := repository22.NewConfigMapHistoryRepositoryImpl(sugaredLogger, db, transactionUtilImpl) configMapHistoryServiceImpl := configMapAndSecret.NewConfigMapHistoryServiceImpl(sugaredLogger, configMapHistoryRepositoryImpl, pipelineRepositoryImpl, configMapRepositoryImpl, userServiceImpl, scopedVariableCMCSManagerImpl) prePostCdScriptHistoryServiceImpl := history.NewPrePostCdScriptHistoryServiceImpl(sugaredLogger, prePostCdScriptHistoryRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl) - gitMaterialHistoryRepositoryImpl := repository21.NewGitMaterialHistoryRepositoyImpl(db) + gitMaterialHistoryRepositoryImpl := repository22.NewGitMaterialHistoryRepositoyImpl(db) gitMaterialHistoryServiceImpl := history.NewGitMaterialHistoryServiceImpl(gitMaterialHistoryRepositoryImpl, sugaredLogger) - ciPipelineHistoryRepositoryImpl := repository21.NewCiPipelineHistoryRepositoryImpl(db, sugaredLogger) + ciPipelineHistoryRepositoryImpl := repository22.NewCiPipelineHistoryRepositoryImpl(db, sugaredLogger) ciPipelineHistoryServiceImpl := history.NewCiPipelineHistoryServiceImpl(ciPipelineHistoryRepositoryImpl, sugaredLogger, ciPipelineRepositoryImpl) ciBuildConfigRepositoryImpl := pipelineConfig.NewCiBuildConfigRepositoryImpl(db, sugaredLogger) ciBuildConfigServiceImpl := pipeline.NewCiBuildConfigServiceImpl(sugaredLogger, ciBuildConfigRepositoryImpl) ciTemplateServiceImpl := pipeline.NewCiTemplateServiceImpl(sugaredLogger, ciBuildConfigServiceImpl, ciTemplateRepositoryImpl, ciTemplateOverrideRepositoryImpl) pipelineConfigRepositoryImpl := chartConfig.NewPipelineConfigRepository(db) configMapServiceImpl := pipeline.NewConfigMapServiceImpl(chartRepositoryImpl, sugaredLogger, chartRepoRepositoryImpl, mergeUtil, pipelineConfigRepositoryImpl, configMapRepositoryImpl, commonServiceImpl, appRepositoryImpl, configMapHistoryServiceImpl, environmentRepositoryImpl, scopedVariableCMCSManagerImpl) - deploymentTemplateHistoryRepositoryImpl := repository21.NewDeploymentTemplateHistoryRepositoryImpl(sugaredLogger, db) + deploymentTemplateHistoryRepositoryImpl := repository22.NewDeploymentTemplateHistoryRepositoryImpl(sugaredLogger, db) deploymentTemplateHistoryServiceImpl := deploymentTemplate.NewDeploymentTemplateHistoryServiceImpl(sugaredLogger, deploymentTemplateHistoryRepositoryImpl, pipelineRepositoryImpl, chartRepositoryImpl, userServiceImpl, cdWorkflowRepositoryImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) chartReadServiceImpl := read16.NewChartReadServiceImpl(sugaredLogger, chartRepositoryImpl, deploymentConfigServiceImpl, deployedAppMetricsServiceImpl, gitOpsConfigReadServiceImpl, chartRefReadServiceImpl) chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, mergeUtil, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl, envConfigOverrideReadServiceImpl, chartReadServiceImpl) ciCdPipelineOrchestratorImpl := pipeline.NewCiCdPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, clientImpl, ciCdConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, pipelineStageServiceImpl, gitMaterialHistoryServiceImpl, ciPipelineHistoryServiceImpl, ciTemplateReadServiceImpl, ciTemplateServiceImpl, dockerArtifactStoreRepositoryImpl, ciArtifactRepositoryImpl, configMapServiceImpl, customTagServiceImpl, genericNoteServiceImpl, chartServiceImpl, transactionUtilImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl, deploymentConfigReadServiceImpl, chartReadServiceImpl) - ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, workFlowStageStatusServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateReadServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl, infraProviderImpl, ciCdPipelineOrchestratorImpl, attributesServiceImpl, ciWorkflowRepositoryImpl, transactionUtilImpl) - ciLogServiceImpl, err := pipeline.NewCiLogServiceImpl(sugaredLogger, ciServiceImpl, k8sServiceImpl) - if err != nil { - return nil, err - } - resourceGroupRepositoryImpl := resourceGroup.NewResourceGroupRepositoryImpl(db) - resourceGroupMappingRepositoryImpl := resourceGroup.NewResourceGroupMappingRepositoryImpl(db) - resourceGroupServiceImpl := resourceGroup2.NewResourceGroupServiceImpl(sugaredLogger, resourceGroupRepositoryImpl, resourceGroupMappingRepositoryImpl, enforcerUtilImpl, devtronResourceSearchableKeyServiceImpl, appStatusRepositoryImpl) - imageTaggingRepositoryImpl := repository22.NewImageTaggingRepositoryImpl(db, transactionUtilImpl) - imageTaggingReadServiceImpl, err := read17.NewImageTaggingReadServiceImpl(imageTaggingRepositoryImpl, sugaredLogger) + pluginInputVariableParserImpl := pipeline.NewPluginInputVariableParserImpl(sugaredLogger, dockerRegistryConfigImpl, customTagServiceImpl) + ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workFlowStageStatusServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciWorkflowRepositoryImpl, transactionUtilImpl) + ciLogServiceImpl, err := pipeline.NewCiLogServiceImpl(sugaredLogger, k8sServiceImpl) if err != nil { return nil, err } - imageTaggingServiceImpl := imageTagging.NewImageTaggingServiceImpl(imageTaggingRepositoryImpl, imageTaggingReadServiceImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, sugaredLogger) blobStorageConfigServiceImpl := pipeline.NewBlobStorageConfigServiceImpl(sugaredLogger, k8sServiceImpl, ciCdConfig) - ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, workflowServiceImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, k8sServiceImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, environmentServiceImpl, workFlowStageStatusServiceImpl) - gitWebhookRepositoryImpl := repository23.NewGitWebhookRepositoryImpl(db) - gitWebhookServiceImpl := gitWebhook.NewGitWebhookServiceImpl(sugaredLogger, ciHandlerImpl, gitWebhookRepositoryImpl) + handlerServiceImpl := trigger.NewHandlerServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciPipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateReadServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, ciCdPipelineOrchestratorImpl, attributesServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl, ciServiceImpl, ciWorkflowRepositoryImpl, clientImpl, ciLogServiceImpl, blobStorageConfigServiceImpl, clusterServiceImplExtended, environmentServiceImpl, k8sServiceImpl) + gitWebhookServiceImpl := gitWebhook.NewGitWebhookServiceImpl(sugaredLogger, gitWebhookRepositoryImpl, handlerServiceImpl) gitWebhookRestHandlerImpl := restHandler.NewGitWebhookRestHandlerImpl(sugaredLogger, gitWebhookServiceImpl) ecrConfig, err := pipeline.GetEcrConfig() if err != nil { return nil, err } - ciTemplateHistoryRepositoryImpl := repository21.NewCiTemplateHistoryRepositoryImpl(db, sugaredLogger) + ciTemplateHistoryRepositoryImpl := repository22.NewCiTemplateHistoryRepositoryImpl(db, sugaredLogger) ciTemplateHistoryServiceImpl := history.NewCiTemplateHistoryServiceImpl(ciTemplateHistoryRepositoryImpl, sugaredLogger) + resourceGroupRepositoryImpl := resourceGroup.NewResourceGroupRepositoryImpl(db) + resourceGroupMappingRepositoryImpl := resourceGroup.NewResourceGroupMappingRepositoryImpl(db) + resourceGroupServiceImpl := resourceGroup2.NewResourceGroupServiceImpl(sugaredLogger, resourceGroupRepositoryImpl, resourceGroupMappingRepositoryImpl, enforcerUtilImpl, devtronResourceSearchableKeyServiceImpl, appStatusRepositoryImpl) buildPipelineSwitchServiceImpl := pipeline.NewBuildPipelineSwitchServiceImpl(sugaredLogger, ciPipelineConfigReadServiceImpl, ciPipelineRepositoryImpl, ciCdPipelineOrchestratorImpl, pipelineRepositoryImpl, ciWorkflowRepositoryImpl, appWorkflowRepositoryImpl, ciPipelineHistoryServiceImpl, ciTemplateOverrideRepositoryImpl, ciPipelineMaterialRepositoryImpl) ciPipelineConfigServiceImpl := pipeline.NewCiPipelineConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, gitMaterialReadServiceImpl, appRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigReadServiceImpl, ciPipelineRepositoryImpl, ecrConfig, appWorkflowRepositoryImpl, ciCdConfig, attributesServiceImpl, pipelineStageServiceImpl, ciPipelineMaterialRepositoryImpl, ciTemplateServiceImpl, ciTemplateReadServiceImpl, ciTemplateOverrideRepositoryImpl, ciTemplateHistoryServiceImpl, enforcerUtilImpl, ciWorkflowRepositoryImpl, resourceGroupServiceImpl, customTagServiceImpl, cdWorkflowRepositoryImpl, buildPipelineSwitchServiceImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, appListingServiceImpl) ciMaterialConfigServiceImpl := pipeline.NewCiMaterialConfigServiceImpl(sugaredLogger, materialRepositoryImpl, ciTemplateReadServiceImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, gitMaterialHistoryServiceImpl, pipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, transactionUtilImpl, gitMaterialReadServiceImpl) + imageTaggingRepositoryImpl := repository23.NewImageTaggingRepositoryImpl(db, transactionUtilImpl) + imageTaggingReadServiceImpl, err := read17.NewImageTaggingReadServiceImpl(imageTaggingRepositoryImpl, sugaredLogger) + if err != nil { + return nil, err + } + imageTaggingServiceImpl := imageTagging.NewImageTaggingServiceImpl(imageTaggingRepositoryImpl, imageTaggingReadServiceImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, sugaredLogger) deploymentGroupRepositoryImpl := repository2.NewDeploymentGroupRepositoryImpl(sugaredLogger, db) - pipelineStrategyHistoryRepositoryImpl := repository21.NewPipelineStrategyHistoryRepositoryImpl(sugaredLogger, db) + pipelineStrategyHistoryRepositoryImpl := repository22.NewPipelineStrategyHistoryRepositoryImpl(sugaredLogger, db) pipelineStrategyHistoryServiceImpl := history.NewPipelineStrategyHistoryServiceImpl(sugaredLogger, pipelineStrategyHistoryRepositoryImpl, userServiceImpl) propertiesConfigServiceImpl := pipeline.NewPropertiesConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, envConfigOverrideReadServiceImpl, deploymentConfigServiceImpl, chartServiceImpl) installedAppDBExtendedServiceImpl := FullMode.NewInstalledAppDBExtendedServiceImpl(installedAppDBServiceImpl, appStatusServiceImpl, gitOpsConfigReadServiceImpl) @@ -733,7 +738,8 @@ func InitializeApp() (*App, error) { deploymentTemplateValidationServiceEntImpl := validator.NewDeploymentTemplateValidationServiceEntImpl() deploymentTemplateValidationServiceImpl := validator.NewDeploymentTemplateValidationServiceImpl(sugaredLogger, chartRefServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, deploymentTemplateValidationServiceEntImpl) devtronAppGitOpConfigServiceImpl := gitOpsConfig.NewDevtronAppGitOpConfigServiceImpl(sugaredLogger, chartRepositoryImpl, chartServiceImpl, gitOpsConfigReadServiceImpl, gitOpsValidationServiceImpl, argoClientWrapperServiceImpl, deploymentConfigServiceImpl, chartReadServiceImpl) - cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, workflowServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, customTagServiceImpl, deploymentConfigServiceImpl, workFlowStageStatusServiceImpl, cdWorkflowRunnerServiceImpl) + ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, workFlowStageStatusServiceImpl) + cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, customTagServiceImpl, deploymentConfigServiceImpl, workFlowStageStatusServiceImpl, cdWorkflowRunnerServiceImpl) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl, chartServiceImpl, deploymentConfigServiceImpl, pipelineBuilderImpl) appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, attributesServiceImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, pipelineStageServiceImpl, ciTemplateReadServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigServiceImpl, gitOpsConfigReadServiceImpl, chartReadServiceImpl) deploymentTemplateRepositoryImpl := repository2.NewDeploymentTemplateRepositoryImpl(db, sugaredLogger) @@ -752,7 +758,6 @@ func InitializeApp() (*App, error) { policyServiceImpl := imageScanning.NewPolicyServiceImpl(environmentServiceImpl, sugaredLogger, appRepositoryImpl, pipelineOverrideRepositoryImpl, cvePolicyRepositoryImpl, clusterServiceImplExtended, pipelineRepositoryImpl, imageScanResultRepositoryImpl, imageScanDeployInfoRepositoryImpl, imageScanObjectMetaRepositoryImpl, httpClient, ciArtifactRepositoryImpl, ciCdConfig, imageScanHistoryReadServiceImpl, cveStoreRepositoryImpl, ciTemplateRepositoryImpl, clusterReadServiceImpl, transactionUtilImpl) imageScanResultReadServiceImpl := read18.NewImageScanResultReadServiceImpl(sugaredLogger, imageScanResultRepositoryImpl) draftAwareConfigServiceImpl := draftAwareConfigService.NewDraftAwareResourceServiceImpl(sugaredLogger, configMapServiceImpl, chartServiceImpl, propertiesConfigServiceImpl) - pipelineConfigRestHandlerImpl := configure.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, deploymentTemplateValidationServiceImpl, chartServiceImpl, devtronAppGitOpConfigServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, generateManifestDeploymentTemplateServiceImpl, appWorkflowServiceImpl, gitMaterialReadServiceImpl, policyServiceImpl, imageScanResultReadServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingReadServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, ciCdPipelineOrchestratorImpl, gitProviderReadServiceImpl, teamReadServiceImpl, environmentRepositoryImpl, chartReadServiceImpl, draftAwareConfigServiceImpl) gitOpsManifestPushServiceImpl := publish.NewGitOpsManifestPushServiceImpl(sugaredLogger, pipelineStatusTimelineServiceImpl, pipelineOverrideRepositoryImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, chartServiceImpl, gitOperationServiceImpl, argoClientWrapperServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, chartTemplateServiceImpl) manifestCreationServiceImpl := manifest.NewManifestCreationServiceImpl(sugaredLogger, dockerRegistryIpsConfigServiceImpl, chartRefServiceImpl, scopedVariableCMCSManagerImpl, k8sCommonServiceImpl, deployedAppMetricsServiceImpl, imageDigestPolicyServiceImpl, utilMergeUtil, appCrudOperationServiceImpl, deploymentTemplateServiceImpl, argoClientWrapperServiceImpl, configMapHistoryRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineOverrideRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, pipelineConfigRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, deploymentConfigServiceImpl, envConfigOverrideReadServiceImpl) configMapHistoryReadServiceImpl := read19.NewConfigMapHistoryReadService(sugaredLogger, configMapHistoryRepositoryImpl, scopedVariableCMCSManagerImpl) @@ -761,16 +766,17 @@ func InitializeApp() (*App, error) { userDeploymentRequestServiceImpl := service3.NewUserDeploymentRequestServiceImpl(sugaredLogger, userDeploymentRequestRepositoryImpl) imageScanDeployInfoReadServiceImpl := read18.NewImageScanDeployInfoReadService(sugaredLogger, imageScanDeployInfoRepositoryImpl) imageScanDeployInfoServiceImpl := imageScanning.NewImageScanDeployInfoService(sugaredLogger, imageScanDeployInfoRepositoryImpl) - manifestPushConfigRepositoryImpl := repository18.NewManifestPushConfigRepository(sugaredLogger, db) + manifestPushConfigRepositoryImpl := repository19.NewManifestPushConfigRepository(sugaredLogger, db) scanToolExecutionHistoryMappingRepositoryImpl := repository24.NewScanToolExecutionHistoryMappingRepositoryImpl(db, sugaredLogger) cdWorkflowReadServiceImpl := read20.NewCdWorkflowReadServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) imageScanServiceImpl := imageScanning.NewImageScanServiceImpl(sugaredLogger, imageScanHistoryRepositoryImpl, imageScanResultRepositoryImpl, imageScanObjectMetaRepositoryImpl, cveStoreRepositoryImpl, imageScanDeployInfoRepositoryImpl, userServiceImpl, appRepositoryImpl, environmentServiceImpl, ciArtifactRepositoryImpl, policyServiceImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, scanToolMetadataRepositoryImpl, scanToolExecutionHistoryMappingRepositoryImpl, cvePolicyRepositoryImpl, cdWorkflowReadServiceImpl) - triggerServiceImpl, err := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, workflowServiceImpl, imageDigestPolicyServiceImpl, userServiceImpl, clientImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryReadServiceImpl, imageScanDeployInfoReadServiceImpl, imageScanDeployInfoServiceImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateReadServiceImpl, gitMaterialReadServiceImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, ciCdPipelineOrchestratorImpl, gitOperationServiceImpl, attributesServiceImpl, clusterRepositoryImpl, cdWorkflowRunnerServiceImpl) + devtronAppsHandlerServiceImpl, err := devtronApps.NewHandlerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, imageDigestPolicyServiceImpl, userServiceImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryReadServiceImpl, imageScanDeployInfoReadServiceImpl, imageScanDeployInfoServiceImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateReadServiceImpl, gitMaterialReadServiceImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, ciCdPipelineOrchestratorImpl, gitOperationServiceImpl, attributesServiceImpl, clusterRepositoryImpl, cdWorkflowRunnerServiceImpl, clusterServiceImplExtended, ciLogServiceImpl, workflowServiceImpl, blobStorageConfigServiceImpl) if err != nil { return nil, err } + pipelineConfigRestHandlerImpl := configure.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, deploymentTemplateValidationServiceImpl, chartServiceImpl, devtronAppGitOpConfigServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, generateManifestDeploymentTemplateServiceImpl, appWorkflowServiceImpl, gitMaterialReadServiceImpl, policyServiceImpl, imageScanResultReadServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingReadServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, ciCdPipelineOrchestratorImpl, gitProviderReadServiceImpl, teamReadServiceImpl, environmentRepositoryImpl, chartReadServiceImpl, draftAwareConfigServiceImpl, handlerServiceImpl, devtronAppsHandlerServiceImpl) commonArtifactServiceImpl := artifacts.NewCommonArtifactServiceImpl(sugaredLogger, ciArtifactRepositoryImpl) - workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, cdWorkflowRunnerServiceImpl, ciServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl, runnable, imageScanHistoryRepositoryImpl, imageScanServiceImpl) + workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, cdWorkflowRunnerServiceImpl, ciServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, devtronAppsHandlerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl, runnable, imageScanHistoryRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, environmentRepositoryImpl, k8sCommonServiceImpl, workflowServiceImpl, handlerServiceImpl) externalCiRestHandlerImpl := restHandler.NewExternalCiRestHandlerImpl(sugaredLogger, validate, userServiceImpl, enforcerImpl, workflowDagExecutorImpl) pubSubClientRestHandlerImpl := restHandler.NewPubSubClientRestHandlerImpl(pubSubClientServiceImpl, sugaredLogger, ciCdConfig) webhookRouterImpl := router.NewWebhookRouterImpl(gitWebhookRestHandlerImpl, pipelineConfigRestHandlerImpl, externalCiRestHandlerImpl, pubSubClientRestHandlerImpl) @@ -933,15 +939,15 @@ func InitializeApp() (*App, error) { return nil, err } providerIdentifierServiceImpl := providerIdentifier.NewProviderIdentifierServiceImpl(sugaredLogger) - telemetryEventClientImplExtended, err := telemetry.NewTelemetryEventClientImplExtended(sugaredLogger, httpClient, clusterServiceImplExtended, k8sServiceImpl, acdAuthConfig, environmentServiceImpl, userServiceImpl, appListingRepositoryImpl, posthogClient, ciPipelineConfigReadServiceImpl, pipelineRepositoryImpl, gitProviderRepositoryImpl, attributesRepositoryImpl, ssoLoginServiceImpl, appRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, gitMaterialReadServiceImpl, ciTemplateRepositoryImpl, chartRepositoryImpl, userAuditServiceImpl, ciBuildConfigServiceImpl, moduleRepositoryImpl, serverDataStoreServerDataStore, helmAppClientImpl, installedAppReadServiceImpl, userAttributesRepositoryImpl, providerIdentifierServiceImpl, cronLoggerImpl, gitOpsConfigReadServiceImpl, environmentVariables) + telemetryEventClientImplExtended, err := telemetry2.NewTelemetryEventClientImplExtended(sugaredLogger, httpClient, clusterServiceImplExtended, k8sServiceImpl, acdAuthConfig, environmentServiceImpl, userServiceImpl, appListingRepositoryImpl, posthogClient, serviceImpl, ciPipelineConfigReadServiceImpl, pipelineRepositoryImpl, gitProviderRepositoryImpl, attributesRepositoryImpl, ssoLoginServiceImpl, appRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, gitMaterialReadServiceImpl, ciTemplateRepositoryImpl, chartRepositoryImpl, userAuditServiceImpl, ciBuildConfigServiceImpl, moduleRepositoryImpl, serverDataStoreServerDataStore, helmAppClientImpl, installedAppReadServiceImpl, userAttributesRepositoryImpl, providerIdentifierServiceImpl, cronLoggerImpl, gitOpsConfigReadServiceImpl, environmentVariables) if err != nil { return nil, err } telemetryRestHandlerImpl := restHandler.NewTelemetryRestHandlerImpl(sugaredLogger, telemetryEventClientImplExtended, enforcerImpl, userServiceImpl) telemetryRouterImpl := router.NewTelemetryRouterImpl(sugaredLogger, telemetryRestHandlerImpl) bulkUpdateRepositoryImpl := bulkUpdate.NewBulkUpdateRepository(db, sugaredLogger) - deployedAppServiceImpl := deployedApp.NewDeployedAppServiceImpl(sugaredLogger, k8sCommonServiceImpl, triggerServiceImpl, environmentRepositoryImpl, pipelineRepositoryImpl, cdWorkflowRepositoryImpl) - bulkUpdateServiceImpl := service7.NewBulkUpdateServiceImpl(bulkUpdateRepositoryImpl, sugaredLogger, environmentRepositoryImpl, pipelineRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineBuilderImpl, enforcerUtilImpl, ciHandlerImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, appWorkflowServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, deployedAppServiceImpl, cdPipelineEventPublishServiceImpl) + deployedAppServiceImpl := deployedApp.NewDeployedAppServiceImpl(sugaredLogger, k8sCommonServiceImpl, devtronAppsHandlerServiceImpl, environmentRepositoryImpl, pipelineRepositoryImpl, cdWorkflowRepositoryImpl) + bulkUpdateServiceImpl := service7.NewBulkUpdateServiceImpl(bulkUpdateRepositoryImpl, sugaredLogger, environmentRepositoryImpl, pipelineRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineBuilderImpl, enforcerUtilImpl, ciHandlerImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, appWorkflowServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, deployedAppServiceImpl, cdPipelineEventPublishServiceImpl, handlerServiceImpl) bulkUpdateRestHandlerImpl := restHandler.NewBulkUpdateRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, bulkUpdateServiceImpl, chartServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, environmentServiceImpl, gitRegistryConfigImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl) bulkUpdateRouterImpl := router.NewBulkUpdateRouterImpl(bulkUpdateRestHandlerImpl) webhookSecretValidatorImpl := gitWebhook.NewWebhookSecretValidatorImpl(sugaredLogger) @@ -952,18 +958,18 @@ func InitializeApp() (*App, error) { webhookListenerRouterImpl := router.NewWebhookListenerRouterImpl(webhookEventHandlerImpl) appFilteringRestHandlerImpl := appList.NewAppFilteringRestHandlerImpl(sugaredLogger, teamServiceImpl, enforcerImpl, userServiceImpl, clusterServiceImplExtended, environmentServiceImpl, teamReadServiceImpl) appFilteringRouterImpl := appList2.NewAppFilteringRouterImpl(appFilteringRestHandlerImpl) - serviceImpl := resourceTree.NewServiceImpl(sugaredLogger, appListingServiceImpl, appStatusServiceImpl, argoApplicationServiceExtendedImpl, cdApplicationStatusUpdateHandlerImpl, helmAppReadServiceImpl, helmAppServiceImpl, k8sApplicationServiceImpl, k8sCommonServiceImpl, environmentReadServiceImpl) - appListingRestHandlerImpl := appList.NewAppListingRestHandlerImpl(appListingServiceImpl, enforcerImpl, pipelineBuilderImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, userServiceImpl, k8sCommonServiceImpl, installedAppDBExtendedServiceImpl, installedAppResourceServiceImpl, pipelineRepositoryImpl, k8sApplicationServiceImpl, deploymentConfigServiceImpl, serviceImpl) + resourceTreeServiceImpl := resourceTree.NewServiceImpl(sugaredLogger, appListingServiceImpl, appStatusServiceImpl, argoApplicationServiceExtendedImpl, cdApplicationStatusUpdateHandlerImpl, helmAppReadServiceImpl, helmAppServiceImpl, k8sApplicationServiceImpl, k8sCommonServiceImpl, environmentReadServiceImpl) + appListingRestHandlerImpl := appList.NewAppListingRestHandlerImpl(appListingServiceImpl, enforcerImpl, pipelineBuilderImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, userServiceImpl, k8sCommonServiceImpl, installedAppDBExtendedServiceImpl, installedAppResourceServiceImpl, pipelineRepositoryImpl, k8sApplicationServiceImpl, deploymentConfigServiceImpl, resourceTreeServiceImpl) appListingRouterImpl := appList2.NewAppListingRouterImpl(appListingRestHandlerImpl) appInfoRestHandlerImpl := appInfo.NewAppInfoRestHandlerImpl(sugaredLogger, appCrudOperationServiceImpl, userServiceImpl, validate, enforcerUtilImpl, enforcerImpl, helmAppServiceImpl, enforcerUtilHelmImpl, genericNoteServiceImpl, commonEnforcementUtilImpl) appInfoRouterImpl := appInfo2.NewAppInfoRouterImpl(sugaredLogger, appInfoRestHandlerImpl) pipelineDeploymentConfigServiceImpl := pipeline.NewPipelineDeploymentConfigServiceImpl(sugaredLogger, chartRepositoryImpl, pipelineRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, scopedVariableCMCSManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, configMapHistoryReadServiceImpl, envConfigOverrideReadServiceImpl) - pipelineTriggerRestHandlerImpl := trigger.NewPipelineRestHandler(appServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, pipelineDeploymentConfigServiceImpl, deployedAppServiceImpl, triggerServiceImpl, workflowEventPublishServiceImpl) + pipelineTriggerRestHandlerImpl := trigger2.NewPipelineRestHandler(appServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, pipelineDeploymentConfigServiceImpl, deployedAppServiceImpl, devtronAppsHandlerServiceImpl, workflowEventPublishServiceImpl) sseSSE := sse.NewSSE() - pipelineTriggerRouterImpl := trigger2.NewPipelineTriggerRouter(pipelineTriggerRestHandlerImpl, sseSSE) + pipelineTriggerRouterImpl := trigger3.NewPipelineTriggerRouter(pipelineTriggerRestHandlerImpl, sseSSE) webhookDataRestHandlerImpl := webhook.NewWebhookDataRestHandlerImpl(sugaredLogger, userServiceImpl, ciPipelineMaterialRepositoryImpl, enforcerUtilImpl, enforcerImpl, clientImpl, webhookEventDataConfigImpl) pipelineConfigRouterImpl := configure2.NewPipelineRouterImpl(pipelineConfigRestHandlerImpl, webhookDataRestHandlerImpl) - prePostCiScriptHistoryRepositoryImpl := repository21.NewPrePostCiScriptHistoryRepositoryImpl(sugaredLogger, db) + prePostCiScriptHistoryRepositoryImpl := repository22.NewPrePostCiScriptHistoryRepositoryImpl(sugaredLogger, db) prePostCiScriptHistoryServiceImpl := history.NewPrePostCiScriptHistoryServiceImpl(sugaredLogger, prePostCiScriptHistoryRepositoryImpl) pipelineHistoryRestHandlerImpl := history2.NewPipelineHistoryRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerImpl, pipelineStrategyHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, prePostCdScriptHistoryServiceImpl, enforcerUtilImpl, deployedConfigurationHistoryServiceImpl) pipelineHistoryRouterImpl := history3.NewPipelineHistoryRouterImpl(pipelineHistoryRestHandlerImpl) @@ -1034,7 +1040,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - ciStatusUpdateCronImpl := cron2.NewCiStatusUpdateCronImpl(sugaredLogger, appServiceImpl, ciWorkflowStatusUpdateConfig, ciPipelineRepositoryImpl, ciHandlerImpl, cronLoggerImpl) + ciStatusUpdateCronImpl := cron2.NewCiStatusUpdateCronImpl(sugaredLogger, appServiceImpl, ciWorkflowStatusUpdateConfig, ciPipelineRepositoryImpl, cronLoggerImpl, workflowDagExecutorImpl) resourceGroupRestHandlerImpl := restHandler.NewResourceGroupRestHandlerImpl(sugaredLogger, enforcerImpl, userServiceImpl, resourceGroupServiceImpl, validate) resourceGroupingRouterImpl := router.NewResourceGroupingRouterImpl(pipelineConfigRestHandlerImpl, appWorkflowRestHandlerImpl, resourceGroupRestHandlerImpl) rbacRoleServiceImpl := user.NewRbacRoleServiceImpl(sugaredLogger, rbacRoleDataRepositoryImpl) @@ -1046,7 +1052,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - ciTriggerCronImpl := cron2.NewCiTriggerCronImpl(sugaredLogger, ciTriggerCronConfig, pipelineStageRepositoryImpl, ciHandlerImpl, ciArtifactRepositoryImpl, globalPluginRepositoryImpl, cronLoggerImpl) + ciTriggerCronImpl := cron2.NewCiTriggerCronImpl(sugaredLogger, ciTriggerCronConfig, pipelineStageRepositoryImpl, ciArtifactRepositoryImpl, globalPluginRepositoryImpl, cronLoggerImpl, handlerServiceImpl) proxyConfig, err := proxy.GetProxyConfig() if err != nil { return nil, err @@ -1082,12 +1088,12 @@ func InitializeApp() (*App, error) { cdWorkflowServiceImpl := cd.NewCdWorkflowServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) cdWorkflowRunnerReadServiceImpl := read20.NewCdWorkflowRunnerReadServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) webhookServiceImpl := pipeline.NewWebhookServiceImpl(ciArtifactRepositoryImpl, sugaredLogger, ciPipelineRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowCommonServiceImpl, workFlowStageStatusServiceImpl, ciServiceImpl) - workflowEventProcessorImpl, err := in.NewWorkflowEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowServiceImpl, cdWorkflowReadServiceImpl, cdWorkflowRunnerServiceImpl, cdWorkflowRunnerReadServiceImpl, workflowDagExecutorImpl, ciHandlerImpl, cdHandlerImpl, eventSimpleFactoryImpl, eventRESTClientImpl, triggerServiceImpl, deployedAppServiceImpl, webhookServiceImpl, validate, environmentVariables, cdWorkflowCommonServiceImpl, cdPipelineConfigServiceImpl, userDeploymentRequestServiceImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, cdWorkflowRepositoryImpl, deploymentConfigServiceImpl) + workflowEventProcessorImpl, err := in.NewWorkflowEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowServiceImpl, cdWorkflowReadServiceImpl, cdWorkflowRunnerServiceImpl, cdWorkflowRunnerReadServiceImpl, workflowDagExecutorImpl, ciHandlerImpl, cdHandlerImpl, eventSimpleFactoryImpl, eventRESTClientImpl, devtronAppsHandlerServiceImpl, deployedAppServiceImpl, webhookServiceImpl, validate, environmentVariables, cdWorkflowCommonServiceImpl, cdPipelineConfigServiceImpl, userDeploymentRequestServiceImpl, serviceImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, cdWorkflowRepositoryImpl, deploymentConfigServiceImpl, handlerServiceImpl) if err != nil { return nil, err } ciPipelineEventProcessorImpl := in.NewCIPipelineEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, gitWebhookServiceImpl) - cdPipelineEventProcessorImpl := in.NewCDPipelineEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowCommonServiceImpl, workflowStatusServiceImpl, triggerServiceImpl, pipelineRepositoryImpl, installedAppReadServiceImpl) + cdPipelineEventProcessorImpl := in.NewCDPipelineEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowCommonServiceImpl, workflowStatusServiceImpl, devtronAppsHandlerServiceImpl, pipelineRepositoryImpl, installedAppReadServiceImpl) deployedApplicationEventProcessorImpl := in.NewDeployedApplicationEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, appServiceImpl, gitOpsConfigReadServiceImpl, installedAppDBExtendedServiceImpl, workflowDagExecutorImpl, cdWorkflowCommonServiceImpl, pipelineBuilderImpl, appStoreDeploymentServiceImpl, pipelineRepositoryImpl, installedAppReadServiceImpl, deploymentConfigServiceImpl) appStoreAppsEventProcessorImpl := in.NewAppStoreAppsEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, chartGroupServiceImpl, installedAppVersionHistoryRepositoryImpl) centralEventProcessor, err := eventProcessor.NewCentralEventProcessor(sugaredLogger, workflowEventProcessorImpl, ciPipelineEventProcessorImpl, cdPipelineEventProcessorImpl, deployedApplicationEventProcessorImpl, appStoreAppsEventProcessorImpl)