From 6517318ae0403bab20e7fbf76d6d2b9770ca520a Mon Sep 17 00:00:00 2001 From: Gireesh Naidu Date: Mon, 22 Jul 2024 12:37:50 +0530 Subject: [PATCH 1/3] feat: expose git commit data as env vars for ci stage --- .../devtronApps/PreStageTriggerService.go | 63 +++------------- .../trigger/devtronApps/TriggerService.go | 3 + pkg/pipeline/CiMaterialConfigService.go | 72 ++++++++++++++++++- pkg/pipeline/CiService.go | 12 ++++ pkg/pipeline/types/Workflow.go | 9 +++ wire_gen.go | 46 ++++++------ 6 files changed, 129 insertions(+), 76 deletions(-) diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go index 945b896de9..595a83f328 100644 --- a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go @@ -47,16 +47,13 @@ import ( util2 "github.com/devtron-labs/devtron/util/event" "github.com/go-pg/pg" "go.opentelemetry.io/otel" + "maps" "strconv" "strings" "time" ) const ( - GIT_COMMIT_HASH_PREFIX = "GIT_COMMIT_HASH" - GIT_SOURCE_TYPE_PREFIX = "GIT_SOURCE_TYPE" - GIT_SOURCE_VALUE_PREFIX = "GIT_SOURCE_VALUE" - GIT_SOURCE_COUNT = "GIT_SOURCE_COUNT" APP_LABEL_KEY_PREFIX = "APP_LABEL_KEY" APP_LABEL_VALUE_PREFIX = "APP_LABEL_VALUE" APP_LABEL_COUNT = "APP_LABEL_COUNT" @@ -582,55 +579,17 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow impl.logger.Errorw("error in getting ciWf by artifactId", "err", err, "artifactId", artifact.Id) return nil, err } - var webhookAndCiData *gitSensorClient.WebhookAndCiData - if ciWf != nil && ciWf.GitTriggers != nil { - i := 1 - var gitCommitEnvVariables []types.GitMetadata - - for ciPipelineMaterialId, gitTrigger := range ciWf.GitTriggers { - extraEnvVariables[fmt.Sprintf("%s_%d", GIT_COMMIT_HASH_PREFIX, i)] = gitTrigger.Commit - extraEnvVariables[fmt.Sprintf("%s_%d", GIT_SOURCE_TYPE_PREFIX, i)] = string(gitTrigger.CiConfigureSourceType) - extraEnvVariables[fmt.Sprintf("%s_%d", GIT_SOURCE_VALUE_PREFIX, i)] = gitTrigger.CiConfigureSourceValue - - gitCommitEnvVariables = append(gitCommitEnvVariables, types.GitMetadata{ - GitCommitHash: gitTrigger.Commit, - GitSourceType: string(gitTrigger.CiConfigureSourceType), - GitSourceValue: gitTrigger.CiConfigureSourceValue, - }) - - // CODE-BLOCK starts - store extra environment variables if webhook - if gitTrigger.CiConfigureSourceType == pipelineConfig.SOURCE_TYPE_WEBHOOK { - webhookDataId := gitTrigger.WebhookData.Id - if webhookDataId > 0 { - webhookDataRequest := &gitSensorClient.WebhookDataRequest{ - Id: webhookDataId, - CiPipelineMaterialId: ciPipelineMaterialId, - } - webhookAndCiData, err = impl.gitSensorGrpcClient.GetWebhookData(context.Background(), webhookDataRequest) - if err != nil { - impl.logger.Errorw("err while getting webhook data from git-sensor", "err", err, "webhookDataRequest", webhookDataRequest) - return nil, err - } - if webhookAndCiData != nil { - for extEnvVariableKey, extEnvVariableVal := range webhookAndCiData.ExtraEnvironmentVariables { - extraEnvVariables[extEnvVariableKey] = extEnvVariableVal - } - } - } - } - // CODE_BLOCK ends - i++ - } - gitMetadata, err := json.Marshal(&gitCommitEnvVariables) - if err != nil { - impl.logger.Errorw("err while marshaling git metdata", "err", err) - return nil, err - } - extraEnvVariables[plugin.GIT_METADATA] = string(gitMetadata) + var webhookAndCiData *gitSensorClient.WebhookAndCiData + var gitTriggerEnvVariables map[string]string - extraEnvVariables[GIT_SOURCE_COUNT] = strconv.Itoa(len(ciWf.GitTriggers)) + // get env variables of git trigger data and add it in the extraEnvVariables + gitTriggerEnvVariables, webhookAndCiData, err = impl.ciMaterialConfigService.GetGitCommitEnvVarDataForCICDStage(ciWf.GitTriggers) + if err != nil { + impl.logger.Errorw("error in getting gitTrigger env data for stage", "gitTriggers", ciWf.GitTriggers, "err", err) + return nil, err } + maps.Copy(extraEnvVariables, gitTriggerEnvVariables) childCdIds, err := impl.appWorkflowRepository.FindChildCDIdsByParentCDPipelineId(cdPipeline.Id) if err != nil && err != pg.ErrNoRows { @@ -986,10 +945,10 @@ func (impl *TriggerServiceImpl) sendPreStageNotification(ctx context.Context, cd } func isExtraVariableDynamic(variableName string, webhookAndCiData *gitSensorClient.WebhookAndCiData) bool { - if strings.Contains(variableName, GIT_COMMIT_HASH_PREFIX) || strings.Contains(variableName, GIT_SOURCE_TYPE_PREFIX) || strings.Contains(variableName, GIT_SOURCE_VALUE_PREFIX) || + if strings.Contains(variableName, types.GIT_COMMIT_HASH_PREFIX) || strings.Contains(variableName, types.GIT_SOURCE_TYPE_PREFIX) || strings.Contains(variableName, types.GIT_SOURCE_VALUE_PREFIX) || strings.Contains(variableName, APP_LABEL_VALUE_PREFIX) || strings.Contains(variableName, APP_LABEL_KEY_PREFIX) || strings.Contains(variableName, CHILD_CD_ENV_NAME_PREFIX) || strings.Contains(variableName, CHILD_CD_CLUSTER_NAME_PREFIX) || - strings.Contains(variableName, CHILD_CD_COUNT) || strings.Contains(variableName, APP_LABEL_COUNT) || strings.Contains(variableName, GIT_SOURCE_COUNT) || + strings.Contains(variableName, CHILD_CD_COUNT) || strings.Contains(variableName, APP_LABEL_COUNT) || strings.Contains(variableName, types.GIT_SOURCE_COUNT) || webhookAndCiData != nil { return true diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/TriggerService.go index 10bfb4b59d..32ac02cbac 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService.go @@ -161,6 +161,7 @@ type TriggerServiceImpl struct { transactionUtilImpl *sql.TransactionUtilImpl deploymentConfigService common.DeploymentConfigService deploymentServiceTypeConfig *util3.DeploymentServiceTypeConfig + ciMaterialConfigService pipeline.CiMaterialConfigService } func NewTriggerServiceImpl(logger *zap.SugaredLogger, @@ -215,6 +216,7 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, K8sUtil *util5.K8sServiceImpl, transactionUtilImpl *sql.TransactionUtilImpl, deploymentConfigService common.DeploymentConfigService, + ciMaterialConfigService pipeline.CiMaterialConfigService, ) (*TriggerServiceImpl, error) { impl := &TriggerServiceImpl{ logger: logger, @@ -270,6 +272,7 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, transactionUtilImpl: transactionUtilImpl, deploymentConfigService: deploymentConfigService, deploymentServiceTypeConfig: envVariables.DeploymentServiceTypeConfig, + ciMaterialConfigService: ciMaterialConfigService, } config, err := types.GetCdConfig() if err != nil { diff --git a/pkg/pipeline/CiMaterialConfigService.go b/pkg/pipeline/CiMaterialConfigService.go index 67e2df5828..ea7181574f 100644 --- a/pkg/pipeline/CiMaterialConfigService.go +++ b/pkg/pipeline/CiMaterialConfigService.go @@ -17,14 +17,20 @@ package pipeline import ( + "context" + "encoding/json" "fmt" + gitSensorClient "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/pipeline/history" + "github.com/devtron-labs/devtron/pkg/pipeline/types" + "github.com/devtron-labs/devtron/pkg/plugin" "github.com/devtron-labs/devtron/pkg/sql" "github.com/go-pg/pg" "github.com/juju/errors" "go.uber.org/zap" + "strconv" "strings" "time" ) @@ -41,6 +47,8 @@ type CiMaterialConfigService interface { BulkPatchCiMaterialSource(ciPipelines *bean.CiMaterialBulkPatchRequest, userId int32, token string, checkAppSpecificAccess func(token, action string, appId int) (bool, error)) (*bean.CiMaterialBulkPatchResponse, error) //GetMaterialsForAppId : Retrieve material for given appId GetMaterialsForAppId(appId int) []*bean.GitMaterial + + GetGitCommitEnvVarDataForCICDStage(gitTriggers map[int]pipelineConfig.GitCommit) (map[string]string, *gitSensorClient.WebhookAndCiData, error) } type CiMaterialConfigServiceImpl struct { @@ -52,6 +60,7 @@ type CiMaterialConfigServiceImpl struct { gitMaterialHistoryService history.GitMaterialHistoryService pipelineRepository pipelineConfig.PipelineRepository ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository + gitSensorGrpcClient gitSensorClient.Client transactionManager sql.TransactionWrapper } @@ -63,7 +72,9 @@ func NewCiMaterialConfigServiceImpl( ciPipelineRepository pipelineConfig.CiPipelineRepository, gitMaterialHistoryService history.GitMaterialHistoryService, pipelineRepository pipelineConfig.PipelineRepository, - ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, transactionManager sql.TransactionWrapper) *CiMaterialConfigServiceImpl { + ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, + gitSensorGrpcClient gitSensorClient.Client, + transactionManager sql.TransactionWrapper) *CiMaterialConfigServiceImpl { return &CiMaterialConfigServiceImpl{ logger: logger, @@ -74,6 +85,7 @@ func NewCiMaterialConfigServiceImpl( gitMaterialHistoryService: gitMaterialHistoryService, pipelineRepository: pipelineRepository, ciPipelineMaterialRepository: ciPipelineMaterialRepository, + gitSensorGrpcClient: gitSensorGrpcClient, transactionManager: transactionManager, } } @@ -231,3 +243,61 @@ func (impl *CiMaterialConfigServiceImpl) GetMaterialsForAppId(appId int) []*bean } return gitMaterials } + +func (impl *CiMaterialConfigServiceImpl) GetGitCommitEnvVarDataForCICDStage(gitTriggers map[int]pipelineConfig.GitCommit) (map[string]string, *gitSensorClient.WebhookAndCiData, error) { + var webhookAndCiData *gitSensorClient.WebhookAndCiData + var err error + extraEnvVariables := make(map[string]string) + if gitTriggers != nil { + i := 1 + var gitCommitEnvVariables []types.GitMetadata + + for ciPipelineMaterialId, gitTrigger := range gitTriggers { + extraEnvVariables[fmt.Sprintf("%s_%d", types.GIT_COMMIT_HASH_PREFIX, i)] = gitTrigger.Commit + extraEnvVariables[fmt.Sprintf("%s_%d", types.GIT_SOURCE_TYPE_PREFIX, i)] = string(gitTrigger.CiConfigureSourceType) + extraEnvVariables[fmt.Sprintf("%s_%d", types.GIT_SOURCE_VALUE_PREFIX, i)] = gitTrigger.CiConfigureSourceValue + extraEnvVariables[fmt.Sprintf("%s_%d", types.GIT_REPO_URL_PREFIX, i)] = gitTrigger.GitRepoUrl + + gitCommitEnvVariables = append(gitCommitEnvVariables, types.GitMetadata{ + GitCommitHash: gitTrigger.Commit, + GitSourceType: string(gitTrigger.CiConfigureSourceType), + GitSourceValue: gitTrigger.CiConfigureSourceValue, + GitRepoUrl: gitTrigger.GitRepoUrl, + }) + + // CODE-BLOCK starts - store extra environment variables if webhook + if gitTrigger.CiConfigureSourceType == pipelineConfig.SOURCE_TYPE_WEBHOOK { + webhookDataId := gitTrigger.WebhookData.Id + if webhookDataId > 0 { + webhookDataRequest := &gitSensorClient.WebhookDataRequest{ + Id: webhookDataId, + CiPipelineMaterialId: ciPipelineMaterialId, + } + webhookAndCiData, err = impl.gitSensorGrpcClient.GetWebhookData(context.Background(), webhookDataRequest) + if err != nil { + impl.logger.Errorw("err while getting webhook data from git-sensor", "err", err, "webhookDataRequest", webhookDataRequest) + return nil, nil, err + } + if webhookAndCiData != nil { + for extEnvVariableKey, extEnvVariableVal := range webhookAndCiData.ExtraEnvironmentVariables { + extraEnvVariables[extEnvVariableKey] = extEnvVariableVal + } + } + } + } + // CODE_BLOCK ends + + i++ + } + gitMetadata, err := json.Marshal(&gitCommitEnvVariables) + if err != nil { + impl.logger.Errorw("err while marshaling git metdata", "err", err) + return nil, nil, err + } + extraEnvVariables[plugin.GIT_METADATA] = string(gitMetadata) + + extraEnvVariables[types.GIT_SOURCE_COUNT] = strconv.Itoa(len(gitTriggers)) + } + + return extraEnvVariables, webhookAndCiData, nil +} diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 67e78ab1ad..790b21e169 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -24,6 +24,7 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/adapter" "github.com/devtron-labs/devtron/pkg/pipeline/bean/CiPipeline" "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders" + "maps" "path/filepath" "strconv" "strings" @@ -83,6 +84,7 @@ type CiServiceImpl struct { pluginInputVariableParser PluginInputVariableParser globalPluginService plugin.GlobalPluginService infraProvider infraProviders.InfraProvider + ciMaterialConfigService CiMaterialConfigService } func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService, @@ -98,6 +100,7 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService pluginInputVariableParser PluginInputVariableParser, globalPluginService plugin.GlobalPluginService, infraProvider infraProviders.InfraProvider, + ciMaterialConfigService CiMaterialConfigService, ) *CiServiceImpl { cis := &CiServiceImpl{ Logger: Logger, @@ -118,6 +121,7 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService pluginInputVariableParser: pluginInputVariableParser, globalPluginService: globalPluginService, infraProvider: infraProvider, + ciMaterialConfigService: ciMaterialConfigService, } config, err := types.GetCiConfig() if err != nil { @@ -216,6 +220,14 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) return 0, err } + // get env variables of git trigger data and add it in the extraEnvVariables + gitTriggerEnvVariables, _, err := impl.ciMaterialConfigService.GetGitCommitEnvVarDataForCICDStage(savedCiWf.GitTriggers) + if err != nil { + impl.Logger.Errorw("error in getting gitTrigger env data for stage", "gitTriggers", savedCiWf.GitTriggers, "err", err) + return 0, err + } + maps.Copy(trigger.ExtraEnvironmentVariables, gitTriggerEnvVariables) + workflowRequest, err := impl.buildWfRequestForCiPipeline(pipeline, trigger, ciMaterials, savedCiWf, ciWorkflowConfig, ciPipelineScripts, preCiSteps, postCiSteps, refPluginsData, isJob) if err != nil { impl.Logger.Errorw("make workflow req", "err", err) diff --git a/pkg/pipeline/types/Workflow.go b/pkg/pipeline/types/Workflow.go index 184e30a077..11441d38ee 100644 --- a/pkg/pipeline/types/Workflow.go +++ b/pkg/pipeline/types/Workflow.go @@ -40,6 +40,14 @@ import ( "time" ) +const ( + GIT_REPO_URL_PREFIX = "GIT_REPO_URL" + GIT_COMMIT_HASH_PREFIX = "GIT_COMMIT_HASH" + GIT_SOURCE_TYPE_PREFIX = "GIT_SOURCE_TYPE" + GIT_SOURCE_VALUE_PREFIX = "GIT_SOURCE_VALUE" + GIT_SOURCE_COUNT = "GIT_SOURCE_COUNT" +) + type WorkflowRequest struct { WorkflowNamePrefix string `json:"workflowNamePrefix"` PipelineName string `json:"pipelineName"` @@ -596,6 +604,7 @@ type GitMetadata struct { GitCommitHash string `json:"GIT_COMMIT_HASH"` GitSourceType string `json:"GIT_SOURCE_TYPE"` GitSourceValue string `json:"GIT_SOURCE_VALUE"` + GitRepoUrl string `json:"GIT_REPO_URL"` } type AppLabelMetadata struct { diff --git a/wire_gen.go b/wire_gen.go index a6cd7d215f..b41a225f28 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -88,7 +88,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/deploymentConfig" repository5 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/helper" - repository12 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" + repository13 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/resourceGroup" "github.com/devtron-labs/devtron/internal/sql/repository/security" @@ -181,7 +181,7 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/history" - repository13 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" + repository12 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders" repository10 "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" @@ -538,7 +538,6 @@ func InitializeApp() (*App, error) { customTagServiceImpl := pipeline.NewCustomTagService(sugaredLogger, imageTagRepositoryImpl) pluginInputVariableParserImpl := pipeline.NewPluginInputVariableParserImpl(sugaredLogger, dockerRegistryConfigImpl, customTagServiceImpl) globalPluginServiceImpl := plugin.NewGlobalPluginService(sugaredLogger, globalPluginRepositoryImpl, pipelineStageRepositoryImpl) - ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciWorkflowRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl, infraProviderImpl) clientConfig, err := gitSensor.GetConfig() if err != nil { return nil, err @@ -547,6 +546,22 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } + prePostCdScriptHistoryRepositoryImpl := repository12.NewPrePostCdScriptHistoryRepositoryImpl(sugaredLogger, db) + configMapHistoryRepositoryImpl := repository12.NewConfigMapHistoryRepositoryImpl(sugaredLogger, db, transactionUtilImpl) + configMapHistoryServiceImpl := history.NewConfigMapHistoryServiceImpl(sugaredLogger, configMapHistoryRepositoryImpl, pipelineRepositoryImpl, configMapRepositoryImpl, userServiceImpl, scopedVariableCMCSManagerImpl) + prePostCdScriptHistoryServiceImpl := history.NewPrePostCdScriptHistoryServiceImpl(sugaredLogger, prePostCdScriptHistoryRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl) + gitMaterialHistoryRepositoryImpl := repository12.NewGitMaterialHistoryRepositoyImpl(db) + gitMaterialHistoryServiceImpl := history.NewGitMaterialHistoryServiceImpl(gitMaterialHistoryRepositoryImpl, sugaredLogger) + ciPipelineHistoryRepositoryImpl := repository12.NewCiPipelineHistoryRepositoryImpl(db, sugaredLogger) + ciPipelineHistoryServiceImpl := history.NewCiPipelineHistoryServiceImpl(ciPipelineHistoryRepositoryImpl, sugaredLogger, ciPipelineRepositoryImpl) + pipelineConfigRepositoryImpl := chartConfig.NewPipelineConfigRepository(db) + configMapServiceImpl := pipeline.NewConfigMapServiceImpl(chartRepositoryImpl, sugaredLogger, chartRepoRepositoryImpl, utilMergeUtil, pipelineConfigRepositoryImpl, configMapRepositoryImpl, envConfigOverrideRepositoryImpl, commonServiceImpl, appRepositoryImpl, configMapHistoryServiceImpl, environmentRepositoryImpl, scopedVariableCMCSManagerImpl) + deploymentTemplateHistoryRepositoryImpl := repository12.NewDeploymentTemplateHistoryRepositoryImpl(sugaredLogger, db) + deploymentTemplateHistoryServiceImpl := history.NewDeploymentTemplateHistoryServiceImpl(sugaredLogger, deploymentTemplateHistoryRepositoryImpl, pipelineRepositoryImpl, chartRepositoryImpl, userServiceImpl, cdWorkflowRepositoryImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) + chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, utilMergeUtil, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl) + ciCdPipelineOrchestratorImpl := pipeline.NewCiCdPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, clientImpl, ciCdConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, pipelineStageServiceImpl, gitMaterialHistoryServiceImpl, ciPipelineHistoryServiceImpl, ciTemplateServiceImpl, dockerArtifactStoreRepositoryImpl, ciArtifactRepositoryImpl, configMapServiceImpl, customTagServiceImpl, genericNoteServiceImpl, chartServiceImpl, transactionUtilImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl) + ciMaterialConfigServiceImpl := pipeline.NewCiMaterialConfigServiceImpl(sugaredLogger, materialRepositoryImpl, ciTemplateServiceImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, gitMaterialHistoryServiceImpl, pipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, clientImpl, transactionUtilImpl) + ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciWorkflowRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl, infraProviderImpl, ciMaterialConfigServiceImpl) ciLogServiceImpl, err := pipeline.NewCiLogServiceImpl(sugaredLogger, ciServiceImpl, k8sServiceImpl) if err != nil { return nil, err @@ -554,38 +569,23 @@ func InitializeApp() (*App, error) { resourceGroupRepositoryImpl := resourceGroup.NewResourceGroupRepositoryImpl(db) resourceGroupMappingRepositoryImpl := resourceGroup.NewResourceGroupMappingRepositoryImpl(db) resourceGroupServiceImpl := resourceGroup2.NewResourceGroupServiceImpl(sugaredLogger, resourceGroupRepositoryImpl, resourceGroupMappingRepositoryImpl, enforcerUtilImpl, devtronResourceSearchableKeyServiceImpl, appStatusRepositoryImpl) - imageTaggingRepositoryImpl := repository12.NewImageTaggingRepositoryImpl(db, transactionUtilImpl) + imageTaggingRepositoryImpl := repository13.NewImageTaggingRepositoryImpl(db, transactionUtilImpl) imageTaggingServiceImpl := pipeline.NewImageTaggingServiceImpl(imageTaggingRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, sugaredLogger) blobStorageConfigServiceImpl := pipeline.NewBlobStorageConfigServiceImpl(sugaredLogger, k8sServiceImpl, ciCdConfig) ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, workflowServiceImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, k8sServiceImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, environmentServiceImpl) gitWebhookRepositoryImpl := repository2.NewGitWebhookRepositoryImpl(db) gitWebhookServiceImpl := git2.NewGitWebhookServiceImpl(sugaredLogger, ciHandlerImpl, gitWebhookRepositoryImpl) gitWebhookRestHandlerImpl := restHandler.NewGitWebhookRestHandlerImpl(sugaredLogger, gitWebhookServiceImpl) - prePostCdScriptHistoryRepositoryImpl := repository13.NewPrePostCdScriptHistoryRepositoryImpl(sugaredLogger, db) - configMapHistoryRepositoryImpl := repository13.NewConfigMapHistoryRepositoryImpl(sugaredLogger, db, transactionUtilImpl) - configMapHistoryServiceImpl := history.NewConfigMapHistoryServiceImpl(sugaredLogger, configMapHistoryRepositoryImpl, pipelineRepositoryImpl, configMapRepositoryImpl, userServiceImpl, scopedVariableCMCSManagerImpl) - prePostCdScriptHistoryServiceImpl := history.NewPrePostCdScriptHistoryServiceImpl(sugaredLogger, prePostCdScriptHistoryRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl) - gitMaterialHistoryRepositoryImpl := repository13.NewGitMaterialHistoryRepositoyImpl(db) - gitMaterialHistoryServiceImpl := history.NewGitMaterialHistoryServiceImpl(gitMaterialHistoryRepositoryImpl, sugaredLogger) - ciPipelineHistoryRepositoryImpl := repository13.NewCiPipelineHistoryRepositoryImpl(db, sugaredLogger) - ciPipelineHistoryServiceImpl := history.NewCiPipelineHistoryServiceImpl(ciPipelineHistoryRepositoryImpl, sugaredLogger, ciPipelineRepositoryImpl) - pipelineConfigRepositoryImpl := chartConfig.NewPipelineConfigRepository(db) - configMapServiceImpl := pipeline.NewConfigMapServiceImpl(chartRepositoryImpl, sugaredLogger, chartRepoRepositoryImpl, utilMergeUtil, pipelineConfigRepositoryImpl, configMapRepositoryImpl, envConfigOverrideRepositoryImpl, commonServiceImpl, appRepositoryImpl, configMapHistoryServiceImpl, environmentRepositoryImpl, scopedVariableCMCSManagerImpl) - deploymentTemplateHistoryRepositoryImpl := repository13.NewDeploymentTemplateHistoryRepositoryImpl(sugaredLogger, db) - deploymentTemplateHistoryServiceImpl := history.NewDeploymentTemplateHistoryServiceImpl(sugaredLogger, deploymentTemplateHistoryRepositoryImpl, pipelineRepositoryImpl, chartRepositoryImpl, userServiceImpl, cdWorkflowRepositoryImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) - chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, utilMergeUtil, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl) - ciCdPipelineOrchestratorImpl := pipeline.NewCiCdPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, clientImpl, ciCdConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, pipelineStageServiceImpl, gitMaterialHistoryServiceImpl, ciPipelineHistoryServiceImpl, ciTemplateServiceImpl, dockerArtifactStoreRepositoryImpl, ciArtifactRepositoryImpl, configMapServiceImpl, customTagServiceImpl, genericNoteServiceImpl, chartServiceImpl, transactionUtilImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl) ecrConfig, err := pipeline.GetEcrConfig() if err != nil { return nil, err } - ciTemplateHistoryRepositoryImpl := repository13.NewCiTemplateHistoryRepositoryImpl(db, sugaredLogger) + ciTemplateHistoryRepositoryImpl := repository12.NewCiTemplateHistoryRepositoryImpl(db, sugaredLogger) ciTemplateHistoryServiceImpl := history.NewCiTemplateHistoryServiceImpl(ciTemplateHistoryRepositoryImpl, sugaredLogger) buildPipelineSwitchServiceImpl := pipeline.NewBuildPipelineSwitchServiceImpl(sugaredLogger, ciPipelineRepositoryImpl, ciCdPipelineOrchestratorImpl, pipelineRepositoryImpl, ciWorkflowRepositoryImpl, appWorkflowRepositoryImpl, ciPipelineHistoryServiceImpl, ciTemplateOverrideRepositoryImpl, ciPipelineMaterialRepositoryImpl) ciPipelineConfigServiceImpl := pipeline.NewCiPipelineConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, appRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ecrConfig, appWorkflowRepositoryImpl, ciCdConfig, attributesServiceImpl, pipelineStageServiceImpl, ciPipelineMaterialRepositoryImpl, ciTemplateServiceImpl, ciTemplateOverrideRepositoryImpl, ciTemplateHistoryServiceImpl, enforcerUtilImpl, ciWorkflowRepositoryImpl, resourceGroupServiceImpl, customTagServiceImpl, cdWorkflowRepositoryImpl, buildPipelineSwitchServiceImpl) - ciMaterialConfigServiceImpl := pipeline.NewCiMaterialConfigServiceImpl(sugaredLogger, materialRepositoryImpl, ciTemplateServiceImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, gitMaterialHistoryServiceImpl, pipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, transactionUtilImpl) deploymentGroupRepositoryImpl := repository2.NewDeploymentGroupRepositoryImpl(sugaredLogger, db) - pipelineStrategyHistoryRepositoryImpl := repository13.NewPipelineStrategyHistoryRepositoryImpl(sugaredLogger, db) + pipelineStrategyHistoryRepositoryImpl := repository12.NewPipelineStrategyHistoryRepositoryImpl(sugaredLogger, db) pipelineStrategyHistoryServiceImpl := history.NewPipelineStrategyHistoryServiceImpl(sugaredLogger, pipelineStrategyHistoryRepositoryImpl, userServiceImpl) propertiesConfigServiceImpl := pipeline.NewPropertiesConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl) repositoryServiceClientImpl := repository14.NewServiceClientImpl(sugaredLogger, argoCDConnectionManagerImpl) @@ -644,7 +644,7 @@ func InitializeApp() (*App, error) { manifestPushConfigRepositoryImpl := repository10.NewManifestPushConfigRepository(sugaredLogger, db) scanToolExecutionHistoryMappingRepositoryImpl := security.NewScanToolExecutionHistoryMappingRepositoryImpl(db, sugaredLogger) imageScanServiceImpl := security2.NewImageScanServiceImpl(sugaredLogger, imageScanHistoryRepositoryImpl, imageScanResultRepositoryImpl, imageScanObjectMetaRepositoryImpl, cveStoreRepositoryImpl, imageScanDeployInfoRepositoryImpl, userServiceImpl, teamRepositoryImpl, appRepositoryImpl, environmentServiceImpl, ciArtifactRepositoryImpl, policyServiceImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, scanToolMetadataRepositoryImpl, scanToolExecutionHistoryMappingRepositoryImpl, cvePolicyRepositoryImpl) - triggerServiceImpl, err := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, argoUserServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, workflowServiceImpl, imageDigestPolicyServiceImpl, userServiceImpl, clientImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateServiceImpl, materialRepositoryImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl) + triggerServiceImpl, err := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, argoUserServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, workflowServiceImpl, imageDigestPolicyServiceImpl, userServiceImpl, clientImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateServiceImpl, materialRepositoryImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, ciMaterialConfigServiceImpl) if err != nil { return nil, err } @@ -837,7 +837,7 @@ func InitializeApp() (*App, error) { pipelineTriggerRouterImpl := trigger2.NewPipelineTriggerRouter(pipelineTriggerRestHandlerImpl, sseSSE) webhookDataRestHandlerImpl := webhook.NewWebhookDataRestHandlerImpl(sugaredLogger, userServiceImpl, ciPipelineMaterialRepositoryImpl, enforcerUtilImpl, enforcerImpl, clientImpl, webhookEventDataConfigImpl) pipelineConfigRouterImpl := configure2.NewPipelineRouterImpl(pipelineConfigRestHandlerImpl, webhookDataRestHandlerImpl) - prePostCiScriptHistoryRepositoryImpl := repository13.NewPrePostCiScriptHistoryRepositoryImpl(sugaredLogger, db) + prePostCiScriptHistoryRepositoryImpl := repository12.NewPrePostCiScriptHistoryRepositoryImpl(sugaredLogger, db) prePostCiScriptHistoryServiceImpl := history.NewPrePostCiScriptHistoryServiceImpl(sugaredLogger, prePostCiScriptHistoryRepositoryImpl) pipelineHistoryRestHandlerImpl := history2.NewPipelineHistoryRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerImpl, pipelineStrategyHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, prePostCdScriptHistoryServiceImpl, enforcerUtilImpl, deployedConfigurationHistoryServiceImpl) pipelineHistoryRouterImpl := history3.NewPipelineHistoryRouterImpl(pipelineHistoryRestHandlerImpl) From c98f6e9b393994d0b7a515d8f06a3e5184271e6f Mon Sep 17 00:00:00 2001 From: Gireesh Naidu Date: Mon, 22 Jul 2024 13:16:05 +0530 Subject: [PATCH 2/3] git-sensor client renaming --- pkg/deployment/trigger/devtronApps/TriggerService.go | 6 +++--- pkg/pipeline/CiMaterialConfigService.go | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/TriggerService.go index 32ac02cbac..82a1c62e86 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService.go @@ -131,7 +131,7 @@ type TriggerServiceImpl struct { cdWorkflowService pipeline.WorkflowService imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService userService user.UserService - gitSensorGrpcClient gitSensorClient.Client + gitSensorClient gitSensorClient.Client config *types.CdConfig helmAppService client2.HelmAppService imageScanService security2.ImageScanService @@ -186,7 +186,7 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, cdWorkflowService pipeline.WorkflowService, imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService, userService user.UserService, - gitSensorGrpcClient gitSensorClient.Client, + gitSensorClient gitSensorClient.Client, helmAppService client2.HelmAppService, enforcerUtil rbac.EnforcerUtil, userDeploymentRequestService service.UserDeploymentRequestService, @@ -241,7 +241,7 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, cdWorkflowService: cdWorkflowService, imageDigestPolicyService: imageDigestPolicyService, userService: userService, - gitSensorGrpcClient: gitSensorGrpcClient, + gitSensorClient: gitSensorClient, helmAppService: helmAppService, enforcerUtil: enforcerUtil, eventFactory: eventFactory, diff --git a/pkg/pipeline/CiMaterialConfigService.go b/pkg/pipeline/CiMaterialConfigService.go index ea7181574f..d0db6e32a3 100644 --- a/pkg/pipeline/CiMaterialConfigService.go +++ b/pkg/pipeline/CiMaterialConfigService.go @@ -60,7 +60,7 @@ type CiMaterialConfigServiceImpl struct { gitMaterialHistoryService history.GitMaterialHistoryService pipelineRepository pipelineConfig.PipelineRepository ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository - gitSensorGrpcClient gitSensorClient.Client + gitSensorClient gitSensorClient.Client transactionManager sql.TransactionWrapper } @@ -73,7 +73,7 @@ func NewCiMaterialConfigServiceImpl( gitMaterialHistoryService history.GitMaterialHistoryService, pipelineRepository pipelineConfig.PipelineRepository, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, - gitSensorGrpcClient gitSensorClient.Client, + gitSensorClient gitSensorClient.Client, transactionManager sql.TransactionWrapper) *CiMaterialConfigServiceImpl { return &CiMaterialConfigServiceImpl{ @@ -85,7 +85,7 @@ func NewCiMaterialConfigServiceImpl( gitMaterialHistoryService: gitMaterialHistoryService, pipelineRepository: pipelineRepository, ciPipelineMaterialRepository: ciPipelineMaterialRepository, - gitSensorGrpcClient: gitSensorGrpcClient, + gitSensorClient: gitSensorClient, transactionManager: transactionManager, } } @@ -273,7 +273,7 @@ func (impl *CiMaterialConfigServiceImpl) GetGitCommitEnvVarDataForCICDStage(gitT Id: webhookDataId, CiPipelineMaterialId: ciPipelineMaterialId, } - webhookAndCiData, err = impl.gitSensorGrpcClient.GetWebhookData(context.Background(), webhookDataRequest) + webhookAndCiData, err = impl.gitSensorClient.GetWebhookData(context.Background(), webhookDataRequest) if err != nil { impl.logger.Errorw("err while getting webhook data from git-sensor", "err", err, "webhookDataRequest", webhookDataRequest) return nil, nil, err From 5c5e9d3e3080eec48d0022e03fc0c4299eea7c3b Mon Sep 17 00:00:00 2001 From: Gireesh Naidu Date: Mon, 22 Jul 2024 13:39:33 +0530 Subject: [PATCH 3/3] move the logic to CiCdPipelineOrchestrator.go --- .../devtronApps/PreStageTriggerService.go | 2 +- .../trigger/devtronApps/TriggerService.go | 6 +- pkg/pipeline/CiCdPipelineOrchestrator.go | 60 ++++++++++++++++ pkg/pipeline/CiMaterialConfigService.go | 69 ------------------- pkg/pipeline/CiService.go | 8 +-- wire_gen.go | 6 +- 6 files changed, 71 insertions(+), 80 deletions(-) diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go index 595a83f328..37596e7479 100644 --- a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go @@ -584,7 +584,7 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow var gitTriggerEnvVariables map[string]string // get env variables of git trigger data and add it in the extraEnvVariables - gitTriggerEnvVariables, webhookAndCiData, err = impl.ciMaterialConfigService.GetGitCommitEnvVarDataForCICDStage(ciWf.GitTriggers) + gitTriggerEnvVariables, webhookAndCiData, err = impl.ciCdPipelineOrchestrator.GetGitCommitEnvVarDataForCICDStage(ciWf.GitTriggers) if err != nil { impl.logger.Errorw("error in getting gitTrigger env data for stage", "gitTriggers", ciWf.GitTriggers, "err", err) return nil, err diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/TriggerService.go index 82a1c62e86..bace5ad672 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService.go @@ -161,7 +161,7 @@ type TriggerServiceImpl struct { transactionUtilImpl *sql.TransactionUtilImpl deploymentConfigService common.DeploymentConfigService deploymentServiceTypeConfig *util3.DeploymentServiceTypeConfig - ciMaterialConfigService pipeline.CiMaterialConfigService + ciCdPipelineOrchestrator pipeline.CiCdPipelineOrchestrator } func NewTriggerServiceImpl(logger *zap.SugaredLogger, @@ -216,7 +216,7 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, K8sUtil *util5.K8sServiceImpl, transactionUtilImpl *sql.TransactionUtilImpl, deploymentConfigService common.DeploymentConfigService, - ciMaterialConfigService pipeline.CiMaterialConfigService, + ciCdPipelineOrchestrator pipeline.CiCdPipelineOrchestrator, ) (*TriggerServiceImpl, error) { impl := &TriggerServiceImpl{ logger: logger, @@ -272,7 +272,7 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, transactionUtilImpl: transactionUtilImpl, deploymentConfigService: deploymentConfigService, deploymentServiceTypeConfig: envVariables.DeploymentServiceTypeConfig, - ciMaterialConfigService: ciMaterialConfigService, + ciCdPipelineOrchestrator: ciCdPipelineOrchestrator, } config, err := types.GetCdConfig() if err != nil { diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index 4b58a25968..0ac3195add 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -28,6 +28,7 @@ import ( attributesBean "github.com/devtron-labs/devtron/pkg/attributes/bean" "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" + "github.com/devtron-labs/devtron/pkg/plugin" "golang.org/x/exp/slices" "net/http" "path" @@ -104,6 +105,7 @@ type CiCdPipelineOrchestrator interface { AddPipelineToTemplate(createRequest *bean.CiConfigRequest, isSwitchCiPipelineRequest bool) (resp *bean.CiConfigRequest, err error) GetSourceCiDownStreamFilters(ctx context.Context, sourceCiPipelineId int) (*CiPipeline.SourceCiDownStreamEnv, error) GetSourceCiDownStreamInfo(ctx context.Context, sourceCIPipeline int, req *CiPipeline.SourceCiDownStreamFilters) (pagination.PaginatedResponse[CiPipeline.SourceCiDownStreamResponse], error) + GetGitCommitEnvVarDataForCICDStage(gitTriggers map[int]pipelineConfig.GitCommit) (map[string]string, *gitSensor.WebhookAndCiData, error) } type CiCdPipelineOrchestratorImpl struct { @@ -2252,3 +2254,61 @@ func (impl CiCdPipelineOrchestratorImpl) GetSourceCiDownStreamInfo(ctx context.C response.PushData(data...) return response, nil } + +func (impl *CiCdPipelineOrchestratorImpl) GetGitCommitEnvVarDataForCICDStage(gitTriggers map[int]pipelineConfig.GitCommit) (map[string]string, *gitSensor.WebhookAndCiData, error) { + var webhookAndCiData *gitSensor.WebhookAndCiData + var err error + extraEnvVariables := make(map[string]string) + if gitTriggers != nil { + i := 1 + var gitCommitEnvVariables []types.GitMetadata + + for ciPipelineMaterialId, gitTrigger := range gitTriggers { + extraEnvVariables[fmt.Sprintf("%s_%d", types.GIT_COMMIT_HASH_PREFIX, i)] = gitTrigger.Commit + extraEnvVariables[fmt.Sprintf("%s_%d", types.GIT_SOURCE_TYPE_PREFIX, i)] = string(gitTrigger.CiConfigureSourceType) + extraEnvVariables[fmt.Sprintf("%s_%d", types.GIT_SOURCE_VALUE_PREFIX, i)] = gitTrigger.CiConfigureSourceValue + extraEnvVariables[fmt.Sprintf("%s_%d", types.GIT_REPO_URL_PREFIX, i)] = gitTrigger.GitRepoUrl + + gitCommitEnvVariables = append(gitCommitEnvVariables, types.GitMetadata{ + GitCommitHash: gitTrigger.Commit, + GitSourceType: string(gitTrigger.CiConfigureSourceType), + GitSourceValue: gitTrigger.CiConfigureSourceValue, + GitRepoUrl: gitTrigger.GitRepoUrl, + }) + + // CODE-BLOCK starts - store extra environment variables if webhook + if gitTrigger.CiConfigureSourceType == pipelineConfig.SOURCE_TYPE_WEBHOOK { + webhookDataId := gitTrigger.WebhookData.Id + if webhookDataId > 0 { + webhookDataRequest := &gitSensor.WebhookDataRequest{ + Id: webhookDataId, + CiPipelineMaterialId: ciPipelineMaterialId, + } + webhookAndCiData, err = impl.GitSensorClient.GetWebhookData(context.Background(), webhookDataRequest) + if err != nil { + impl.logger.Errorw("err while getting webhook data from git-sensor", "err", err, "webhookDataRequest", webhookDataRequest) + return nil, nil, err + } + if webhookAndCiData != nil { + for extEnvVariableKey, extEnvVariableVal := range webhookAndCiData.ExtraEnvironmentVariables { + extraEnvVariables[extEnvVariableKey] = extEnvVariableVal + } + } + } + } + // CODE_BLOCK ends + + i++ + } + gitMetadata, err := json.Marshal(&gitCommitEnvVariables) + if err != nil { + impl.logger.Errorw("err while marshaling git metdata", "err", err) + return nil, nil, err + } + extraEnvVariables[plugin.GIT_METADATA] = string(gitMetadata) + + extraEnvVariables[types.GIT_SOURCE_COUNT] = strconv.Itoa(len(gitTriggers)) + } + + return extraEnvVariables, webhookAndCiData, nil +} diff --git a/pkg/pipeline/CiMaterialConfigService.go b/pkg/pipeline/CiMaterialConfigService.go index d0db6e32a3..742c1f636e 100644 --- a/pkg/pipeline/CiMaterialConfigService.go +++ b/pkg/pipeline/CiMaterialConfigService.go @@ -17,20 +17,14 @@ package pipeline import ( - "context" - "encoding/json" "fmt" - gitSensorClient "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/pipeline/history" - "github.com/devtron-labs/devtron/pkg/pipeline/types" - "github.com/devtron-labs/devtron/pkg/plugin" "github.com/devtron-labs/devtron/pkg/sql" "github.com/go-pg/pg" "github.com/juju/errors" "go.uber.org/zap" - "strconv" "strings" "time" ) @@ -47,8 +41,6 @@ type CiMaterialConfigService interface { BulkPatchCiMaterialSource(ciPipelines *bean.CiMaterialBulkPatchRequest, userId int32, token string, checkAppSpecificAccess func(token, action string, appId int) (bool, error)) (*bean.CiMaterialBulkPatchResponse, error) //GetMaterialsForAppId : Retrieve material for given appId GetMaterialsForAppId(appId int) []*bean.GitMaterial - - GetGitCommitEnvVarDataForCICDStage(gitTriggers map[int]pipelineConfig.GitCommit) (map[string]string, *gitSensorClient.WebhookAndCiData, error) } type CiMaterialConfigServiceImpl struct { @@ -60,7 +52,6 @@ type CiMaterialConfigServiceImpl struct { gitMaterialHistoryService history.GitMaterialHistoryService pipelineRepository pipelineConfig.PipelineRepository ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository - gitSensorClient gitSensorClient.Client transactionManager sql.TransactionWrapper } @@ -73,7 +64,6 @@ func NewCiMaterialConfigServiceImpl( gitMaterialHistoryService history.GitMaterialHistoryService, pipelineRepository pipelineConfig.PipelineRepository, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, - gitSensorClient gitSensorClient.Client, transactionManager sql.TransactionWrapper) *CiMaterialConfigServiceImpl { return &CiMaterialConfigServiceImpl{ @@ -85,7 +75,6 @@ func NewCiMaterialConfigServiceImpl( gitMaterialHistoryService: gitMaterialHistoryService, pipelineRepository: pipelineRepository, ciPipelineMaterialRepository: ciPipelineMaterialRepository, - gitSensorClient: gitSensorClient, transactionManager: transactionManager, } } @@ -243,61 +232,3 @@ func (impl *CiMaterialConfigServiceImpl) GetMaterialsForAppId(appId int) []*bean } return gitMaterials } - -func (impl *CiMaterialConfigServiceImpl) GetGitCommitEnvVarDataForCICDStage(gitTriggers map[int]pipelineConfig.GitCommit) (map[string]string, *gitSensorClient.WebhookAndCiData, error) { - var webhookAndCiData *gitSensorClient.WebhookAndCiData - var err error - extraEnvVariables := make(map[string]string) - if gitTriggers != nil { - i := 1 - var gitCommitEnvVariables []types.GitMetadata - - for ciPipelineMaterialId, gitTrigger := range gitTriggers { - extraEnvVariables[fmt.Sprintf("%s_%d", types.GIT_COMMIT_HASH_PREFIX, i)] = gitTrigger.Commit - extraEnvVariables[fmt.Sprintf("%s_%d", types.GIT_SOURCE_TYPE_PREFIX, i)] = string(gitTrigger.CiConfigureSourceType) - extraEnvVariables[fmt.Sprintf("%s_%d", types.GIT_SOURCE_VALUE_PREFIX, i)] = gitTrigger.CiConfigureSourceValue - extraEnvVariables[fmt.Sprintf("%s_%d", types.GIT_REPO_URL_PREFIX, i)] = gitTrigger.GitRepoUrl - - gitCommitEnvVariables = append(gitCommitEnvVariables, types.GitMetadata{ - GitCommitHash: gitTrigger.Commit, - GitSourceType: string(gitTrigger.CiConfigureSourceType), - GitSourceValue: gitTrigger.CiConfigureSourceValue, - GitRepoUrl: gitTrigger.GitRepoUrl, - }) - - // CODE-BLOCK starts - store extra environment variables if webhook - if gitTrigger.CiConfigureSourceType == pipelineConfig.SOURCE_TYPE_WEBHOOK { - webhookDataId := gitTrigger.WebhookData.Id - if webhookDataId > 0 { - webhookDataRequest := &gitSensorClient.WebhookDataRequest{ - Id: webhookDataId, - CiPipelineMaterialId: ciPipelineMaterialId, - } - webhookAndCiData, err = impl.gitSensorClient.GetWebhookData(context.Background(), webhookDataRequest) - if err != nil { - impl.logger.Errorw("err while getting webhook data from git-sensor", "err", err, "webhookDataRequest", webhookDataRequest) - return nil, nil, err - } - if webhookAndCiData != nil { - for extEnvVariableKey, extEnvVariableVal := range webhookAndCiData.ExtraEnvironmentVariables { - extraEnvVariables[extEnvVariableKey] = extEnvVariableVal - } - } - } - } - // CODE_BLOCK ends - - i++ - } - gitMetadata, err := json.Marshal(&gitCommitEnvVariables) - if err != nil { - impl.logger.Errorw("err while marshaling git metdata", "err", err) - return nil, nil, err - } - extraEnvVariables[plugin.GIT_METADATA] = string(gitMetadata) - - extraEnvVariables[types.GIT_SOURCE_COUNT] = strconv.Itoa(len(gitTriggers)) - } - - return extraEnvVariables, webhookAndCiData, nil -} diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 790b21e169..ff8481e525 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -84,7 +84,7 @@ type CiServiceImpl struct { pluginInputVariableParser PluginInputVariableParser globalPluginService plugin.GlobalPluginService infraProvider infraProviders.InfraProvider - ciMaterialConfigService CiMaterialConfigService + ciCdPipelineOrchestrator CiCdPipelineOrchestrator } func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService, @@ -100,7 +100,7 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService pluginInputVariableParser PluginInputVariableParser, globalPluginService plugin.GlobalPluginService, infraProvider infraProviders.InfraProvider, - ciMaterialConfigService CiMaterialConfigService, + ciCdPipelineOrchestrator CiCdPipelineOrchestrator, ) *CiServiceImpl { cis := &CiServiceImpl{ Logger: Logger, @@ -121,7 +121,7 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService pluginInputVariableParser: pluginInputVariableParser, globalPluginService: globalPluginService, infraProvider: infraProvider, - ciMaterialConfigService: ciMaterialConfigService, + ciCdPipelineOrchestrator: ciCdPipelineOrchestrator, } config, err := types.GetCiConfig() if err != nil { @@ -221,7 +221,7 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) } // get env variables of git trigger data and add it in the extraEnvVariables - gitTriggerEnvVariables, _, err := impl.ciMaterialConfigService.GetGitCommitEnvVarDataForCICDStage(savedCiWf.GitTriggers) + gitTriggerEnvVariables, _, err := impl.ciCdPipelineOrchestrator.GetGitCommitEnvVarDataForCICDStage(savedCiWf.GitTriggers) if err != nil { impl.Logger.Errorw("error in getting gitTrigger env data for stage", "gitTriggers", savedCiWf.GitTriggers, "err", err) return 0, err diff --git a/wire_gen.go b/wire_gen.go index b41a225f28..512a320f15 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -560,8 +560,7 @@ func InitializeApp() (*App, error) { deploymentTemplateHistoryServiceImpl := history.NewDeploymentTemplateHistoryServiceImpl(sugaredLogger, deploymentTemplateHistoryRepositoryImpl, pipelineRepositoryImpl, chartRepositoryImpl, userServiceImpl, cdWorkflowRepositoryImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, utilMergeUtil, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl) ciCdPipelineOrchestratorImpl := pipeline.NewCiCdPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, clientImpl, ciCdConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, pipelineStageServiceImpl, gitMaterialHistoryServiceImpl, ciPipelineHistoryServiceImpl, ciTemplateServiceImpl, dockerArtifactStoreRepositoryImpl, ciArtifactRepositoryImpl, configMapServiceImpl, customTagServiceImpl, genericNoteServiceImpl, chartServiceImpl, transactionUtilImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl) - ciMaterialConfigServiceImpl := pipeline.NewCiMaterialConfigServiceImpl(sugaredLogger, materialRepositoryImpl, ciTemplateServiceImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, gitMaterialHistoryServiceImpl, pipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, clientImpl, transactionUtilImpl) - ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciWorkflowRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl, infraProviderImpl, ciMaterialConfigServiceImpl) + ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciWorkflowRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl, infraProviderImpl, ciCdPipelineOrchestratorImpl) ciLogServiceImpl, err := pipeline.NewCiLogServiceImpl(sugaredLogger, ciServiceImpl, k8sServiceImpl) if err != nil { return nil, err @@ -584,6 +583,7 @@ func InitializeApp() (*App, error) { ciTemplateHistoryServiceImpl := history.NewCiTemplateHistoryServiceImpl(ciTemplateHistoryRepositoryImpl, sugaredLogger) buildPipelineSwitchServiceImpl := pipeline.NewBuildPipelineSwitchServiceImpl(sugaredLogger, ciPipelineRepositoryImpl, ciCdPipelineOrchestratorImpl, pipelineRepositoryImpl, ciWorkflowRepositoryImpl, appWorkflowRepositoryImpl, ciPipelineHistoryServiceImpl, ciTemplateOverrideRepositoryImpl, ciPipelineMaterialRepositoryImpl) ciPipelineConfigServiceImpl := pipeline.NewCiPipelineConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, appRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ecrConfig, appWorkflowRepositoryImpl, ciCdConfig, attributesServiceImpl, pipelineStageServiceImpl, ciPipelineMaterialRepositoryImpl, ciTemplateServiceImpl, ciTemplateOverrideRepositoryImpl, ciTemplateHistoryServiceImpl, enforcerUtilImpl, ciWorkflowRepositoryImpl, resourceGroupServiceImpl, customTagServiceImpl, cdWorkflowRepositoryImpl, buildPipelineSwitchServiceImpl) + ciMaterialConfigServiceImpl := pipeline.NewCiMaterialConfigServiceImpl(sugaredLogger, materialRepositoryImpl, ciTemplateServiceImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, gitMaterialHistoryServiceImpl, pipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, transactionUtilImpl) deploymentGroupRepositoryImpl := repository2.NewDeploymentGroupRepositoryImpl(sugaredLogger, db) pipelineStrategyHistoryRepositoryImpl := repository12.NewPipelineStrategyHistoryRepositoryImpl(sugaredLogger, db) pipelineStrategyHistoryServiceImpl := history.NewPipelineStrategyHistoryServiceImpl(sugaredLogger, pipelineStrategyHistoryRepositoryImpl, userServiceImpl) @@ -644,7 +644,7 @@ func InitializeApp() (*App, error) { manifestPushConfigRepositoryImpl := repository10.NewManifestPushConfigRepository(sugaredLogger, db) scanToolExecutionHistoryMappingRepositoryImpl := security.NewScanToolExecutionHistoryMappingRepositoryImpl(db, sugaredLogger) imageScanServiceImpl := security2.NewImageScanServiceImpl(sugaredLogger, imageScanHistoryRepositoryImpl, imageScanResultRepositoryImpl, imageScanObjectMetaRepositoryImpl, cveStoreRepositoryImpl, imageScanDeployInfoRepositoryImpl, userServiceImpl, teamRepositoryImpl, appRepositoryImpl, environmentServiceImpl, ciArtifactRepositoryImpl, policyServiceImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, scanToolMetadataRepositoryImpl, scanToolExecutionHistoryMappingRepositoryImpl, cvePolicyRepositoryImpl) - triggerServiceImpl, err := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, argoUserServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, workflowServiceImpl, imageDigestPolicyServiceImpl, userServiceImpl, clientImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateServiceImpl, materialRepositoryImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, ciMaterialConfigServiceImpl) + triggerServiceImpl, err := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, argoUserServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, workflowServiceImpl, imageDigestPolicyServiceImpl, userServiceImpl, clientImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateServiceImpl, materialRepositoryImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, ciCdPipelineOrchestratorImpl) if err != nil { return nil, err }