Skip to content

Commit ae8578d

Browse files
gireesh-naidugireesh-naidu
authored
feat: expose git commit data as env vars for ci stage (#5534)
* feat: expose git commit data as env vars for ci stage * git-sensor client renaming * move the logic to CiCdPipelineOrchestrator.go
1 parent a4a910f commit ae8578d

File tree

7 files changed

+122
-78
lines changed

7 files changed

+122
-78
lines changed

‎pkg/deployment/trigger/devtronApps/PreStageTriggerService.go

Lines changed: 11 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -47,16 +47,13 @@ import (
4747
util2 "github.com/devtron-labs/devtron/util/event"
4848
"github.com/go-pg/pg"
4949
"go.opentelemetry.io/otel"
50+
"maps"
5051
"strconv"
5152
"strings"
5253
"time"
5354
)
5455

5556
const (
56-
GIT_COMMIT_HASH_PREFIX = "GIT_COMMIT_HASH"
57-
GIT_SOURCE_TYPE_PREFIX = "GIT_SOURCE_TYPE"
58-
GIT_SOURCE_VALUE_PREFIX = "GIT_SOURCE_VALUE"
59-
GIT_SOURCE_COUNT = "GIT_SOURCE_COUNT"
6057
APP_LABEL_KEY_PREFIX = "APP_LABEL_KEY"
6158
APP_LABEL_VALUE_PREFIX = "APP_LABEL_VALUE"
6259
APP_LABEL_COUNT = "APP_LABEL_COUNT"
@@ -582,55 +579,17 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow
582579
impl.logger.Errorw("error in getting ciWf by artifactId", "err", err, "artifactId", artifact.Id)
583580
return nil, err
584581
}
585-
var webhookAndCiData *gitSensorClient.WebhookAndCiData
586-
if ciWf != nil && ciWf.GitTriggers != nil {
587-
i := 1
588-
var gitCommitEnvVariables []types.GitMetadata
589-
590-
for ciPipelineMaterialId, gitTrigger := range ciWf.GitTriggers {
591-
extraEnvVariables[fmt.Sprintf("%s_%d", GIT_COMMIT_HASH_PREFIX, i)] = gitTrigger.Commit
592-
extraEnvVariables[fmt.Sprintf("%s_%d", GIT_SOURCE_TYPE_PREFIX, i)] = string(gitTrigger.CiConfigureSourceType)
593-
extraEnvVariables[fmt.Sprintf("%s_%d", GIT_SOURCE_VALUE_PREFIX, i)] = gitTrigger.CiConfigureSourceValue
594-
595-
gitCommitEnvVariables = append(gitCommitEnvVariables, types.GitMetadata{
596-
GitCommitHash: gitTrigger.Commit,
597-
GitSourceType: string(gitTrigger.CiConfigureSourceType),
598-
GitSourceValue: gitTrigger.CiConfigureSourceValue,
599-
})
600-
601-
// CODE-BLOCK starts - store extra environment variables if webhook
602-
if gitTrigger.CiConfigureSourceType == pipelineConfig.SOURCE_TYPE_WEBHOOK {
603-
webhookDataId := gitTrigger.WebhookData.Id
604-
if webhookDataId > 0 {
605-
webhookDataRequest := &gitSensorClient.WebhookDataRequest{
606-
Id: webhookDataId,
607-
CiPipelineMaterialId: ciPipelineMaterialId,
608-
}
609-
webhookAndCiData, err = impl.gitSensorGrpcClient.GetWebhookData(context.Background(), webhookDataRequest)
610-
if err != nil {
611-
impl.logger.Errorw("err while getting webhook data from git-sensor", "err", err, "webhookDataRequest", webhookDataRequest)
612-
return nil, err
613-
}
614-
if webhookAndCiData != nil {
615-
for extEnvVariableKey, extEnvVariableVal := range webhookAndCiData.ExtraEnvironmentVariables {
616-
extraEnvVariables[extEnvVariableKey] = extEnvVariableVal
617-
}
618-
}
619-
}
620-
}
621-
// CODE_BLOCK ends
622582

623-
i++
624-
}
625-
gitMetadata, err := json.Marshal(&gitCommitEnvVariables)
626-
if err != nil {
627-
impl.logger.Errorw("err while marshaling git metdata", "err", err)
628-
return nil, err
629-
}
630-
extraEnvVariables[plugin.GIT_METADATA] = string(gitMetadata)
583+
var webhookAndCiData *gitSensorClient.WebhookAndCiData
584+
var gitTriggerEnvVariables map[string]string
631585

632-
extraEnvVariables[GIT_SOURCE_COUNT] = strconv.Itoa(len(ciWf.GitTriggers))
586+
// get env variables of git trigger data and add it in the extraEnvVariables
587+
gitTriggerEnvVariables, webhookAndCiData, err = impl.ciCdPipelineOrchestrator.GetGitCommitEnvVarDataForCICDStage(ciWf.GitTriggers)
588+
if err != nil {
589+
impl.logger.Errorw("error in getting gitTrigger env data for stage", "gitTriggers", ciWf.GitTriggers, "err", err)
590+
return nil, err
633591
}
592+
maps.Copy(extraEnvVariables, gitTriggerEnvVariables)
634593

635594
childCdIds, err := impl.appWorkflowRepository.FindChildCDIdsByParentCDPipelineId(cdPipeline.Id)
636595
if err != nil && err != pg.ErrNoRows {
@@ -986,10 +945,10 @@ func (impl *TriggerServiceImpl) sendPreStageNotification(ctx context.Context, cd
986945
}
987946

988947
func isExtraVariableDynamic(variableName string, webhookAndCiData *gitSensorClient.WebhookAndCiData) bool {
989-
if strings.Contains(variableName, GIT_COMMIT_HASH_PREFIX) || strings.Contains(variableName, GIT_SOURCE_TYPE_PREFIX) || strings.Contains(variableName, GIT_SOURCE_VALUE_PREFIX) ||
948+
if strings.Contains(variableName, types.GIT_COMMIT_HASH_PREFIX) || strings.Contains(variableName, types.GIT_SOURCE_TYPE_PREFIX) || strings.Contains(variableName, types.GIT_SOURCE_VALUE_PREFIX) ||
990949
strings.Contains(variableName, APP_LABEL_VALUE_PREFIX) || strings.Contains(variableName, APP_LABEL_KEY_PREFIX) ||
991950
strings.Contains(variableName, CHILD_CD_ENV_NAME_PREFIX) || strings.Contains(variableName, CHILD_CD_CLUSTER_NAME_PREFIX) ||
992-
strings.Contains(variableName, CHILD_CD_COUNT) || strings.Contains(variableName, APP_LABEL_COUNT) || strings.Contains(variableName, GIT_SOURCE_COUNT) ||
951+
strings.Contains(variableName, CHILD_CD_COUNT) || strings.Contains(variableName, APP_LABEL_COUNT) || strings.Contains(variableName, types.GIT_SOURCE_COUNT) ||
993952
webhookAndCiData != nil {
994953

995954
return true

‎pkg/deployment/trigger/devtronApps/TriggerService.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ type TriggerServiceImpl struct {
131131
cdWorkflowService pipeline.WorkflowService
132132
imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService
133133
userService user.UserService
134-
gitSensorGrpcClient gitSensorClient.Client
134+
gitSensorClient gitSensorClient.Client
135135
config *types.CdConfig
136136
helmAppService client2.HelmAppService
137137
imageScanService security2.ImageScanService
@@ -161,6 +161,7 @@ type TriggerServiceImpl struct {
161161
transactionUtilImpl *sql.TransactionUtilImpl
162162
deploymentConfigService common.DeploymentConfigService
163163
deploymentServiceTypeConfig *util3.DeploymentServiceTypeConfig
164+
ciCdPipelineOrchestrator pipeline.CiCdPipelineOrchestrator
164165
}
165166

166167
func NewTriggerServiceImpl(logger *zap.SugaredLogger,
@@ -185,7 +186,7 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger,
185186
cdWorkflowService pipeline.WorkflowService,
186187
imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService,
187188
userService user.UserService,
188-
gitSensorGrpcClient gitSensorClient.Client,
189+
gitSensorClient gitSensorClient.Client,
189190
helmAppService client2.HelmAppService,
190191
enforcerUtil rbac.EnforcerUtil,
191192
userDeploymentRequestService service.UserDeploymentRequestService,
@@ -215,6 +216,7 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger,
215216
K8sUtil *util5.K8sServiceImpl,
216217
transactionUtilImpl *sql.TransactionUtilImpl,
217218
deploymentConfigService common.DeploymentConfigService,
219+
ciCdPipelineOrchestrator pipeline.CiCdPipelineOrchestrator,
218220
) (*TriggerServiceImpl, error) {
219221
impl := &TriggerServiceImpl{
220222
logger: logger,
@@ -239,7 +241,7 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger,
239241
cdWorkflowService: cdWorkflowService,
240242
imageDigestPolicyService: imageDigestPolicyService,
241243
userService: userService,
242-
gitSensorGrpcClient: gitSensorGrpcClient,
244+
gitSensorClient: gitSensorClient,
243245
helmAppService: helmAppService,
244246
enforcerUtil: enforcerUtil,
245247
eventFactory: eventFactory,
@@ -270,6 +272,7 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger,
270272
transactionUtilImpl: transactionUtilImpl,
271273
deploymentConfigService: deploymentConfigService,
272274
deploymentServiceTypeConfig: envVariables.DeploymentServiceTypeConfig,
275+
ciCdPipelineOrchestrator: ciCdPipelineOrchestrator,
273276
}
274277
config, err := types.GetCdConfig()
275278
if err != nil {

‎pkg/pipeline/CiCdPipelineOrchestrator.go

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828
attributesBean "github.com/devtron-labs/devtron/pkg/attributes/bean"
2929
"github.com/devtron-labs/devtron/pkg/deployment/common"
3030
"github.com/devtron-labs/devtron/pkg/deployment/gitOps/config"
31+
"github.com/devtron-labs/devtron/pkg/plugin"
3132
"golang.org/x/exp/slices"
3233
"net/http"
3334
"path"
@@ -104,6 +105,7 @@ type CiCdPipelineOrchestrator interface {
104105
AddPipelineToTemplate(createRequest *bean.CiConfigRequest, isSwitchCiPipelineRequest bool) (resp *bean.CiConfigRequest, err error)
105106
GetSourceCiDownStreamFilters(ctx context.Context, sourceCiPipelineId int) (*CiPipeline.SourceCiDownStreamEnv, error)
106107
GetSourceCiDownStreamInfo(ctx context.Context, sourceCIPipeline int, req *CiPipeline.SourceCiDownStreamFilters) (pagination.PaginatedResponse[CiPipeline.SourceCiDownStreamResponse], error)
108+
GetGitCommitEnvVarDataForCICDStage(gitTriggers map[int]pipelineConfig.GitCommit) (map[string]string, *gitSensor.WebhookAndCiData, error)
107109
}
108110

109111
type CiCdPipelineOrchestratorImpl struct {
@@ -2252,3 +2254,61 @@ func (impl CiCdPipelineOrchestratorImpl) GetSourceCiDownStreamInfo(ctx context.C
22522254
response.PushData(data...)
22532255
return response, nil
22542256
}
2257+
2258+
func (impl *CiCdPipelineOrchestratorImpl) GetGitCommitEnvVarDataForCICDStage(gitTriggers map[int]pipelineConfig.GitCommit) (map[string]string, *gitSensor.WebhookAndCiData, error) {
2259+
var webhookAndCiData *gitSensor.WebhookAndCiData
2260+
var err error
2261+
extraEnvVariables := make(map[string]string)
2262+
if gitTriggers != nil {
2263+
i := 1
2264+
var gitCommitEnvVariables []types.GitMetadata
2265+
2266+
for ciPipelineMaterialId, gitTrigger := range gitTriggers {
2267+
extraEnvVariables[fmt.Sprintf("%s_%d", types.GIT_COMMIT_HASH_PREFIX, i)] = gitTrigger.Commit
2268+
extraEnvVariables[fmt.Sprintf("%s_%d", types.GIT_SOURCE_TYPE_PREFIX, i)] = string(gitTrigger.CiConfigureSourceType)
2269+
extraEnvVariables[fmt.Sprintf("%s_%d", types.GIT_SOURCE_VALUE_PREFIX, i)] = gitTrigger.CiConfigureSourceValue
2270+
extraEnvVariables[fmt.Sprintf("%s_%d", types.GIT_REPO_URL_PREFIX, i)] = gitTrigger.GitRepoUrl
2271+
2272+
gitCommitEnvVariables = append(gitCommitEnvVariables, types.GitMetadata{
2273+
GitCommitHash: gitTrigger.Commit,
2274+
GitSourceType: string(gitTrigger.CiConfigureSourceType),
2275+
GitSourceValue: gitTrigger.CiConfigureSourceValue,
2276+
GitRepoUrl: gitTrigger.GitRepoUrl,
2277+
})
2278+
2279+
// CODE-BLOCK starts - store extra environment variables if webhook
2280+
if gitTrigger.CiConfigureSourceType == pipelineConfig.SOURCE_TYPE_WEBHOOK {
2281+
webhookDataId := gitTrigger.WebhookData.Id
2282+
if webhookDataId > 0 {
2283+
webhookDataRequest := &gitSensor.WebhookDataRequest{
2284+
Id: webhookDataId,
2285+
CiPipelineMaterialId: ciPipelineMaterialId,
2286+
}
2287+
webhookAndCiData, err = impl.GitSensorClient.GetWebhookData(context.Background(), webhookDataRequest)
2288+
if err != nil {
2289+
impl.logger.Errorw("err while getting webhook data from git-sensor", "err", err, "webhookDataRequest", webhookDataRequest)
2290+
return nil, nil, err
2291+
}
2292+
if webhookAndCiData != nil {
2293+
for extEnvVariableKey, extEnvVariableVal := range webhookAndCiData.ExtraEnvironmentVariables {
2294+
extraEnvVariables[extEnvVariableKey] = extEnvVariableVal
2295+
}
2296+
}
2297+
}
2298+
}
2299+
// CODE_BLOCK ends
2300+
2301+
i++
2302+
}
2303+
gitMetadata, err := json.Marshal(&gitCommitEnvVariables)
2304+
if err != nil {
2305+
impl.logger.Errorw("err while marshaling git metdata", "err", err)
2306+
return nil, nil, err
2307+
}
2308+
extraEnvVariables[plugin.GIT_METADATA] = string(gitMetadata)
2309+
2310+
extraEnvVariables[types.GIT_SOURCE_COUNT] = strconv.Itoa(len(gitTriggers))
2311+
}
2312+
2313+
return extraEnvVariables, webhookAndCiData, nil
2314+
}

‎pkg/pipeline/CiMaterialConfigService.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,8 @@ func NewCiMaterialConfigServiceImpl(
6363
ciPipelineRepository pipelineConfig.CiPipelineRepository,
6464
gitMaterialHistoryService history.GitMaterialHistoryService,
6565
pipelineRepository pipelineConfig.PipelineRepository,
66-
ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, transactionManager sql.TransactionWrapper) *CiMaterialConfigServiceImpl {
66+
ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository,
67+
transactionManager sql.TransactionWrapper) *CiMaterialConfigServiceImpl {
6768

6869
return &CiMaterialConfigServiceImpl{
6970
logger: logger,

‎pkg/pipeline/CiService.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"github.com/devtron-labs/devtron/pkg/pipeline/adapter"
2525
"github.com/devtron-labs/devtron/pkg/pipeline/bean/CiPipeline"
2626
"github.com/devtron-labs/devtron/pkg/pipeline/infraProviders"
27+
"maps"
2728
"path/filepath"
2829
"strconv"
2930
"strings"
@@ -83,6 +84,7 @@ type CiServiceImpl struct {
8384
pluginInputVariableParser PluginInputVariableParser
8485
globalPluginService plugin.GlobalPluginService
8586
infraProvider infraProviders.InfraProvider
87+
ciCdPipelineOrchestrator CiCdPipelineOrchestrator
8688
}
8789

8890
func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService,
@@ -98,6 +100,7 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService
98100
pluginInputVariableParser PluginInputVariableParser,
99101
globalPluginService plugin.GlobalPluginService,
100102
infraProvider infraProviders.InfraProvider,
103+
ciCdPipelineOrchestrator CiCdPipelineOrchestrator,
101104
) *CiServiceImpl {
102105
cis := &CiServiceImpl{
103106
Logger: Logger,
@@ -118,6 +121,7 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService
118121
pluginInputVariableParser: pluginInputVariableParser,
119122
globalPluginService: globalPluginService,
120123
infraProvider: infraProvider,
124+
ciCdPipelineOrchestrator: ciCdPipelineOrchestrator,
121125
}
122126
config, err := types.GetCiConfig()
123127
if err != nil {
@@ -222,6 +226,14 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error)
222226
return 0, err
223227
}
224228

229+
// get env variables of git trigger data and add it in the extraEnvVariables
230+
gitTriggerEnvVariables, _, err := impl.ciCdPipelineOrchestrator.GetGitCommitEnvVarDataForCICDStage(savedCiWf.GitTriggers)
231+
if err != nil {
232+
impl.Logger.Errorw("error in getting gitTrigger env data for stage", "gitTriggers", savedCiWf.GitTriggers, "err", err)
233+
return 0, err
234+
}
235+
maps.Copy(trigger.ExtraEnvironmentVariables, gitTriggerEnvVariables)
236+
225237
workflowRequest, err := impl.buildWfRequestForCiPipeline(pipeline, trigger, ciMaterials, savedCiWf, ciWorkflowConfig, ciPipelineScripts, preCiSteps, postCiSteps, refPluginsData, isJob)
226238
if err != nil {
227239
impl.Logger.Errorw("make workflow req", "err", err)

‎pkg/pipeline/types/Workflow.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,14 @@ import (
4040
"time"
4141
)
4242

43+
const (
44+
GIT_REPO_URL_PREFIX = "GIT_REPO_URL"
45+
GIT_COMMIT_HASH_PREFIX = "GIT_COMMIT_HASH"
46+
GIT_SOURCE_TYPE_PREFIX = "GIT_SOURCE_TYPE"
47+
GIT_SOURCE_VALUE_PREFIX = "GIT_SOURCE_VALUE"
48+
GIT_SOURCE_COUNT = "GIT_SOURCE_COUNT"
49+
)
50+
4351
type WorkflowRequest struct {
4452
WorkflowNamePrefix string `json:"workflowNamePrefix"`
4553
PipelineName string `json:"pipelineName"`
@@ -596,6 +604,7 @@ type GitMetadata struct {
596604
GitCommitHash string `json:"GIT_COMMIT_HASH"`
597605
GitSourceType string `json:"GIT_SOURCE_TYPE"`
598606
GitSourceValue string `json:"GIT_SOURCE_VALUE"`
607+
GitRepoUrl string `json:"GIT_REPO_URL"`
599608
}
600609

601610
type AppLabelMetadata struct {

0 commit comments

Comments
 (0)