Skip to content

feat: expose git commit data as env vars for ci stage #5534

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jul 22, 2024
Merged

feat: expose git commit data as env vars for ci stage #5534

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6517318
feat: expose git commit data as env vars for ci stage
gireesh-naidu Jul 22, 2024
c98f6e9
git-sensor client renaming
gireesh-naidu Jul 22, 2024
5c5e9d3
move the logic to CiCdPipelineOrchestrator.go
gireesh-naidu Jul 22, 2024
b76e755
Merge branch 'main' into git-trigger-env-data-ci-oss
gireesh-naidu Jul 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 11 additions & 52 deletions pkg/deployment/trigger/devtronApps/PreStageTriggerService.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,16 +47,13 @@ import (
util2 "github.com/devtron-labs/devtron/util/event"
"github.com/go-pg/pg"
"go.opentelemetry.io/otel"
"maps"
"strconv"
"strings"
"time"
)

const (
GIT_COMMIT_HASH_PREFIX = "GIT_COMMIT_HASH"
GIT_SOURCE_TYPE_PREFIX = "GIT_SOURCE_TYPE"
GIT_SOURCE_VALUE_PREFIX = "GIT_SOURCE_VALUE"
GIT_SOURCE_COUNT = "GIT_SOURCE_COUNT"
APP_LABEL_KEY_PREFIX = "APP_LABEL_KEY"
APP_LABEL_VALUE_PREFIX = "APP_LABEL_VALUE"
APP_LABEL_COUNT = "APP_LABEL_COUNT"
Expand Down Expand Up @@ -582,55 +579,17 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow
impl.logger.Errorw("error in getting ciWf by artifactId", "err", err, "artifactId", artifact.Id)
return nil, err
}
var webhookAndCiData *gitSensorClient.WebhookAndCiData
if ciWf != nil && ciWf.GitTriggers != nil {
i := 1
var gitCommitEnvVariables []types.GitMetadata

for ciPipelineMaterialId, gitTrigger := range ciWf.GitTriggers {
extraEnvVariables[fmt.Sprintf("%s_%d", GIT_COMMIT_HASH_PREFIX, i)] = gitTrigger.Commit
extraEnvVariables[fmt.Sprintf("%s_%d", GIT_SOURCE_TYPE_PREFIX, i)] = string(gitTrigger.CiConfigureSourceType)
extraEnvVariables[fmt.Sprintf("%s_%d", GIT_SOURCE_VALUE_PREFIX, i)] = gitTrigger.CiConfigureSourceValue

gitCommitEnvVariables = append(gitCommitEnvVariables, types.GitMetadata{
GitCommitHash: gitTrigger.Commit,
GitSourceType: string(gitTrigger.CiConfigureSourceType),
GitSourceValue: gitTrigger.CiConfigureSourceValue,
})

// CODE-BLOCK starts - store extra environment variables if webhook
if gitTrigger.CiConfigureSourceType == pipelineConfig.SOURCE_TYPE_WEBHOOK {
webhookDataId := gitTrigger.WebhookData.Id
if webhookDataId > 0 {
webhookDataRequest := &gitSensorClient.WebhookDataRequest{
Id: webhookDataId,
CiPipelineMaterialId: ciPipelineMaterialId,
}
webhookAndCiData, err = impl.gitSensorGrpcClient.GetWebhookData(context.Background(), webhookDataRequest)
if err != nil {
impl.logger.Errorw("err while getting webhook data from git-sensor", "err", err, "webhookDataRequest", webhookDataRequest)
return nil, err
}
if webhookAndCiData != nil {
for extEnvVariableKey, extEnvVariableVal := range webhookAndCiData.ExtraEnvironmentVariables {
extraEnvVariables[extEnvVariableKey] = extEnvVariableVal
}
}
}
}
// CODE_BLOCK ends

i++
}
gitMetadata, err := json.Marshal(&gitCommitEnvVariables)
if err != nil {
impl.logger.Errorw("err while marshaling git metdata", "err", err)
return nil, err
}
extraEnvVariables[plugin.GIT_METADATA] = string(gitMetadata)
var webhookAndCiData *gitSensorClient.WebhookAndCiData
var gitTriggerEnvVariables map[string]string

extraEnvVariables[GIT_SOURCE_COUNT] = strconv.Itoa(len(ciWf.GitTriggers))
// get env variables of git trigger data and add it in the extraEnvVariables
gitTriggerEnvVariables, webhookAndCiData, err = impl.ciCdPipelineOrchestrator.GetGitCommitEnvVarDataForCICDStage(ciWf.GitTriggers)
if err != nil {
impl.logger.Errorw("error in getting gitTrigger env data for stage", "gitTriggers", ciWf.GitTriggers, "err", err)
return nil, err
}
maps.Copy(extraEnvVariables, gitTriggerEnvVariables)

childCdIds, err := impl.appWorkflowRepository.FindChildCDIdsByParentCDPipelineId(cdPipeline.Id)
if err != nil && err != pg.ErrNoRows {
Expand Down Expand Up @@ -986,10 +945,10 @@ func (impl *TriggerServiceImpl) sendPreStageNotification(ctx context.Context, cd
}

func isExtraVariableDynamic(variableName string, webhookAndCiData *gitSensorClient.WebhookAndCiData) bool {
if strings.Contains(variableName, GIT_COMMIT_HASH_PREFIX) || strings.Contains(variableName, GIT_SOURCE_TYPE_PREFIX) || strings.Contains(variableName, GIT_SOURCE_VALUE_PREFIX) ||
if strings.Contains(variableName, types.GIT_COMMIT_HASH_PREFIX) || strings.Contains(variableName, types.GIT_SOURCE_TYPE_PREFIX) || strings.Contains(variableName, types.GIT_SOURCE_VALUE_PREFIX) ||
strings.Contains(variableName, APP_LABEL_VALUE_PREFIX) || strings.Contains(variableName, APP_LABEL_KEY_PREFIX) ||
strings.Contains(variableName, CHILD_CD_ENV_NAME_PREFIX) || strings.Contains(variableName, CHILD_CD_CLUSTER_NAME_PREFIX) ||
strings.Contains(variableName, CHILD_CD_COUNT) || strings.Contains(variableName, APP_LABEL_COUNT) || strings.Contains(variableName, GIT_SOURCE_COUNT) ||
strings.Contains(variableName, CHILD_CD_COUNT) || strings.Contains(variableName, APP_LABEL_COUNT) || strings.Contains(variableName, types.GIT_SOURCE_COUNT) ||
webhookAndCiData != nil {

return true
Expand Down
9 changes: 6 additions & 3 deletions pkg/deployment/trigger/devtronApps/TriggerService.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ type TriggerServiceImpl struct {
cdWorkflowService pipeline.WorkflowService
imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService
userService user.UserService
gitSensorGrpcClient gitSensorClient.Client
gitSensorClient gitSensorClient.Client
config *types.CdConfig
helmAppService client2.HelmAppService
imageScanService security2.ImageScanService
Expand Down Expand Up @@ -161,6 +161,7 @@ type TriggerServiceImpl struct {
transactionUtilImpl *sql.TransactionUtilImpl
deploymentConfigService common.DeploymentConfigService
deploymentServiceTypeConfig *util3.DeploymentServiceTypeConfig
ciCdPipelineOrchestrator pipeline.CiCdPipelineOrchestrator
}

func NewTriggerServiceImpl(logger *zap.SugaredLogger,
Expand All @@ -185,7 +186,7 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger,
cdWorkflowService pipeline.WorkflowService,
imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService,
userService user.UserService,
gitSensorGrpcClient gitSensorClient.Client,
gitSensorClient gitSensorClient.Client,
helmAppService client2.HelmAppService,
enforcerUtil rbac.EnforcerUtil,
userDeploymentRequestService service.UserDeploymentRequestService,
Expand Down Expand Up @@ -215,6 +216,7 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger,
K8sUtil *util5.K8sServiceImpl,
transactionUtilImpl *sql.TransactionUtilImpl,
deploymentConfigService common.DeploymentConfigService,
ciCdPipelineOrchestrator pipeline.CiCdPipelineOrchestrator,
) (*TriggerServiceImpl, error) {
impl := &TriggerServiceImpl{
logger: logger,
Expand All @@ -239,7 +241,7 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger,
cdWorkflowService: cdWorkflowService,
imageDigestPolicyService: imageDigestPolicyService,
userService: userService,
gitSensorGrpcClient: gitSensorGrpcClient,
gitSensorClient: gitSensorClient,
helmAppService: helmAppService,
enforcerUtil: enforcerUtil,
eventFactory: eventFactory,
Expand Down Expand Up @@ -270,6 +272,7 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger,
transactionUtilImpl: transactionUtilImpl,
deploymentConfigService: deploymentConfigService,
deploymentServiceTypeConfig: envVariables.DeploymentServiceTypeConfig,
ciCdPipelineOrchestrator: ciCdPipelineOrchestrator,
}
config, err := types.GetCdConfig()
if err != nil {
Expand Down
60 changes: 60 additions & 0 deletions pkg/pipeline/CiCdPipelineOrchestrator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
attributesBean "github.com/devtron-labs/devtron/pkg/attributes/bean"
"github.com/devtron-labs/devtron/pkg/deployment/common"
"github.com/devtron-labs/devtron/pkg/deployment/gitOps/config"
"github.com/devtron-labs/devtron/pkg/plugin"
"golang.org/x/exp/slices"
"net/http"
"path"
Expand Down Expand Up @@ -104,6 +105,7 @@ type CiCdPipelineOrchestrator interface {
AddPipelineToTemplate(createRequest *bean.CiConfigRequest, isSwitchCiPipelineRequest bool) (resp *bean.CiConfigRequest, err error)
GetSourceCiDownStreamFilters(ctx context.Context, sourceCiPipelineId int) (*CiPipeline.SourceCiDownStreamEnv, error)
GetSourceCiDownStreamInfo(ctx context.Context, sourceCIPipeline int, req *CiPipeline.SourceCiDownStreamFilters) (pagination.PaginatedResponse[CiPipeline.SourceCiDownStreamResponse], error)
GetGitCommitEnvVarDataForCICDStage(gitTriggers map[int]pipelineConfig.GitCommit) (map[string]string, *gitSensor.WebhookAndCiData, error)
}

type CiCdPipelineOrchestratorImpl struct {
Expand Down Expand Up @@ -2252,3 +2254,61 @@ func (impl CiCdPipelineOrchestratorImpl) GetSourceCiDownStreamInfo(ctx context.C
response.PushData(data...)
return response, nil
}

func (impl *CiCdPipelineOrchestratorImpl) GetGitCommitEnvVarDataForCICDStage(gitTriggers map[int]pipelineConfig.GitCommit) (map[string]string, *gitSensor.WebhookAndCiData, error) {
var webhookAndCiData *gitSensor.WebhookAndCiData
var err error
extraEnvVariables := make(map[string]string)
if gitTriggers != nil {
i := 1
var gitCommitEnvVariables []types.GitMetadata

for ciPipelineMaterialId, gitTrigger := range gitTriggers {
extraEnvVariables[fmt.Sprintf("%s_%d", types.GIT_COMMIT_HASH_PREFIX, i)] = gitTrigger.Commit
extraEnvVariables[fmt.Sprintf("%s_%d", types.GIT_SOURCE_TYPE_PREFIX, i)] = string(gitTrigger.CiConfigureSourceType)
extraEnvVariables[fmt.Sprintf("%s_%d", types.GIT_SOURCE_VALUE_PREFIX, i)] = gitTrigger.CiConfigureSourceValue
extraEnvVariables[fmt.Sprintf("%s_%d", types.GIT_REPO_URL_PREFIX, i)] = gitTrigger.GitRepoUrl

gitCommitEnvVariables = append(gitCommitEnvVariables, types.GitMetadata{
GitCommitHash: gitTrigger.Commit,
GitSourceType: string(gitTrigger.CiConfigureSourceType),
GitSourceValue: gitTrigger.CiConfigureSourceValue,
GitRepoUrl: gitTrigger.GitRepoUrl,
})

// CODE-BLOCK starts - store extra environment variables if webhook
if gitTrigger.CiConfigureSourceType == pipelineConfig.SOURCE_TYPE_WEBHOOK {
webhookDataId := gitTrigger.WebhookData.Id
if webhookDataId > 0 {
webhookDataRequest := &gitSensor.WebhookDataRequest{
Id: webhookDataId,
CiPipelineMaterialId: ciPipelineMaterialId,
}
webhookAndCiData, err = impl.GitSensorClient.GetWebhookData(context.Background(), webhookDataRequest)
if err != nil {
impl.logger.Errorw("err while getting webhook data from git-sensor", "err", err, "webhookDataRequest", webhookDataRequest)
return nil, nil, err
}
if webhookAndCiData != nil {
for extEnvVariableKey, extEnvVariableVal := range webhookAndCiData.ExtraEnvironmentVariables {
extraEnvVariables[extEnvVariableKey] = extEnvVariableVal
}
}
}
}
// CODE_BLOCK ends

i++
}
gitMetadata, err := json.Marshal(&gitCommitEnvVariables)
if err != nil {
impl.logger.Errorw("err while marshaling git metdata", "err", err)
return nil, nil, err
}
extraEnvVariables[plugin.GIT_METADATA] = string(gitMetadata)

extraEnvVariables[types.GIT_SOURCE_COUNT] = strconv.Itoa(len(gitTriggers))
}

return extraEnvVariables, webhookAndCiData, nil
}
3 changes: 2 additions & 1 deletion pkg/pipeline/CiMaterialConfigService.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@ func NewCiMaterialConfigServiceImpl(
ciPipelineRepository pipelineConfig.CiPipelineRepository,
gitMaterialHistoryService history.GitMaterialHistoryService,
pipelineRepository pipelineConfig.PipelineRepository,
ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, transactionManager sql.TransactionWrapper) *CiMaterialConfigServiceImpl {
ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository,
transactionManager sql.TransactionWrapper) *CiMaterialConfigServiceImpl {

return &CiMaterialConfigServiceImpl{
logger: logger,
Expand Down
12 changes: 12 additions & 0 deletions pkg/pipeline/CiService.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"github.com/devtron-labs/devtron/pkg/pipeline/adapter"
"github.com/devtron-labs/devtron/pkg/pipeline/bean/CiPipeline"
"github.com/devtron-labs/devtron/pkg/pipeline/infraProviders"
"maps"
"path/filepath"
"strconv"
"strings"
Expand Down Expand Up @@ -83,6 +84,7 @@ type CiServiceImpl struct {
pluginInputVariableParser PluginInputVariableParser
globalPluginService plugin.GlobalPluginService
infraProvider infraProviders.InfraProvider
ciCdPipelineOrchestrator CiCdPipelineOrchestrator
}

func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService,
Expand All @@ -98,6 +100,7 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService
pluginInputVariableParser PluginInputVariableParser,
globalPluginService plugin.GlobalPluginService,
infraProvider infraProviders.InfraProvider,
ciCdPipelineOrchestrator CiCdPipelineOrchestrator,
) *CiServiceImpl {
cis := &CiServiceImpl{
Logger: Logger,
Expand All @@ -118,6 +121,7 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService
pluginInputVariableParser: pluginInputVariableParser,
globalPluginService: globalPluginService,
infraProvider: infraProvider,
ciCdPipelineOrchestrator: ciCdPipelineOrchestrator,
}
config, err := types.GetCiConfig()
if err != nil {
Expand Down Expand Up @@ -222,6 +226,14 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error)
return 0, err
}

// get env variables of git trigger data and add it in the extraEnvVariables
gitTriggerEnvVariables, _, err := impl.ciCdPipelineOrchestrator.GetGitCommitEnvVarDataForCICDStage(savedCiWf.GitTriggers)
if err != nil {
impl.Logger.Errorw("error in getting gitTrigger env data for stage", "gitTriggers", savedCiWf.GitTriggers, "err", err)
return 0, err
}
maps.Copy(trigger.ExtraEnvironmentVariables, gitTriggerEnvVariables)

workflowRequest, err := impl.buildWfRequestForCiPipeline(pipeline, trigger, ciMaterials, savedCiWf, ciWorkflowConfig, ciPipelineScripts, preCiSteps, postCiSteps, refPluginsData, isJob)
if err != nil {
impl.Logger.Errorw("make workflow req", "err", err)
Expand Down
9 changes: 9 additions & 0 deletions pkg/pipeline/types/Workflow.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,14 @@ import (
"time"
)

const (
GIT_REPO_URL_PREFIX = "GIT_REPO_URL"
GIT_COMMIT_HASH_PREFIX = "GIT_COMMIT_HASH"
GIT_SOURCE_TYPE_PREFIX = "GIT_SOURCE_TYPE"
GIT_SOURCE_VALUE_PREFIX = "GIT_SOURCE_VALUE"
GIT_SOURCE_COUNT = "GIT_SOURCE_COUNT"
)

type WorkflowRequest struct {
WorkflowNamePrefix string `json:"workflowNamePrefix"`
PipelineName string `json:"pipelineName"`
Expand Down Expand Up @@ -596,6 +604,7 @@ type GitMetadata struct {
GitCommitHash string `json:"GIT_COMMIT_HASH"`
GitSourceType string `json:"GIT_SOURCE_TYPE"`
GitSourceValue string `json:"GIT_SOURCE_VALUE"`
GitRepoUrl string `json:"GIT_REPO_URL"`
}

type AppLabelMetadata struct {
Expand Down
Loading
Loading