diff --git a/api/bean/ConfigMapAndSecret.go b/api/bean/ConfigMapAndSecret.go index c054f44ed3..fcd5340bea 100644 --- a/api/bean/ConfigMapAndSecret.go +++ b/api/bean/ConfigMapAndSecret.go @@ -57,14 +57,19 @@ type ConfigSecretMap struct { SubPath bool `json:"subPath"` ESOSubPath []string `json:"esoSubPath"` FilePermission string `json:"filePermission"` + ConfigSecretMapEnt } -func (configSecret ConfigSecretMap) GetDataMap() (map[string]string, error) { - var datamap map[string]string - err := json.Unmarshal(configSecret.Data, &datamap) - return datamap, err +func (configSecret *ConfigSecretMap) GetDataMap() (map[string]string, error) { + if len(configSecret.Data) == 0 { + return make(map[string]string), nil + } + var dataMap map[string]string + err := json.Unmarshal(configSecret.Data, &dataMap) + return dataMap, err } -func (configSecretJson ConfigSecretJson) GetDereferencedSecrets() []ConfigSecretMap { + +func (configSecretJson *ConfigSecretJson) GetDereferencedSecrets() []ConfigSecretMap { return sliceUtil.GetDeReferencedSlice(configSecretJson.Secrets) } @@ -95,3 +100,14 @@ func GetTransformedDataForSecretRootJsonData(data string, mode util.SecretTransf } return string(marshal), nil } + +type ConfigType string + +func (c ConfigType) String() string { + return string(c) +} + +const ( + ConfigMap ConfigType = "cm" + Secret ConfigType = "cs" +) diff --git a/api/bean/ConfigMapAndSecret_ent.go b/api/bean/ConfigMapAndSecret_ent.go new file mode 100644 index 0000000000..0b2e110a01 --- /dev/null +++ b/api/bean/ConfigMapAndSecret_ent.go @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package bean + +type ConfigSecretMapEnt struct { +} + +func (configSecret *ConfigSecretMap) AddDataToKey(keyName string, data []byte) (*ConfigSecretMap, error) { + return configSecret, nil +} + +func (configSecret *ConfigSecretMap) GetBinaryDataMap() map[string][]byte { + return nil +} diff --git a/go.mod b/go.mod index 5160cd6137..02d2e2c75f 100644 --- a/go.mod +++ b/go.mod @@ -307,8 +307,8 @@ require ( replace ( github.com/argoproj/argo-workflows/v3 v3.5.13 => github.com/devtron-labs/argo-workflows/v3 v3.5.13 - github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250428081808-439aa2bff7da - github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250428081808-439aa2bff7da + github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250504164106-4a6b1415f5d9 + github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250504164106-4a6b1415f5d9 github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127 github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5 k8s.io/api => k8s.io/api v0.29.7 diff --git a/go.sum b/go.sum index 375d57fd91..cd7fd2e092 100644 --- a/go.sum +++ b/go.sum @@ -829,10 +829,10 @@ github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzq github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/devtron-labs/argo-workflows/v3 v3.5.13 h1:3pINq0gXOSeTw2z/vYe+j80lRpSN5Rp/8mfQORh8SmU= github.com/devtron-labs/argo-workflows/v3 v3.5.13/go.mod h1:/vqxcovDPT4zqr4DjR5v7CF8ggpY1l3TSa2CIG3jmjA= -github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250428081808-439aa2bff7da h1:F9S3HhyMbHCvp7mZdhnghTmaKFwWvevnM7PRTJn4YRE= -github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250428081808-439aa2bff7da/go.mod h1:FfaLDXN1ZXxyRpnskBqVIYkpkWDCzBmDgIO9xqLnxdQ= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250428081808-439aa2bff7da h1:vYQF82zOxzgSxuwc/h67Cbcbvjs/m9oUI+ewjfK1CrA= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250428081808-439aa2bff7da/go.mod h1:zkNShlkcHxsmnL0gKNbs0uyRL8lZonGKr5Km63uTLI0= +github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250504164106-4a6b1415f5d9 h1:+RV5OQ8pNMjqebMbnnhgUSlkmJbDws87jZt9ii7O9X8= +github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250504164106-4a6b1415f5d9/go.mod h1:FfaLDXN1ZXxyRpnskBqVIYkpkWDCzBmDgIO9xqLnxdQ= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250504164106-4a6b1415f5d9 h1:Uig0hJPJw20IfGP7Gc47z5HccKkdar3BjlIvhBZh1QM= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250504164106-4a6b1415f5d9/go.mod h1:zkNShlkcHxsmnL0gKNbs0uyRL8lZonGKr5Km63uTLI0= github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU= github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y= github.com/devtron-labs/protos v0.0.3-0.20250323220609-ecf8a0f7305e h1:U6UdYbW8a7xn5IzFPd8cywjVVPfutGJCudjePAfL/Hs= diff --git a/pkg/deployment/trigger/devtronApps/preStageHandlerCode.go b/pkg/deployment/trigger/devtronApps/preStageHandlerCode.go index ad0051e76d..4f30f19ecc 100644 --- a/pkg/deployment/trigger/devtronApps/preStageHandlerCode.go +++ b/pkg/deployment/trigger/devtronApps/preStageHandlerCode.go @@ -23,7 +23,7 @@ import ( "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" blob_storage "github.com/devtron-labs/common-lib/blob-storage" commonBean "github.com/devtron-labs/common-lib/workflow" - bean2 "github.com/devtron-labs/devtron/api/bean" + apiBean "github.com/devtron-labs/devtron/api/bean" gitSensorClient "github.com/devtron-labs/devtron/client/gitSensor" constants2 "github.com/devtron-labs/devtron/internal/sql/constants" "github.com/devtron-labs/devtron/internal/sql/repository" @@ -62,7 +62,7 @@ import ( ) func (impl *HandlerServiceImpl) TriggerPreStage(request bean.TriggerRequest) (*bean6.ManifestPushTemplate, error) { - request.WorkflowType = bean2.CD_WORKFLOW_TYPE_PRE + request.WorkflowType = apiBean.CD_WORKFLOW_TYPE_PRE // setting triggeredAt variable to have consistent data for various audit log places in db for deployment time triggeredAt := time.Now() triggeredBy := request.TriggeredBy @@ -219,7 +219,7 @@ func (impl *HandlerServiceImpl) TriggerAutoCDOnPreStageSuccess(triggerContext be func (impl *HandlerServiceImpl) checkDeploymentTriggeredAlready(wfId int) bool { deploymentTriggeredAlready := false // TODO : need to check this logic for status check in case of multiple deployments requirement for same workflow - workflowRunner, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(context.Background(), wfId, bean2.CD_WORKFLOW_TYPE_DEPLOY) + workflowRunner, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(context.Background(), wfId, apiBean.CD_WORKFLOW_TYPE_DEPLOY) if err != nil { impl.logger.Errorw("error occurred while fetching workflow runner", "wfId", wfId, "err", err) return deploymentTriggeredAlready @@ -236,7 +236,7 @@ func (impl *HandlerServiceImpl) createStartingWfAndRunner(request bean.TriggerRe //in case of pre stage manual trigger auth is already applied and for auto triggers there is no need for auth check here cdWf := request.CdWf var err error - if cdWf == nil && request.WorkflowType == bean2.CD_WORKFLOW_TYPE_PRE { + if cdWf == nil && request.WorkflowType == apiBean.CD_WORKFLOW_TYPE_PRE { cdWf = &pipelineConfig.CdWorkflow{ CiArtifactId: artifact.Id, PipelineId: pipeline.Id, @@ -279,9 +279,9 @@ func (impl *HandlerServiceImpl) getEnvAndNsIfRunStageInEnv(ctx context.Context, var err error namespace := impl.config.GetDefaultNamespace() runStageInEnv := false - if workflowStage == bean2.CD_WORKFLOW_TYPE_PRE { + if workflowStage == apiBean.CD_WORKFLOW_TYPE_PRE { runStageInEnv = pipeline.RunPreStageInEnv - } else if workflowStage == bean2.CD_WORKFLOW_TYPE_POST { + } else if workflowStage == apiBean.CD_WORKFLOW_TYPE_POST { runStageInEnv = pipeline.RunPostStageInEnv } _, span := otel.Tracer("orchestrator").Start(ctx, "envRepository.FindById") @@ -588,7 +588,7 @@ func (impl *HandlerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow } if pipelineStage != nil { var variableSnapshot map[string]string - if runner.WorkflowType == bean2.CD_WORKFLOW_TYPE_PRE { + if runner.WorkflowType == apiBean.CD_WORKFLOW_TYPE_PRE { // TODO: use const from pipeline.WorkflowService:95 request := pipelineConfigBean.NewBuildPrePostStepDataReq(cdPipeline.Id, "preCD", scope) prePostAndRefPluginResponse, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(request) @@ -599,7 +599,7 @@ func (impl *HandlerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow preDeploySteps = prePostAndRefPluginResponse.PreStageSteps refPluginsData = prePostAndRefPluginResponse.RefPluginData variableSnapshot = prePostAndRefPluginResponse.VariableSnapshot - } else if runner.WorkflowType == bean2.CD_WORKFLOW_TYPE_POST { + } else if runner.WorkflowType == apiBean.CD_WORKFLOW_TYPE_POST { // TODO: use const from pipeline.WorkflowService:96 request := pipelineConfigBean.NewBuildPrePostStepDataReq(cdPipeline.Id, "postCD", scope) prePostAndRefPluginResponse, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(request) @@ -630,9 +630,9 @@ func (impl *HandlerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow } } else { //in this case no plugin script is not present for this cdPipeline hence going with attaching preStage or postStage config - if runner.WorkflowType == bean2.CD_WORKFLOW_TYPE_PRE { + if runner.WorkflowType == apiBean.CD_WORKFLOW_TYPE_PRE { stageYaml = cdPipeline.PreStageConfig - } else if runner.WorkflowType == bean2.CD_WORKFLOW_TYPE_POST { + } else if runner.WorkflowType == apiBean.CD_WORKFLOW_TYPE_POST { stageYaml = cdPipeline.PostStageConfig deployStageWfr, deployStageTriggeredByUserEmail, pipelineReleaseCounter, err = impl.getDeployStageDetails(cdPipeline.Id) if err != nil { @@ -813,11 +813,11 @@ func (impl *HandlerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow // For Pre-CD / Post-CD workflow, cache is not uploaded; hence no need to set cache bucket cdWorkflowConfigCdCacheBucket := "" - if runner.WorkflowType == bean2.CD_WORKFLOW_TYPE_PRE { + if runner.WorkflowType == apiBean.CD_WORKFLOW_TYPE_PRE { // populate input variables of steps with extra env variables setExtraEnvVariableInDeployStep(preDeploySteps, runtimeParams.GetSystemVariables(), webhookAndCiData) cdStageWorkflowRequest.PrePostDeploySteps = preDeploySteps - } else if runner.WorkflowType == bean2.CD_WORKFLOW_TYPE_POST { + } else if runner.WorkflowType == apiBean.CD_WORKFLOW_TYPE_POST { // populate input variables of steps with extra env variables setExtraEnvVariableInDeployStep(postDeploySteps, runtimeParams.GetSystemVariables(), webhookAndCiData) cdStageWorkflowRequest.PrePostDeploySteps = postDeploySteps @@ -1007,7 +1007,7 @@ func setExtraEnvVariableInDeployStep(deploySteps []*pipelineConfigBean.StepObjec func (impl *HandlerServiceImpl) getDeployStageDetails(pipelineId int) (pipelineConfig.CdWorkflowRunner, string, int, error) { deployStageWfr := pipelineConfig.CdWorkflowRunner{} //getting deployment pipeline latest wfr by pipelineId - deployStageWfr, err := impl.cdWorkflowRepository.FindLatestByPipelineIdAndRunnerType(pipelineId, bean2.CD_WORKFLOW_TYPE_DEPLOY) + deployStageWfr, err := impl.cdWorkflowRepository.FindLatestByPipelineIdAndRunnerType(pipelineId, apiBean.CD_WORKFLOW_TYPE_DEPLOY) if err != nil { impl.logger.Errorw("error in getting latest status of deploy type wfr by pipelineId", "err", err, "pipelineId", pipelineId) return deployStageWfr, "", 0, err @@ -1046,14 +1046,14 @@ func ReplaceImageTagWithDigest(image, digest string) string { } func (impl *HandlerServiceImpl) sendPreStageNotification(ctx context.Context, cdWf *pipelineConfig.CdWorkflow, pipeline *pipelineConfig.Pipeline) error { - wfr, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, cdWf.Id, bean2.CD_WORKFLOW_TYPE_PRE) + wfr, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, cdWf.Id, apiBean.CD_WORKFLOW_TYPE_PRE) if err != nil { return err } event, _ := impl.eventFactory.Build(util2.Trigger, &pipeline.Id, pipeline.AppId, &pipeline.EnvironmentId, util2.CD) impl.logger.Debugw("event PreStageTrigger", "event", event) - event = impl.eventFactory.BuildExtraCDData(event, &wfr, 0, bean2.CD_WORKFLOW_TYPE_PRE) + event = impl.eventFactory.BuildExtraCDData(event, &wfr, 0, apiBean.CD_WORKFLOW_TYPE_PRE) _, span := otel.Tracer("orchestrator").Start(ctx, "eventClient.WriteNotificationEvent") _, evtErr := impl.eventClient.WriteNotificationEvent(event) span.End() diff --git a/pkg/executor/WorkflowService.go b/pkg/executor/WorkflowService.go index 14ff02b5b8..2b32743bce 100644 --- a/pkg/executor/WorkflowService.go +++ b/pkg/executor/WorkflowService.go @@ -20,7 +20,6 @@ import ( "context" "encoding/json" "errors" - "fmt" v1alpha12 "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned/typed/workflow/v1alpha1" "github.com/argoproj/argo-workflows/v3/workflow/util" "github.com/devtron-labs/common-lib/utils" @@ -324,23 +323,16 @@ func (impl *WorkflowServiceImpl) prepareCmCsForWorkflowTemplate(workflowRequest return nil, nil, err } allowAll := workflowRequest.IsDevtronJob() || workflowRequest.IsDevtronCI() - namePrefix := workflowRequest.GetExistingCmCsNamePrefix() for _, cm := range workflowConfigMaps { // HERE we are allowing all existingSecrets in case of JOB/ BUILD Infra if _, ok := pipelineLevelConfigMaps[cm.Name]; ok || allowAll { - if !cm.External { - cm.Name = fmt.Sprintf("%s-cm-%s", cm.Name, namePrefix) - } - modifiedWorkflowConfigMaps = append(modifiedWorkflowConfigMaps, cm) + modifiedWorkflowConfigMaps = append(modifiedWorkflowConfigMaps, workflowRequest.ModifyConfigSecretMap(cm, bean.ConfigMap)) } } for _, secret := range workflowSecrets { // HERE we are allowing all existingSecrets in case of JOB/ BUILD Infra if _, ok := pipelineLevelSecrets[secret.Name]; ok || allowAll { - if !secret.External { - secret.Name = fmt.Sprintf("%s-cs-%s", secret.Name, namePrefix) - } - modifiedWorkflowSecrets = append(modifiedWorkflowSecrets, secret) + modifiedWorkflowSecrets = append(modifiedWorkflowSecrets, workflowRequest.ModifyConfigSecretMap(secret, bean.Secret)) } } return modifiedWorkflowConfigMaps, modifiedWorkflowSecrets, nil diff --git a/pkg/pipeline/ArgoWorkflowExecutor_test.go b/pkg/pipeline/ArgoWorkflowExecutor_test.go index afde725241..41e84402db 100644 --- a/pkg/pipeline/ArgoWorkflowExecutor_test.go +++ b/pkg/pipeline/ArgoWorkflowExecutor_test.go @@ -75,8 +75,8 @@ func TestExecuteWorkflow(t *testing.T) { secretKeySecret := s3Artifact.SecretKeySecret assert.NotNil(t, accessKeySecret) assert.NotNil(t, secretKeySecret) - assert.True(t, reflect.DeepEqual(accessKeySecret, executors.ACCESS_KEY_SELECTOR)) - assert.True(t, reflect.DeepEqual(secretKeySecret, executors.SECRET_KEY_SELECTOR)) + assert.True(t, reflect.DeepEqual(accessKeySecret, executors.AccessKeySelector)) + assert.True(t, reflect.DeepEqual(secretKeySecret, executors.SecretKeySelector)) }) t.Run("validate s3 blob storage with endpoint", func(t *testing.T) { @@ -105,7 +105,7 @@ func TestExecuteWorkflow(t *testing.T) { assert.Equal(t, gcpBlobStorage.LogBucketName, gcsArtifact.Bucket) secretKeySecret := gcsArtifact.ServiceAccountKeySecret assert.NotNil(t, secretKeySecret) - assert.True(t, reflect.DeepEqual(secretKeySecret, executors.SECRET_KEY_SELECTOR)) + assert.True(t, reflect.DeepEqual(secretKeySecret, executors.SecretKeySelector)) }) t.Run("validate env specific cm and secret", func(t *testing.T) { workflowTemplate := getBaseWorkflowTemplate(cdConfig) diff --git a/pkg/pipeline/PipelineStageService.go b/pkg/pipeline/PipelineStageService.go index 2a538b7282..58f8d69ebe 100644 --- a/pkg/pipeline/PipelineStageService.go +++ b/pkg/pipeline/PipelineStageService.go @@ -1985,14 +1985,14 @@ func (impl *PipelineStageServiceImpl) buildPipelineStepDataForWfRequest(step *re func (impl *PipelineStageServiceImpl) buildVariableAndConditionDataForWfRequest(stepId int) (*bean.VariableAndConditionDataForStep, error) { variableAndConditionData := bean.NewVariableAndConditionDataForStep() - //getting all variables in the step - variables, err := impl.pipelineStageRepository.GetVariablesByStepId(stepId) + // getting all variables in the step + stepVariables, err := impl.pipelineStageRepository.GetVariablesByStepId(stepId) if err != nil && !util.IsErrNoRows(err) { impl.logger.Errorw("error in getting variables by stepId", "err", err, "stepId", stepId) return variableAndConditionData, err } variableNameIdMap := make(map[int]string) - for _, variable := range variables { + for _, variable := range stepVariables { variableNameIdMap[variable.Id] = variable.Name // getting format // ignoring error as it is already validated in func validatePipelineStageStepVariableForTrigger diff --git a/pkg/pipeline/bean/pipelineStage.go b/pkg/pipeline/bean/pipelineStage.go index f18e699284..8cb2b699a0 100644 --- a/pkg/pipeline/bean/pipelineStage.go +++ b/pkg/pipeline/bean/pipelineStage.go @@ -20,7 +20,7 @@ import ( commonBean "github.com/devtron-labs/common-lib/workflow" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/pipeline/repository" - repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository" + pluginRepo "github.com/devtron-labs/devtron/pkg/plugin/repository" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" ) @@ -46,23 +46,23 @@ type PipelineStageStepDto struct { } type InlineStepDetailDto struct { - ScriptType repository2.ScriptType `json:"scriptType" validate:"omitempty,oneof=SHELL DOCKERFILE CONTAINER_IMAGE"` - Script string `json:"script"` - StoreScriptAt string `json:"storeScriptAt"` - DockerfileExists bool `json:"dockerfileExists,omitempty"` - MountPath string `json:"mountPath,omitempty"` - MountCodeToContainer bool `json:"mountCodeToContainer,omitempty"` - MountCodeToContainerPath string `json:"mountCodeToContainerPath,omitempty"` - MountDirectoryFromHost bool `json:"mountDirectoryFromHost"` - ContainerImagePath string `json:"containerImagePath,omitempty"` - ImagePullSecretType repository2.ScriptImagePullSecretType `json:"imagePullSecretType,omitempty" validate:"omitempty,oneof=CONTAINER_REGISTRY SECRET_PATH"` - ImagePullSecret string `json:"imagePullSecret,omitempty"` - MountPathMap []*MountPathMap `json:"mountPathMap,omitempty" validate:"omitempty,dive"` - CommandArgsMap []*CommandArgsMap `json:"commandArgsMap,omitempty" validate:"omitempty,dive"` - PortMap []*PortMap `json:"portMap,omitempty" validate:"omitempty,dive"` - InputVariables []*StepVariableDto `json:"inputVariables" validate:"dive"` - OutputVariables []*StepVariableDto `json:"outputVariables" validate:"dive"` - ConditionDetails []*ConditionDetailDto `json:"conditionDetails" validate:"dive"` + ScriptType pluginRepo.ScriptType `json:"scriptType" validate:"omitempty,oneof=SHELL DOCKERFILE CONTAINER_IMAGE"` + Script string `json:"script"` + StoreScriptAt string `json:"storeScriptAt"` + DockerfileExists bool `json:"dockerfileExists,omitempty"` + MountPath string `json:"mountPath,omitempty"` + MountCodeToContainer bool `json:"mountCodeToContainer,omitempty"` + MountCodeToContainerPath string `json:"mountCodeToContainerPath,omitempty"` + MountDirectoryFromHost bool `json:"mountDirectoryFromHost"` + ContainerImagePath string `json:"containerImagePath,omitempty"` + ImagePullSecretType pluginRepo.ScriptImagePullSecretType `json:"imagePullSecretType,omitempty" validate:"omitempty,oneof=CONTAINER_REGISTRY SECRET_PATH"` + ImagePullSecret string `json:"imagePullSecret,omitempty"` + MountPathMap []*MountPathMap `json:"mountPathMap,omitempty" validate:"omitempty,dive"` + CommandArgsMap []*CommandArgsMap `json:"commandArgsMap,omitempty" validate:"omitempty,dive"` + PortMap []*PortMap `json:"portMap,omitempty" validate:"omitempty,dive"` + InputVariables []*StepVariableDto `json:"inputVariables" validate:"dive"` + OutputVariables []*StepVariableDto `json:"outputVariables" validate:"dive"` + ConditionDetails []*ConditionDetailDto `json:"conditionDetails" validate:"dive"` } type RefPluginStepDetailDto struct { @@ -92,9 +92,6 @@ type StepVariableDto struct { StepVariableEntDto } -type StepVariableEntDto struct { -} - func (s *StepVariableDto) GetValue() string { if s == nil { return "" @@ -181,6 +178,7 @@ type VariableAndConditionDataForStep struct { outputVariables []*commonBean.VariableObject triggerSkipConditions []*ConditionObject successFailureConditions []*ConditionObject + VariableAndConditionDataForStepEnt } func NewVariableAndConditionDataForStep() *VariableAndConditionDataForStep { diff --git a/pkg/pipeline/bean/pipelineStage_ent.go b/pkg/pipeline/bean/pipelineStage_ent.go new file mode 100644 index 0000000000..3b2d16dfd6 --- /dev/null +++ b/pkg/pipeline/bean/pipelineStage_ent.go @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package bean + +import apiBean "github.com/devtron-labs/devtron/api/bean" + +type StepVariableEntDto struct { +} + +type VariableAndConditionDataForStepEnt struct { +} + +func (v *VariableAndConditionDataForStep) GetFileReferenceConfigMap() []*apiBean.ConfigSecretMap { + return nil +} + +func (v *VariableAndConditionDataForStep) AddFileReferenceConfigMap(configMap *apiBean.ConfigSecretMap) *VariableAndConditionDataForStep { + return v +} diff --git a/pkg/pipeline/bean/workFlowRequestBean.go b/pkg/pipeline/bean/workFlowRequestBean.go index b9d29cf28f..16e42db83a 100644 --- a/pkg/pipeline/bean/workFlowRequestBean.go +++ b/pkg/pipeline/bean/workFlowRequestBean.go @@ -42,6 +42,7 @@ type PrePostAndRefPluginStepsResponse struct { PostStageSteps []*StepObject RefPluginData []*RefPluginObject VariableSnapshot map[string]string + PrePostAndRefPluginStepsResponseEnt } type StepObject struct { diff --git a/pkg/pipeline/bean/workFlowRequestBean_ent.go b/pkg/pipeline/bean/workFlowRequestBean_ent.go new file mode 100644 index 0000000000..65fe7d7f9f --- /dev/null +++ b/pkg/pipeline/bean/workFlowRequestBean_ent.go @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package bean + +type PrePostAndRefPluginStepsResponseEnt struct { +} diff --git a/pkg/pipeline/executors/adapter/adapter.go b/pkg/pipeline/executors/adapter/adapter.go new file mode 100644 index 0000000000..cd9ac10bcb --- /dev/null +++ b/pkg/pipeline/executors/adapter/adapter.go @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2020-2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package adapter + +import ( + "encoding/json" + "github.com/devtron-labs/common-lib/utils/k8s/commonBean" + "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/types" + k8sApiV1 "k8s.io/api/core/v1" + k8sMetaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func GetConfigMapJson(configMapSecretDto types.ConfigMapSecretDto) (string, error) { + configMapBody := GetConfigMapBody(configMapSecretDto) + configMapJson, err := json.Marshal(configMapBody) + if err != nil { + return "", err + } + return string(configMapJson), err +} + +func GetSecretJson(configMapSecretDto types.ConfigMapSecretDto) (string, error) { + secretBody, err := GetSecretBody(configMapSecretDto) + if err != nil { + return "", err + } + secretJson, err := json.Marshal(secretBody) + if err != nil { + return "", err + } + return string(secretJson), err +} + +func GetConfigMapSecretDto(configSecretMap bean.ConfigSecretMap, ownerRef k8sMetaV1.OwnerReference, isSecret bool) (types.ConfigMapSecretDto, error) { + configDataMap, err := configSecretMap.GetDataMap() + if err != nil { + return types.ConfigMapSecretDto{}, err + } + configMapSecretDto := types.ConfigMapSecretDto{ + Name: configSecretMap.Name, + Data: configDataMap, + OwnerRef: ownerRef, + } + return updateBinaryDataInConfigMapSecretDto(configSecretMap, configMapSecretDto, isSecret), nil +} + +func GetConfigMapBody(configMapSecretDto types.ConfigMapSecretDto) k8sApiV1.ConfigMap { + configMap := k8sApiV1.ConfigMap{ + TypeMeta: k8sMetaV1.TypeMeta{ + Kind: commonBean.ConfigMapKind, + APIVersion: commonBean.V1VERSION, + }, + ObjectMeta: k8sMetaV1.ObjectMeta{ + Name: configMapSecretDto.Name, + OwnerReferences: []k8sMetaV1.OwnerReference{configMapSecretDto.OwnerRef}, + }, + Data: configMapSecretDto.Data, + } + return getConfigMapBodyEnt(configMapSecretDto, configMap) +} + +func GetSecretBody(configMapSecretDto types.ConfigMapSecretDto) (k8sApiV1.Secret, error) { + secretDataMap := make(map[string][]byte) + // adding handling to get base64 decoded value in map value + cmsDataMarshaled, err := json.Marshal(configMapSecretDto.Data) + if err != nil { + return k8sApiV1.Secret{}, err + } + err = json.Unmarshal(cmsDataMarshaled, &secretDataMap) + if err != nil { + return k8sApiV1.Secret{}, err + } + secret := k8sApiV1.Secret{ + TypeMeta: k8sMetaV1.TypeMeta{ + Kind: commonBean.SecretKind, + APIVersion: commonBean.V1VERSION, + }, + ObjectMeta: k8sMetaV1.ObjectMeta{ + Name: configMapSecretDto.Name, + OwnerReferences: []k8sMetaV1.OwnerReference{configMapSecretDto.OwnerRef}, + }, + Data: secretDataMap, + Type: k8sApiV1.SecretTypeOpaque, + } + return secret, nil +} diff --git a/pkg/pipeline/executors/adapter/adapter_ent.go b/pkg/pipeline/executors/adapter/adapter_ent.go new file mode 100644 index 0000000000..d75792ba88 --- /dev/null +++ b/pkg/pipeline/executors/adapter/adapter_ent.go @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package adapter + +import ( + "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/types" + k8sApiV1 "k8s.io/api/core/v1" +) + +func updateBinaryDataInConfigMapSecretDto(configSecretMap bean.ConfigSecretMap, configMapSecretDto types.ConfigMapSecretDto, isSecret bool) types.ConfigMapSecretDto { + return configMapSecretDto +} + +func getConfigMapBodyEnt(configMapSecretDto types.ConfigMapSecretDto, configMap k8sApiV1.ConfigMap) k8sApiV1.ConfigMap { + return configMap +} diff --git a/pkg/pipeline/executors/ArgoWorkflowExecutor.go b/pkg/pipeline/executors/argoWorkflow.go similarity index 73% rename from pkg/pipeline/executors/ArgoWorkflowExecutor.go rename to pkg/pipeline/executors/argoWorkflow.go index e54c04ac3b..b5a8ca772d 100644 --- a/pkg/pipeline/executors/ArgoWorkflowExecutor.go +++ b/pkg/pipeline/executors/argoWorkflow.go @@ -21,16 +21,17 @@ import ( "encoding/json" "errors" "fmt" - "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + argoWfApiV1 "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned" - v1alpha12 "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned/typed/workflow/v1alpha1" + argoWfClientV1 "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned/typed/workflow/v1alpha1" "github.com/argoproj/argo-workflows/v3/workflow/util" - bean2 "github.com/devtron-labs/devtron/api/bean" + apiBean "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/executors/adapter" "github.com/devtron-labs/devtron/pkg/pipeline/types" "go.uber.org/zap" - v12 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sApiV1 "k8s.io/api/core/v1" + k8sMetaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" @@ -38,22 +39,6 @@ import ( "net/url" ) -const ( - STEP_NAME_REGEX = "create-env-%s-gb-%d" - TEMPLATE_NAME_REGEX = "%s-gb-%d" - WORKFLOW_MINIO_CRED = "workflow-minio-cred" - CRED_ACCESS_KEY = "accessKey" - CRED_SECRET_KEY = "secretKey" - S3_ENDPOINT_URL = "s3.amazonaws.com" - DEVTRON_WORKFLOW_LABEL_KEY = "devtron.ai/workflow-purpose" - DEVTRON_WORKFLOW_LABEL_VALUE = "cd" - WORKFLOW_GENERATE_NAME_REGEX = "%s-" - RESOURCE_CREATE_ACTION = "create" -) - -var ACCESS_KEY_SELECTOR = &v12.SecretKeySelector{Key: CRED_ACCESS_KEY, LocalObjectReference: v12.LocalObjectReference{Name: WORKFLOW_MINIO_CRED}} -var SECRET_KEY_SELECTOR = &v12.SecretKeySelector{Key: CRED_SECRET_KEY, LocalObjectReference: v12.LocalObjectReference{Name: WORKFLOW_MINIO_CRED}} - type WorkflowExecutor interface { ExecuteWorkflow(workflowTemplate bean.WorkflowTemplate) (*unstructured.UnstructuredList, error) TerminateWorkflow(workflowName string, namespace string, clusterConfig *rest.Config) error @@ -81,7 +66,7 @@ func (impl *ArgoWorkflowExecutorImpl) TerminateWorkflow(workflowName string, nam impl.logger.Errorw("cannot build wf client", "wfName", workflowName, "err", err) return err } - _, err = wfClient.Get(context.Background(), workflowName, v1.GetOptions{}) + _, err = wfClient.Get(context.Background(), workflowName, k8sMetaV1.GetOptions{}) if err != nil { impl.logger.Errorw("cannot find workflow", "name", workflowName, "err", err) return errors.New("cannot find workflow " + workflowName) @@ -98,7 +83,7 @@ func (impl *ArgoWorkflowExecutorImpl) TerminateDanglingWorkflow(workflowGenerate return err } jobSelectorLabel := fmt.Sprintf("%s=%s", bean.WorkflowGenerateNamePrefix, workflowGenerateName) - wfList, err := wfClient.List(context.Background(), v1.ListOptions{LabelSelector: jobSelectorLabel}) + wfList, err := wfClient.List(context.Background(), k8sMetaV1.ListOptions{LabelSelector: jobSelectorLabel}) if err != nil { impl.logger.Errorw("error in fetching list of workflows", "namespace", namespace, "err", err) return err @@ -127,7 +112,7 @@ func (impl *ArgoWorkflowExecutorImpl) ExecuteWorkflow(workflowTemplate bean.Work } wfContainer := workflowTemplate.Containers[0] - ciCdTemplate := v1alpha1.Template{ + ciCdTemplate := argoWfApiV1.Template{ Name: workflowTemplate.WorkflowType, Container: &wfContainer, ActiveDeadlineSeconds: &intstr.IntOrString{ @@ -140,20 +125,20 @@ func (impl *ArgoWorkflowExecutorImpl) ExecuteWorkflow(workflowTemplate bean.Work objectMeta := workflowTemplate.CreateObjectMetadata() var ( - ciCdWorkflow = v1alpha1.Workflow{ + ciCdWorkflow = argoWfApiV1.Workflow{ ObjectMeta: *objectMeta, - Spec: v1alpha1.WorkflowSpec{ + Spec: argoWfApiV1.WorkflowSpec{ ServiceAccountName: workflowTemplate.ServiceAccountName, NodeSelector: workflowTemplate.NodeSelector, Tolerations: workflowTemplate.Tolerations, Entrypoint: entryPoint, - TTLStrategy: &v1alpha1.TTLStrategy{ + TTLStrategy: &argoWfApiV1.TTLStrategy{ SecondsAfterCompletion: workflowTemplate.TTLValue, }, Templates: templates, Volumes: workflowTemplate.Volumes, - PodGC: &v1alpha1.PodGC{ - Strategy: v1alpha1.PodGCOnWorkflowCompletion, + PodGC: &argoWfApiV1.PodGC{ + Strategy: argoWfApiV1.PodGCOnWorkflowCompletion, }, }, } @@ -172,7 +157,7 @@ func (impl *ArgoWorkflowExecutorImpl) ExecuteWorkflow(workflowTemplate bean.Work return nil, err } - createdWf, err := wfClient.Create(context.Background(), &ciCdWorkflow, v1.CreateOptions{}) + createdWf, err := wfClient.Create(context.Background(), &ciCdWorkflow, k8sMetaV1.CreateOptions{}) if err != nil { impl.logger.Errorw("error in wf trigger", "err", err) return nil, err @@ -202,13 +187,13 @@ func (impl *ArgoWorkflowExecutorImpl) GetWorkflowStatus(workflowName string, nam return wfStatus, err } -func (impl *ArgoWorkflowExecutorImpl) getWorkflow(workflowName string, namespace string, clusterConfig *rest.Config) (*v1alpha1.Workflow, error) { +func (impl *ArgoWorkflowExecutorImpl) getWorkflow(workflowName string, namespace string, clusterConfig *rest.Config) (*argoWfApiV1.Workflow, error) { wfClient, err := impl.getClientInstance(namespace, clusterConfig) if err != nil { impl.logger.Errorw("cannot build wf client", "wfName", workflowName, "err", err) return nil, err } - wf, err := wfClient.Get(context.Background(), workflowName, v1.GetOptions{}) + wf, err := wfClient.Get(context.Background(), workflowName, k8sMetaV1.GetOptions{}) if err != nil { impl.logger.Errorw("cannot find workflow", "name", workflowName, "err", err) return nil, fmt.Errorf("cannot find workflow %s", workflowName) @@ -226,13 +211,13 @@ func (impl *ArgoWorkflowExecutorImpl) convertToUnstructured(cdWorkflow interface return unstructuredList } -func (impl *ArgoWorkflowExecutorImpl) updateBlobStorageConfig(workflowTemplate bean.WorkflowTemplate, cdTemplate *v1alpha1.Template) { - cdTemplate.ArchiveLocation = &v1alpha1.ArtifactLocation{ +func (impl *ArgoWorkflowExecutorImpl) updateBlobStorageConfig(workflowTemplate bean.WorkflowTemplate, cdTemplate *argoWfApiV1.Template) { + cdTemplate.ArchiveLocation = &argoWfApiV1.ArtifactLocation{ ArchiveLogs: &workflowTemplate.ArchiveLogs, } if workflowTemplate.BlobStorageConfigured { - var s3Artifact *v1alpha1.S3Artifact - var gcsArtifact *v1alpha1.GCSArtifact + var s3Artifact *argoWfApiV1.S3Artifact + var gcsArtifact *argoWfApiV1.GCSArtifact blobStorageS3Config := workflowTemplate.BlobStorageS3Config gcpBlobConfig := workflowTemplate.GcpBlobConfig cloudStorageKey := workflowTemplate.CloudStorageKey @@ -249,25 +234,15 @@ func (impl *ArgoWorkflowExecutorImpl) updateBlobStorageConfig(workflowTemplate b } } isInsecure := blobStorageS3Config.IsInSecure - var accessKeySelector *v12.SecretKeySelector - var secretKeySelector *v12.SecretKeySelector + var accessKeySelector *k8sApiV1.SecretKeySelector + var secretKeySelector *k8sApiV1.SecretKeySelector if blobStorageS3Config.AccessKey != "" { - accessKeySelector = &v12.SecretKeySelector{ - Key: CRED_ACCESS_KEY, - LocalObjectReference: v12.LocalObjectReference{ - Name: WORKFLOW_MINIO_CRED, - }, - } - secretKeySelector = &v12.SecretKeySelector{ - Key: CRED_SECRET_KEY, - LocalObjectReference: v12.LocalObjectReference{ - Name: WORKFLOW_MINIO_CRED, - }, - } + accessKeySelector = AccessKeySelector + secretKeySelector = SecretKeySelector } - s3Artifact = &v1alpha1.S3Artifact{ + s3Artifact = &argoWfApiV1.S3Artifact{ Key: cloudStorageKey, - S3Bucket: v1alpha1.S3Bucket{ + S3Bucket: argoWfApiV1.S3Bucket{ Endpoint: s3CompatibleEndpointUrl, AccessKeySecret: accessKeySelector, SecretKeySecret: secretKeySelector, @@ -280,16 +255,11 @@ func (impl *ArgoWorkflowExecutorImpl) updateBlobStorageConfig(workflowTemplate b s3Artifact.Region = blobStorageS3Config.CiLogRegion } } else if gcpBlobConfig != nil { - gcsArtifact = &v1alpha1.GCSArtifact{ + gcsArtifact = &argoWfApiV1.GCSArtifact{ Key: cloudStorageKey, - GCSBucket: v1alpha1.GCSBucket{ - Bucket: gcpBlobConfig.LogBucketName, - ServiceAccountKeySecret: &v12.SecretKeySelector{ - Key: CRED_SECRET_KEY, - LocalObjectReference: v12.LocalObjectReference{ - Name: WORKFLOW_MINIO_CRED, - }, - }, + GCSBucket: argoWfApiV1.GCSBucket{ + Bucket: gcpBlobConfig.LogBucketName, + ServiceAccountKeySecret: SecretKeySelector, }, } } @@ -300,9 +270,9 @@ func (impl *ArgoWorkflowExecutorImpl) updateBlobStorageConfig(workflowTemplate b } } -func (impl *ArgoWorkflowExecutorImpl) getArgoTemplates(configMaps []bean2.ConfigSecretMap, secrets []bean2.ConfigSecretMap, isCi bool) ([]v1alpha1.Template, error) { - var templates []v1alpha1.Template - var steps []v1alpha1.ParallelSteps +func (impl *ArgoWorkflowExecutorImpl) getArgoTemplates(configMaps []apiBean.ConfigSecretMap, secrets []apiBean.ConfigSecretMap, isCi bool) ([]argoWfApiV1.Template, error) { + var templates []argoWfApiV1.Template + var steps []argoWfApiV1.ParallelSteps cmIndex := 0 csIndex := 0 for _, configMap := range configMaps { @@ -339,8 +309,8 @@ func (impl *ArgoWorkflowExecutorImpl) getArgoTemplates(configMaps []bean2.Config templateName = bean.CI_WORKFLOW_WITH_STAGES } - steps = append(steps, v1alpha1.ParallelSteps{ - Steps: []v1alpha1.WorkflowStep{ + steps = append(steps, argoWfApiV1.ParallelSteps{ + Steps: []argoWfApiV1.WorkflowStep{ { Name: "run-wf", Template: stepName, @@ -348,7 +318,7 @@ func (impl *ArgoWorkflowExecutorImpl) getArgoTemplates(configMaps []bean2.Config }, }) - templates = append(templates, v1alpha1.Template{ + templates = append(templates, argoWfApiV1.Template{ Name: templateName, Steps: steps, }) @@ -356,22 +326,30 @@ func (impl *ArgoWorkflowExecutorImpl) getArgoTemplates(configMaps []bean2.Config return templates, nil } -func (impl *ArgoWorkflowExecutorImpl) appendCMCSToStepAndTemplate(isSecret bool, configSecretMap bean2.ConfigSecretMap, cmSecretIndex int) (v1alpha1.ParallelSteps, v1alpha1.Template, error) { - var parallelStep v1alpha1.ParallelSteps - var argoTemplate v1alpha1.Template - configDataMap, err := configSecretMap.GetDataMap() - if err != nil { - impl.logger.Errorw("error occurred while extracting data map", "Data", configSecretMap.Data, "err", err) - return parallelStep, argoTemplate, err - } - - var cmSecretJson string - configMapSecretDto := types.ConfigMapSecretDto{Name: configSecretMap.Name, Data: configDataMap, OwnerRef: ArgoWorkflowOwnerRef} +func (impl *ArgoWorkflowExecutorImpl) getConfigMapOrSecretJson(configMapSecretDto types.ConfigMapSecretDto, isSecret bool) (cmSecretJson string, err error) { if isSecret { - cmSecretJson, err = GetSecretJson(configMapSecretDto) + cmSecretJson, err = adapter.GetSecretJson(configMapSecretDto) + if err != nil { + impl.logger.Errorw("error occurred while extracting cm/secret json", "secretName", configMapSecretDto.Name, "err", err) + return cmSecretJson, err + } } else { - cmSecretJson, err = GetConfigMapJson(configMapSecretDto) + cmSecretJson, err = adapter.GetConfigMapJson(configMapSecretDto) + if err != nil { + impl.logger.Errorw("error occurred while extracting cm/secret json", "configMapName", configMapSecretDto.Name, "err", err) + return cmSecretJson, err + } + } + return cmSecretJson, nil +} + +func (impl *ArgoWorkflowExecutorImpl) appendCMCSToStepAndTemplate(isSecret bool, configSecretMap apiBean.ConfigSecretMap, cmSecretIndex int) (parallelStep argoWfApiV1.ParallelSteps, argoTemplate argoWfApiV1.Template, err error) { + configMapSecretDto, err := adapter.GetConfigMapSecretDto(configSecretMap, ArgoWorkflowOwnerRef, isSecret) + if err != nil { + impl.logger.Errorw("error occurred while extracting config map secret dto", "configSecretName", configSecretMap.Name, "err", err) + return parallelStep, argoTemplate, err } + cmSecretJson, err := impl.getConfigMapOrSecretJson(configMapSecretDto, isSecret) if err != nil { impl.logger.Errorw("error occurred while extracting cm/secret json", "configSecretName", configSecretMap.Name, "err", err) return parallelStep, argoTemplate, err @@ -380,24 +358,24 @@ func (impl *ArgoWorkflowExecutorImpl) appendCMCSToStepAndTemplate(isSecret bool, return parallelStep, argoTemplate, nil } -func (impl *ArgoWorkflowExecutorImpl) createStepAndTemplate(isSecret bool, cmSecretIndex int, cmSecretJson string) (v1alpha1.ParallelSteps, v1alpha1.Template) { +func (impl *ArgoWorkflowExecutorImpl) createStepAndTemplate(isSecret bool, cmSecretIndex int, cmSecretJson string) (argoWfApiV1.ParallelSteps, argoWfApiV1.Template) { stepName := fmt.Sprintf(STEP_NAME_REGEX, "cm", cmSecretIndex) templateName := fmt.Sprintf(TEMPLATE_NAME_REGEX, "cm", cmSecretIndex) if isSecret { stepName = fmt.Sprintf(STEP_NAME_REGEX, "secret", cmSecretIndex) templateName = fmt.Sprintf(TEMPLATE_NAME_REGEX, "secret", cmSecretIndex) } - parallelStep := v1alpha1.ParallelSteps{ - Steps: []v1alpha1.WorkflowStep{ + parallelStep := argoWfApiV1.ParallelSteps{ + Steps: []argoWfApiV1.WorkflowStep{ { Name: stepName, Template: templateName, }, }, } - argoTemplate := v1alpha1.Template{ + argoTemplate := argoWfApiV1.Template{ Name: templateName, - Resource: &v1alpha1.ResourceTemplate{ + Resource: &argoWfApiV1.ResourceTemplate{ Action: RESOURCE_CREATE_ACTION, SetOwnerReference: true, Manifest: string(cmSecretJson), @@ -406,7 +384,7 @@ func (impl *ArgoWorkflowExecutorImpl) createStepAndTemplate(isSecret bool, cmSec return parallelStep, argoTemplate } -func (impl *ArgoWorkflowExecutorImpl) getClientInstance(namespace string, clusterConfig *rest.Config) (v1alpha12.WorkflowInterface, error) { +func (impl *ArgoWorkflowExecutorImpl) getClientInstance(namespace string, clusterConfig *rest.Config) (argoWfClientV1.WorkflowInterface, error) { clientSet, err := versioned.NewForConfig(clusterConfig) if err != nil { impl.logger.Errorw("error occurred while creating client from config", "err", err) diff --git a/pkg/pipeline/executors/bean.go b/pkg/pipeline/executors/bean.go new file mode 100644 index 0000000000..b46626898f --- /dev/null +++ b/pkg/pipeline/executors/bean.go @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package executors + +import ( + k8sApiV1 "k8s.io/api/core/v1" + k8sMetaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + STEP_NAME_REGEX = "create-env-%s-gb-%d" + TEMPLATE_NAME_REGEX = "%s-gb-%d" + WORKFLOW_MINIO_CRED = "workflow-minio-cred" + CRED_ACCESS_KEY = "accessKey" + CRED_SECRET_KEY = "secretKey" + S3_ENDPOINT_URL = "s3.amazonaws.com" + DEVTRON_WORKFLOW_LABEL_KEY = "devtron.ai/workflow-purpose" + DEVTRON_WORKFLOW_LABEL_VALUE = "cd" + WORKFLOW_GENERATE_NAME_REGEX = "%s-" + RESOURCE_CREATE_ACTION = "create" +) + +var ArgoWorkflowOwnerRef = k8sMetaV1.OwnerReference{APIVersion: "argoproj.io/v1alpha1", Kind: "Workflow", Name: "{{workflow.name}}", UID: "{{workflow.uid}}", BlockOwnerDeletion: &[]bool{true}[0]} + +var AccessKeySelector = &k8sApiV1.SecretKeySelector{Key: CRED_ACCESS_KEY, LocalObjectReference: k8sApiV1.LocalObjectReference{Name: WORKFLOW_MINIO_CRED}} + +var SecretKeySelector = &k8sApiV1.SecretKeySelector{Key: CRED_SECRET_KEY, LocalObjectReference: k8sApiV1.LocalObjectReference{Name: WORKFLOW_MINIO_CRED}} + +const ( + WorkflowJobBackoffLimit = 0 + WorkflowJobFinalizer = "foregroundDeletion" +) diff --git a/pkg/pipeline/executors/SystemWorkflowExecutor.go b/pkg/pipeline/executors/systemWorkflow.go similarity index 91% rename from pkg/pipeline/executors/SystemWorkflowExecutor.go rename to pkg/pipeline/executors/systemWorkflow.go index 6d58738467..acfbb578ad 100644 --- a/pkg/pipeline/executors/SystemWorkflowExecutor.go +++ b/pkg/pipeline/executors/systemWorkflow.go @@ -22,6 +22,7 @@ import ( "github.com/devtron-labs/common-lib/utils/k8s" k8sCommonBean "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/executors/adapter" types2 "github.com/devtron-labs/devtron/pkg/pipeline/types" "go.uber.org/zap" v1 "k8s.io/api/batch/v1" @@ -35,11 +36,6 @@ import ( "k8s.io/utils/pointer" ) -const ( - WORKFLOW_JOB_BACKOFF_LIMIT = 0 - WORKFLOW_JOB_FINALIZER = "foregroundDeletion" -) - type SystemWorkflowExecutor interface { WorkflowExecutor } @@ -185,7 +181,7 @@ func (impl *SystemWorkflowExecutorImpl) GetWorkflowStatus(workflowName string, n } func (impl *SystemWorkflowExecutorImpl) getJobTemplate(workflowTemplate bean.WorkflowTemplate) *v1.Job { - workflowLabels := GetWorkflowLabelsForSystemExecutor(workflowTemplate) + workflowLabels := getWorkflowLabelsForSystemExecutor(workflowTemplate) //setting TerminationGracePeriodSeconds in PodSpec //which ensures Pod has enough time to execute cleanup on SIGTERM event @@ -199,10 +195,10 @@ func (impl *SystemWorkflowExecutorImpl) getJobTemplate(workflowTemplate bean.Wor GenerateName: fmt.Sprintf(WORKFLOW_GENERATE_NAME_REGEX, workflowTemplate.WorkflowNamePrefix), //Annotations: map[string]string{"workflows.argoproj.io/controller-instanceid": workflowTemplate.WfControllerInstanceID}, Labels: workflowLabels, - Finalizers: []string{WORKFLOW_JOB_FINALIZER}, + Finalizers: []string{WorkflowJobFinalizer}, }, Spec: v1.JobSpec{ - BackoffLimit: pointer.Int32Ptr(WORKFLOW_JOB_BACKOFF_LIMIT), + BackoffLimit: pointer.Int32Ptr(WorkflowJobBackoffLimit), ActiveDeadlineSeconds: workflowTemplate.ActiveDeadlineSeconds, TTLSecondsAfterFinished: workflowTemplate.TTLValue, Template: corev1.PodTemplateSpec{ @@ -225,13 +221,12 @@ func (impl *SystemWorkflowExecutorImpl) getCmAndSecrets(workflowTemplate bean.Wo if configSecretMap.External { continue } - configDataMap, err := configSecretMap.GetDataMap() + configMapSecretDto, err := adapter.GetConfigMapSecretDto(configSecretMap, impl.createJobOwnerRefVal(createdJob), false) if err != nil { - impl.logger.Errorw("error occurred while extracting data map", "Data", configSecretMap.Data, "err", err) + impl.logger.Errorw("error occurred while creating config map dto", "err", err) return configMaps, secrets, err } - configMapSecretDto := types2.ConfigMapSecretDto{Name: configSecretMap.Name, Data: configDataMap, OwnerRef: impl.createJobOwnerRefVal(createdJob)} - configMap := GetConfigMapBody(configMapSecretDto) + configMap := adapter.GetConfigMapBody(configMapSecretDto) configMaps = append(configMaps, configMap) } secretMaps := workflowTemplate.Secrets @@ -239,13 +234,16 @@ func (impl *SystemWorkflowExecutorImpl) getCmAndSecrets(workflowTemplate bean.Wo if secretMapData.External { continue } - dataMap, err := secretMapData.GetDataMap() + configMapSecretDto, err := adapter.GetConfigMapSecretDto(secretMapData, impl.createJobOwnerRefVal(createdJob), true) + if err != nil { + impl.logger.Errorw("error occurred while creating config map dto", "err", err) + return configMaps, secrets, err + } + secretBody, err := adapter.GetSecretBody(configMapSecretDto) if err != nil { - impl.logger.Errorw("error occurred while extracting data map", "Data", secretMapData.Data, "err", err) + impl.logger.Errorw("error occurred while creating secret body", "err", err) return configMaps, secrets, err } - configMapSecretDto := types2.ConfigMapSecretDto{Name: secretMapData.Name, Data: dataMap, OwnerRef: impl.createJobOwnerRefVal(createdJob)} - secretBody := GetSecretBody(configMapSecretDto) secrets = append(secrets, secretBody) } return configMaps, secrets, nil diff --git a/pkg/pipeline/executors/WorkflowUtils.go b/pkg/pipeline/executors/utils.go similarity index 55% rename from pkg/pipeline/executors/WorkflowUtils.go rename to pkg/pipeline/executors/utils.go index bbf7439660..ba27356d0c 100644 --- a/pkg/pipeline/executors/WorkflowUtils.go +++ b/pkg/pipeline/executors/utils.go @@ -18,28 +18,26 @@ package executors import ( "encoding/json" - "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + argoWfApiV1 "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned" - v1alpha12 "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned/typed/workflow/v1alpha1" + argoWfClientV1 "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned/typed/workflow/v1alpha1" informerBean "github.com/devtron-labs/common-lib/informer" "github.com/devtron-labs/common-lib/utils" - bean2 "github.com/devtron-labs/devtron/api/bean" + apiBean "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/executors/adapter" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/util" - v12 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sApiV1 "k8s.io/api/core/v1" "k8s.io/client-go/rest" "strconv" ) -var ArgoWorkflowOwnerRef = v1.OwnerReference{APIVersion: "argoproj.io/v1alpha1", Kind: "Workflow", Name: "{{workflow.name}}", UID: "{{workflow.uid}}", BlockOwnerDeletion: &[]bool{true}[0]} - -func ExtractVolumes(configMaps []bean2.ConfigSecretMap, secrets []bean2.ConfigSecretMap) []v12.Volume { - var volumes []v12.Volume - configMapVolumes := ExtractVolumesFromCmCs(configMaps, secrets) +func ExtractVolumes(configMaps []apiBean.ConfigSecretMap, secrets []apiBean.ConfigSecretMap) []k8sApiV1.Volume { + var volumes []k8sApiV1.Volume + configMapVolumes := extractVolumesFromCmCs(configMaps, secrets) volumes = append(volumes, configMapVolumes...) // Add downwardAPI volume @@ -48,8 +46,8 @@ func ExtractVolumes(configMaps []bean2.ConfigSecretMap, secrets []bean2.ConfigSe return volumes } -func ExtractVolumesFromCmCs(configMaps []bean2.ConfigSecretMap, secrets []bean2.ConfigSecretMap) []v12.Volume { - var volumes []v12.Volume +func extractVolumesFromCmCs(configMaps []apiBean.ConfigSecretMap, secrets []apiBean.ConfigSecretMap) []k8sApiV1.Volume { + var volumes []k8sApiV1.Volume configMapVolumes := extractVolumesFromConfigSecretMaps(true, configMaps) secretVolumes := extractVolumesFromConfigSecretMaps(false, secrets) @@ -63,21 +61,21 @@ func ExtractVolumesFromCmCs(configMaps []bean2.ConfigSecretMap, secrets []bean2. return volumes } -func createDownwardAPIVolume() v12.Volume { - return v12.Volume{ +func createDownwardAPIVolume() k8sApiV1.Volume { + return k8sApiV1.Volume{ Name: utils.DEVTRON_SELF_DOWNWARD_API_VOLUME, - VolumeSource: v12.VolumeSource{ - DownwardAPI: &v12.DownwardAPIVolumeSource{ - Items: []v12.DownwardAPIVolumeFile{ + VolumeSource: k8sApiV1.VolumeSource{ + DownwardAPI: &k8sApiV1.DownwardAPIVolumeSource{ + Items: []k8sApiV1.DownwardAPIVolumeFile{ { Path: utils.POD_LABELS, - FieldRef: &v12.ObjectFieldSelector{ + FieldRef: &k8sApiV1.ObjectFieldSelector{ FieldPath: "metadata." + utils.POD_LABELS, }, }, { Path: utils.POD_ANNOTATIONS, - FieldRef: &v12.ObjectFieldSelector{ + FieldRef: &k8sApiV1.ObjectFieldSelector{ FieldPath: "metadata." + utils.POD_ANNOTATIONS, }, }, @@ -87,30 +85,30 @@ func createDownwardAPIVolume() v12.Volume { } } -func extractVolumesFromConfigSecretMaps(isCm bool, configSecretMaps []bean2.ConfigSecretMap) []v12.Volume { - var volumes []v12.Volume +func extractVolumesFromConfigSecretMaps(isCm bool, configSecretMaps []apiBean.ConfigSecretMap) []k8sApiV1.Volume { + var volumes []k8sApiV1.Volume for _, configSecretMap := range configSecretMaps { if configSecretMap.Type != util.ConfigMapSecretUsageTypeVolume { // not volume type so ignoring continue } - var volumeSource v12.VolumeSource + var volumeSource k8sApiV1.VolumeSource if isCm { - volumeSource = v12.VolumeSource{ - ConfigMap: &v12.ConfigMapVolumeSource{ - LocalObjectReference: v12.LocalObjectReference{ + volumeSource = k8sApiV1.VolumeSource{ + ConfigMap: &k8sApiV1.ConfigMapVolumeSource{ + LocalObjectReference: k8sApiV1.LocalObjectReference{ Name: configSecretMap.Name, }, }, } } else { - volumeSource = v12.VolumeSource{ - Secret: &v12.SecretVolumeSource{ + volumeSource = k8sApiV1.VolumeSource{ + Secret: &k8sApiV1.SecretVolumeSource{ SecretName: configSecretMap.Name, }, } } - volumes = append(volumes, v12.Volume{ + volumes = append(volumes, k8sApiV1.Volume{ Name: configSecretMap.Name + "-vol", VolumeSource: volumeSource, }) @@ -118,62 +116,9 @@ func extractVolumesFromConfigSecretMaps(isCm bool, configSecretMaps []bean2.Conf return volumes } -func GetConfigMapJson(configMapSecretDto types.ConfigMapSecretDto) (string, error) { - configMapBody := GetConfigMapBody(configMapSecretDto) - configMapJson, err := json.Marshal(configMapBody) - if err != nil { - return "", err - } - return string(configMapJson), err -} - -func GetConfigMapBody(configMapSecretDto types.ConfigMapSecretDto) v12.ConfigMap { - return v12.ConfigMap{ - TypeMeta: v1.TypeMeta{ - Kind: "ConfigMap", - APIVersion: "v1", - }, - ObjectMeta: v1.ObjectMeta{ - Name: configMapSecretDto.Name, - OwnerReferences: []v1.OwnerReference{configMapSecretDto.OwnerRef}, - }, - Data: configMapSecretDto.Data, - } -} - -func GetSecretJson(configMapSecretDto types.ConfigMapSecretDto) (string, error) { - secretBody := GetSecretBody(configMapSecretDto) - secretJson, err := json.Marshal(secretBody) - if err != nil { - return "", err - } - return string(secretJson), err -} - -func GetSecretBody(configMapSecretDto types.ConfigMapSecretDto) v12.Secret { - secretDataMap := make(map[string][]byte) - - // adding handling to get base64 decoded value in map value - cmsDataMarshaled, _ := json.Marshal(configMapSecretDto.Data) - json.Unmarshal(cmsDataMarshaled, &secretDataMap) - - return v12.Secret{ - TypeMeta: v1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: v1.ObjectMeta{ - Name: configMapSecretDto.Name, - OwnerReferences: []v1.OwnerReference{configMapSecretDto.OwnerRef}, - }, - Data: secretDataMap, - Type: "Opaque", - } -} - -func GetFromGlobalCmCsDtos(globalCmCsConfigs []*bean.GlobalCMCSDto) ([]bean2.ConfigSecretMap, []bean2.ConfigSecretMap, error) { - workflowConfigMaps := make([]bean2.ConfigSecretMap, 0, len(globalCmCsConfigs)) - workflowSecrets := make([]bean2.ConfigSecretMap, 0, len(globalCmCsConfigs)) +func GetFromGlobalCmCsDtos(globalCmCsConfigs []*bean.GlobalCMCSDto) ([]apiBean.ConfigSecretMap, []apiBean.ConfigSecretMap, error) { + workflowConfigMaps := make([]apiBean.ConfigSecretMap, 0, len(globalCmCsConfigs)) + workflowSecrets := make([]apiBean.ConfigSecretMap, 0, len(globalCmCsConfigs)) for _, config := range globalCmCsConfigs { configSecretMap, err := config.ConvertToConfigSecretMap() @@ -189,39 +134,39 @@ func GetFromGlobalCmCsDtos(globalCmCsConfigs []*bean.GlobalCMCSDto) ([]bean2.Con return workflowConfigMaps, workflowSecrets, nil } -func AddTemplatesForGlobalSecretsInWorkflowTemplate(globalCmCsConfigs []*bean.GlobalCMCSDto, steps *[]v1alpha1.ParallelSteps, volumes *[]v12.Volume, templates *[]v1alpha1.Template) error { +func AddTemplatesForGlobalSecretsInWorkflowTemplate(globalCmCsConfigs []*bean.GlobalCMCSDto, steps *[]argoWfApiV1.ParallelSteps, volumes *[]k8sApiV1.Volume, templates *[]argoWfApiV1.Template) error { cmIndex := 0 csIndex := 0 for _, config := range globalCmCsConfigs { if config.ConfigType == repository.CM_TYPE_CONFIG { - cmJson, err := GetConfigMapJson(types.ConfigMapSecretDto{Name: config.Name, Data: config.Data, OwnerRef: ArgoWorkflowOwnerRef}) + cmJson, err := adapter.GetConfigMapJson(types.ConfigMapSecretDto{Name: config.Name, Data: config.Data, OwnerRef: ArgoWorkflowOwnerRef}) if err != nil { return err } if config.Type == repository.VOLUME_CONFIG { - *volumes = append(*volumes, v12.Volume{ + *volumes = append(*volumes, k8sApiV1.Volume{ Name: config.Name + "-vol", - VolumeSource: v12.VolumeSource{ - ConfigMap: &v12.ConfigMapVolumeSource{ - LocalObjectReference: v12.LocalObjectReference{ + VolumeSource: k8sApiV1.VolumeSource{ + ConfigMap: &k8sApiV1.ConfigMapVolumeSource{ + LocalObjectReference: k8sApiV1.LocalObjectReference{ Name: config.Name, }, }, }, }) } - *steps = append(*steps, v1alpha1.ParallelSteps{ - Steps: []v1alpha1.WorkflowStep{ + *steps = append(*steps, argoWfApiV1.ParallelSteps{ + Steps: []argoWfApiV1.WorkflowStep{ { Name: "create-env-cm-gb-" + strconv.Itoa(cmIndex), Template: "cm-gb-" + strconv.Itoa(cmIndex), }, }, }) - *templates = append(*templates, v1alpha1.Template{ + *templates = append(*templates, argoWfApiV1.Template{ Name: "cm-gb-" + strconv.Itoa(cmIndex), - Resource: &v1alpha1.ResourceTemplate{ + Resource: &argoWfApiV1.ResourceTemplate{ Action: "create", SetOwnerReference: true, Manifest: string(cmJson), @@ -241,31 +186,31 @@ func AddTemplatesForGlobalSecretsInWorkflowTemplate(globalCmCsConfigs []*bean.Gl return err } - secretJson, err := GetSecretJson(types.ConfigMapSecretDto{Name: config.Name, Data: encodedSecretDataMap, OwnerRef: ArgoWorkflowOwnerRef}) + secretJson, err := adapter.GetSecretJson(types.ConfigMapSecretDto{Name: config.Name, Data: encodedSecretDataMap, OwnerRef: ArgoWorkflowOwnerRef}) if err != nil { return err } if config.Type == repository.VOLUME_CONFIG { - *volumes = append(*volumes, v12.Volume{ + *volumes = append(*volumes, k8sApiV1.Volume{ Name: config.Name + "-vol", - VolumeSource: v12.VolumeSource{ - Secret: &v12.SecretVolumeSource{ + VolumeSource: k8sApiV1.VolumeSource{ + Secret: &k8sApiV1.SecretVolumeSource{ SecretName: config.Name, }, }, }) } - *steps = append(*steps, v1alpha1.ParallelSteps{ - Steps: []v1alpha1.WorkflowStep{ + *steps = append(*steps, argoWfApiV1.ParallelSteps{ + Steps: []argoWfApiV1.WorkflowStep{ { Name: "create-env-sec-gb-" + strconv.Itoa(csIndex), Template: "sec-gb-" + strconv.Itoa(csIndex), }, }, }) - *templates = append(*templates, v1alpha1.Template{ + *templates = append(*templates, argoWfApiV1.Template{ Name: "sec-gb-" + strconv.Itoa(csIndex), - Resource: &v1alpha1.ResourceTemplate{ + Resource: &argoWfApiV1.ResourceTemplate{ Action: "create", SetOwnerReference: true, Manifest: string(secretJson), @@ -278,7 +223,7 @@ func AddTemplatesForGlobalSecretsInWorkflowTemplate(globalCmCsConfigs []*bean.Gl return nil } -func GetClientInstance(config *rest.Config, namespace string) (v1alpha12.WorkflowInterface, error) { +func GetClientInstance(config *rest.Config, namespace string) (argoWfClientV1.WorkflowInterface, error) { clientSet, err := versioned.NewForConfig(config) if err != nil { return nil, err @@ -288,7 +233,7 @@ func GetClientInstance(config *rest.Config, namespace string) (v1alpha12.Workflo } func CheckIfReTriggerRequired(status, message, workflowRunnerStatus string) bool { - return (status == string(v1alpha1.NodeError) || status == string(v1alpha1.NodeFailed)) && + return (status == string(argoWfApiV1.NodeError) || status == string(argoWfApiV1.NodeFailed)) && message == cdWorkflow.POD_DELETED_MESSAGE && workflowRunnerStatus != cdWorkflow.WorkflowCancel && workflowRunnerStatus != cdWorkflow.WorkflowAborted && @@ -296,7 +241,7 @@ func CheckIfReTriggerRequired(status, message, workflowRunnerStatus string) bool } -func GetWorkflowLabelsForSystemExecutor(workflowTemplate bean.WorkflowTemplate) map[string]string { +func getWorkflowLabelsForSystemExecutor(workflowTemplate bean.WorkflowTemplate) map[string]string { return map[string]string{ DEVTRON_WORKFLOW_LABEL_KEY: DEVTRON_WORKFLOW_LABEL_VALUE, bean.DevtronLabelPurposeKey: bean.DevtronLabelPurposeWorkflow, diff --git a/pkg/pipeline/types/Workflow.go b/pkg/pipeline/types/Workflow.go index dceb9a03b0..3a4461929a 100644 --- a/pkg/pipeline/types/Workflow.go +++ b/pkg/pipeline/types/Workflow.go @@ -22,9 +22,9 @@ import ( "github.com/argoproj/argo-workflows/v3/workflow/common" "github.com/devtron-labs/common-lib/blob-storage" "github.com/devtron-labs/common-lib/utils" - bean7 "github.com/devtron-labs/common-lib/utils/bean" + commonBean "github.com/devtron-labs/common-lib/utils/bean" "github.com/devtron-labs/common-lib/utils/workFlow" - bean3 "github.com/devtron-labs/devtron/api/bean" + apiBean "github.com/devtron-labs/devtron/api/bean" repository2 "github.com/devtron-labs/devtron/internal/sql/repository" repository3 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" @@ -153,6 +153,7 @@ type WorkflowRequest struct { AsyncBuildxCacheExport bool `json:"asyncBuildxCacheExport"` UseDockerApiToGetDigest bool `json:"useDockerApiToGetDigest"` HostUrl string `json:"hostUrl"` + WorkflowRequestEnt } func (workflowRequest *WorkflowRequest) updateExternalRunMetadata() { @@ -495,7 +496,7 @@ func (workflowRequest *WorkflowRequest) getWorkflowImage() string { } } -func (workflowRequest *WorkflowRequest) GetWorkflowMainContainer(config *CiCdConfig, infraConfigurations *infraBean.InfraConfig, workflowJson []byte, workflowTemplate *bean.WorkflowTemplate, workflowConfigMaps []bean3.ConfigSecretMap, workflowSecrets []bean3.ConfigSecretMap) (v1.Container, error) { +func (workflowRequest *WorkflowRequest) GetWorkflowMainContainer(config *CiCdConfig, infraConfigurations *infraBean.InfraConfig, workflowJson []byte, workflowTemplate *bean.WorkflowTemplate, workflowConfigMaps []apiBean.ConfigSecretMap, workflowSecrets []apiBean.ConfigSecretMap) (v1.Container, error) { privileged := true pvc := workflowRequest.getPVCForWorkflowRequest() containerEnvVariables := workflowRequest.getContainerEnvVariables(config, workflowJson) @@ -570,7 +571,15 @@ func (workflowRequest *WorkflowRequest) updateVolumeMountsForCi(config *CiCdConf return nil } -func UpdateContainerEnvsFromCmCs(workflowMainContainer *v1.Container, configMaps []bean3.ConfigSecretMap, secrets []bean3.ConfigSecretMap) { +func (workflowRequest *WorkflowRequest) ModifyConfigSecretMap(workflowConfigSecretMap apiBean.ConfigSecretMap, configType apiBean.ConfigType) apiBean.ConfigSecretMap { + namePrefix := workflowRequest.GetExistingCmCsNamePrefix() + if !workflowConfigSecretMap.External { + workflowConfigSecretMap.Name = fmt.Sprintf("%s-%s-%s", workflowConfigSecretMap.Name, configType.String(), namePrefix) + } + return workflowConfigSecretMap +} + +func UpdateContainerEnvsFromCmCs(workflowMainContainer *v1.Container, configMaps []apiBean.ConfigSecretMap, secrets []apiBean.ConfigSecretMap) { for _, configMap := range configMaps { updateContainerEnvs(true, workflowMainContainer, configMap) } @@ -580,7 +589,7 @@ func UpdateContainerEnvsFromCmCs(workflowMainContainer *v1.Container, configMaps } } -func updateContainerEnvs(isCM bool, workflowMainContainer *v1.Container, configSecretMap bean3.ConfigSecretMap) { +func updateContainerEnvs(isCM bool, workflowMainContainer *v1.Container, configSecretMap apiBean.ConfigSecretMap) { if configSecretMap.Type == repository2.VOLUME_CONFIG { workflowMainContainer.VolumeMounts = append(workflowMainContainer.VolumeMounts, v1.VolumeMount{ Name: configSecretMap.Name + "-vol", @@ -676,10 +685,10 @@ type WorkflowResponse struct { ImageReleaseTags []*repository3.ImageTag `json:"imageReleaseTags"` ImageComment *repository3.ImageComment `json:"imageComment"` AppWorkflowId int `json:"appWorkflowId"` - CustomTag *bean3.CustomTagErrorResponse `json:"customTag,omitempty"` + CustomTag *apiBean.CustomTagErrorResponse `json:"customTag,omitempty"` PipelineType string `json:"pipelineType"` ReferenceWorkflowId int `json:"referenceWorkflowId"` - TargetPlatforms []*bean7.TargetPlatform `json:"targetPlatforms"` + TargetPlatforms []*commonBean.TargetPlatform `json:"targetPlatforms"` WorkflowExecutionStage map[string][]*bean6.WorkflowStageDto `json:"workflowExecutionStages"` } @@ -687,6 +696,7 @@ type ConfigMapSecretDto struct { Name string Data map[string]string OwnerRef v12.OwnerReference + ConfigMapSecretEntDto } type WorkflowStatus struct { diff --git a/pkg/pipeline/types/Workflow_ent.go b/pkg/pipeline/types/Workflow_ent.go index ecf78b9e1a..5a3fcfec4c 100644 --- a/pkg/pipeline/types/Workflow_ent.go +++ b/pkg/pipeline/types/Workflow_ent.go @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package types import ( @@ -5,6 +21,12 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/bean" ) +type WorkflowRequestEnt struct { +} + +type ConfigMapSecretEntDto struct { +} + type ImageScanningSteps struct { Steps []*bean.StepObject `json:"steps"` ScanToolId int `json:"scanToolId"` diff --git a/util/sliceUtil/SliceUtil.go b/util/sliceUtil/SliceUtil.go index 8c816d4164..4f4ef79bac 100644 --- a/util/sliceUtil/SliceUtil.go +++ b/util/sliceUtil/SliceUtil.go @@ -88,6 +88,9 @@ func Difference[T comparable](a, b []T) []T { func GetDeReferencedSlice[T any](ptrObjects []*T) []T { deReferencedArray := make([]T, 0) for _, item := range ptrObjects { + if item == nil { + continue + } deReferencedArray = append(deReferencedArray, *item) } return deReferencedArray diff --git a/vendor/github.com/devtron-labs/common-lib/utils/k8s/commonBean/bean.go b/vendor/github.com/devtron-labs/common-lib/utils/k8s/commonBean/bean.go index 621b39edfb..77dbb59d1b 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/k8s/commonBean/bean.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/k8s/commonBean/bean.go @@ -24,6 +24,7 @@ import ( ) const ( + ConfigMapKind = "ConfigMap" SecretKind = "Secret" ServiceKind = "Service" ServiceAccountKind = "ServiceAccount" @@ -300,8 +301,8 @@ type PodMetadata struct { EphemeralContainers []*EphemeralContainerData `json:"ephemeralContainers"` } -// use value field as generic type // InfoItem contains arbitrary, human readable information about an application +// use value field as generic type type InfoItem struct { // Name is a human readable title for this piece of information. Name string `json:"name,omitempty"` diff --git a/vendor/github.com/devtron-labs/common-lib/workflow/bean.go b/vendor/github.com/devtron-labs/common-lib/workflow/bean.go index 3147762955..7f31dcfd45 100644 --- a/vendor/github.com/devtron-labs/common-lib/workflow/bean.go +++ b/vendor/github.com/devtron-labs/common-lib/workflow/bean.go @@ -1,7 +1,6 @@ package workflow import ( - "encoding/base64" "encoding/json" "fmt" "os" @@ -168,8 +167,8 @@ type VariableObject struct { ReferenceVariableName string `json:"referenceVariableName"` ReferenceVariableStepIndex int `json:"referenceVariableStepIndex"` VariableStepIndexInPlugin int `json:"variableStepIndexInPlugin"` - FileContent string `json:"fileContent"` // FileContent is the base64 encoded content of the file - TypedValue interface{} `json:"-"` // TypedValue is the formatted value of the variable after type conversion + FileReferencePath string `json:"fileReferencePath"` // FileContent is the base64 encoded content of the file + TypedValue interface{} `json:"-"` // TypedValue is the formatted value of the variable after type conversion } // TypeCheck converts the VariableObject.Value to the VariableObject.Format type @@ -188,19 +187,25 @@ func (v *VariableObject) TypeCheck() error { return nil } -// SetFileContent decodes the base64 encoded file content and writes it to the file at filePath +// SetFileContent sets the content of the file referenced by the VariableObject.FileReferencePath field. func (v *VariableObject) SetFileContent(filePath string) error { if v.Format != FormatTypeFile { return nil } + if len(v.FileReferencePath) == 0 { + // for plugins we may receive the filePath as "", + // even thought VariableObject.Value is there; + // for this case don't mount the file. + return nil + } file, fileErr := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm) if fileErr != nil { return fileErr } defer file.Close() - fileBytes, fileErr := base64.StdEncoding.DecodeString(v.FileContent) - if fileErr != nil { - return fileErr + fileBytes, err := os.ReadFile(v.FileReferencePath) + if err != nil { + return err } _, wErr := file.Write(fileBytes) if wErr != nil { diff --git a/vendor/modules.txt b/vendor/modules.txt index 725caf74e0..7d44ed8bee 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -472,7 +472,7 @@ github.com/davecgh/go-spew/spew # github.com/deckarep/golang-set v1.8.0 ## explicit; go 1.17 github.com/deckarep/golang-set -# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250428081808-439aa2bff7da +# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250504164106-4a6b1415f5d9 ## explicit; go 1.21 github.com/devtron-labs/authenticator/apiToken github.com/devtron-labs/authenticator/client @@ -480,7 +480,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250428081808-439aa2bff7da +# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250504164106-4a6b1415f5d9 ## explicit; go 1.21 github.com/devtron-labs/common-lib/async github.com/devtron-labs/common-lib/blob-storage @@ -2352,8 +2352,8 @@ xorm.io/xorm/log xorm.io/xorm/names xorm.io/xorm/schemas xorm.io/xorm/tags -# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250428081808-439aa2bff7da -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250428081808-439aa2bff7da +# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250504164106-4a6b1415f5d9 +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250504164106-4a6b1415f5d9 # github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127 # github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5 # k8s.io/api => k8s.io/api v0.29.7