Merge branch 'main' into feat-notifier-behind-nats

Author: komalreddy3

config.md

@@ -97,6 +97,7 @@
 | DEVTRON_HELM_RELEASE_NAME                | devtron                                             | Name of the Devtron Helm release.                   |
 | ENABLE_LEGACY_API                        | "false"                                             | Enable the legacy API.                              |
 | INSTALLATION_THROUGH_HELM                | "True"                                              | Installation through Helm (True or False).          |
+| USE_IMAGE_TAG_FROM_GIT_PROVIDER_FOR_TAG_BASED_BUILD | "True"                                    | Tag Image same as Git provider for Tag based Build (True or False) | 
 
 
 # DEVTRON SECRET PARAMETER

internal/sql/repository/security/CveStoreRepository.go

@@ -30,7 +30,7 @@ type CveStore struct {
 	tableName    struct{} `sql:"cve_store" pg:",discard_unknown_columns"`
 	Name         string   `sql:"name,pk"`
 	Severity     Severity `sql:"severity,notnull"`
-	Package      string   `sql:"package,notnull"`
+	Package      string   `sql:"package,notnull"` // deprecated
 	Version      string   `sql:"version,notnull"`
 	FixedVersion string   `sql:"fixed_version,notnull"`
 	sql.AuditLog

internal/sql/repository/security/ImageScanResultRepository.go

@@ -27,6 +27,7 @@ type ImageScanExecutionResult struct {
 	CveStoreName                string   `sql:"cve_store_name,notnull"`
 	ImageScanExecutionHistoryId int      `sql:"image_scan_execution_history_id"`
 	ScanToolId                  int      `sql:"scan_tool_id"`
+	Package                     string   `sql:"package"`
 	CveStore                    CveStore
 	ImageScanExecutionHistory   ImageScanExecutionHistory
 }

pkg/app/AppCrudOperationService.go

@@ -613,11 +613,25 @@ func (impl AppCrudOperationServiceImpl) getExtraAppLabelsToPropagate(appId int,
 		impl.logger.Errorw("error in finding app and project by appId", "appId", appId, "err", err)
 		return nil, err
 	}
-	return map[string]string{
-		bean3.AppNameDevtronLabel:     appName,
-		bean3.EnvNameDevtronLabel:     envName,
-		bean3.ProjectNameDevtronLabel: appMetaInfo.Team.Name,
-	}, nil
+	regexp := regexp.MustCompile(LabelMatchingRegex)
+	extraAppLabels := make(map[string]string)
+
+	extraAppLabels[bean3.AppNameDevtronLabel] = appName
+	extraAppLabels[bean3.EnvNameDevtronLabel] = envName
+	extraAppLabels[bean3.ProjectNameDevtronLabel] = appMetaInfo.Team.Name
+
+	extraAppLabels = sanitizeLabels(extraAppLabels)
+	for labelKey, labelValue := range extraAppLabels {
+		if regexp.MatchString(labelValue) {
+			extraAppLabels[labelKey] = labelValue
+		} else {
+			// in case extra labels are failing k8s official label matching regex  even after sanitization then
+			//delete the label as this can break deployments.
+			impl.logger.Warnw("extra label failed LabelMatchingRegex validation, regex:- ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$", "labelKey", labelKey, "labelValue", labelValue)
+			delete(extraAppLabels, labelKey)
+		}
+	}
+	return extraAppLabels, nil
 }
 
 func (impl AppCrudOperationServiceImpl) GetAppLabelsForDeployment(appId int, appName, envName string) ([]byte, error) {

pkg/app/helper.go

@@ -16,6 +16,11 @@
 
 package app
 
+import "strings"
+
+// LabelMatchingRegex is the official k8s label matching regex, pls refer https://github.com/kubernetes/apimachinery/blob/bfd2aff97e594f6aad77acbe2cbbe190acc93cbc/pkg/util/validation/validation.go#L167
+const LabelMatchingRegex = "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$"
+
 // MergeChildMapToParentMap merges child map of generic type map into parent map of generic type
 // and returns merged mapping, if parentMap is nil then nil is returned.
 func MergeChildMapToParentMap[T comparable, R any](parentMap map[T]R, toMergeMap map[T]R) map[T]R {
@@ -29,3 +34,12 @@ func MergeChildMapToParentMap[T comparable, R any](parentMap map[T]R, toMergeMap
 	}
 	return parentMap
 }
+
+func sanitizeLabels(extraAppLabels map[string]string) map[string]string {
+	for lkey, lvalue := range extraAppLabels {
+		if strings.Contains(lvalue, " ") {
+			extraAppLabels[lkey] = strings.ReplaceAll(lvalue, " ", "_")
+		}
+	}
+	return extraAppLabels
+}

pkg/deployment/gitOps/git/GitOpsHelper.go

@@ -96,13 +96,19 @@ func (impl *GitOpsHelper) Pull(repoRoot string) (err error) {
 	return impl.gitCommandManager.Pull(ctx, repoRoot)
 }
 
+const PushErrorMessage = "failed to push some refs"
+
 func (impl GitOpsHelper) CommitAndPushAllChanges(repoRoot, commitMsg, name, emailId string) (commitHash string, err error) {
 	start := time.Now()
 	defer func() {
 		util.TriggerGitOpsMetrics("CommitAndPushAllChanges", "GitService", start, err)
 	}()
 	ctx := git.BuildGitContext(context.Background()).WithCredentials(impl.Auth)
-	return impl.gitCommandManager.CommitAndPush(ctx, repoRoot, commitMsg, name, emailId)
+	commitHash, err = impl.gitCommandManager.CommitAndPush(ctx, repoRoot, commitMsg, name, emailId)
+	if err != nil && strings.Contains(err.Error(), PushErrorMessage) {
+		return commitHash, fmt.Errorf("%s %v", "push failed due to conflicts", err)
+	}
+	return commitHash, nil
 }
 
 func (impl *GitOpsHelper) pullFromBranch(ctx git.GitContext, rootDir string) (string, string, error) {

pkg/deployment/gitOps/git/commandManager/GitCliManager.go

@@ -132,7 +132,7 @@ func (impl *GitCliManagerImpl) add(ctx GitContext, rootDir string) (response, er
 
 func (impl *GitCliManagerImpl) push(ctx GitContext, rootDir string) (response, errMsg string, err error) {
 	impl.logger.Debugw("git push ", "location", rootDir)
-	cmd, cancel := impl.createCmdWithContext(ctx, "git", "-C", rootDir, "push", "origin", "master", "--force")
+	cmd, cancel := impl.createCmdWithContext(ctx, "git", "-C", rootDir, "push", "origin", "master")
 	defer cancel()
 	output, errMsg, err := impl.runCommandWithCred(cmd, ctx.auth)
 	impl.logger.Debugw("git add output", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err)

pkg/deployment/gitOps/git/commandManager/GitCommandBaseManager.go

@@ -92,7 +92,7 @@ func (impl *GitManagerBaseImpl) runCommand(cmd *exec.Cmd) (response, errMsg stri
 	if err != nil {
 		exErr, ok := err.(*exec.ExitError)
 		if !ok {
-			return "", "", err
+			return "", "", fmt.Errorf("%s %v", outBytes, err)
 		}
 		errOutput := string(exErr.Stderr)
 		return "", errOutput, err

pkg/eventProcessor/in/WorkflowEventProcessorService.go

@@ -152,6 +152,16 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDStageCompleteEvent() error {
 			impl.logger.Errorw("could not get wf runner", "err", err)
 			return
 		}
+
+		if wfr.Status != string(v1alpha1.NodeSucceeded) {
+			impl.logger.Debugw("event received from ci runner, updating workflow runner status as succeeded", "savedWorkflowRunnerId", wfr.Id, "oldStatus", wfr.Status, "podStatus", wfr.PodStatus)
+			err = impl.cdWorkflowRunnerService.UpdateWfrStatus(wfr, string(v1alpha1.NodeSucceeded), 1)
+			if err != nil {
+				impl.logger.Errorw("update cd-wf-runner failed for id ", "cdWfrId", wfr.Id, "err", err)
+				return
+			}
+		}
+
 		triggerContext := bean5.TriggerContext{
 			ReferenceId: pointer.String(msg.MsgId),
 		}

pkg/security/ImageScanService.go

@@ -372,6 +372,10 @@ func (impl ImageScanServiceImpl) FetchExecutionDetailResult(request *ImageScanRe
 				Severity: item.CveStore.Severity.String(),
 				//Permission: "BLOCK", TODO
 			}
+			if len(item.Package) > 0 {
+				// data already migrated hence get package from image_scan_execution_result
+				vulnerability.Package = item.Package
+			}
 			if item.CveStore.Severity == security.Critical {
 				highCount = highCount + 1
 			} else if item.CveStore.Severity == security.Medium {

pkg/security/policyService.go

@@ -242,8 +242,10 @@ func (impl *PolicyServiceImpl) VerifyImage(verifyImageRequest *VerifyImageReques
 			impl.logger.Errorw("error in fetching vulnerability ", "err", err)
 			return nil, err
 		}
+		cveNameToScanResultPackageNameMapping := make(map[string]string)
 		var cveStores []*security.CveStore
 		for _, scanResult := range scanResults {
+			cveNameToScanResultPackageNameMapping[scanResult.CveStoreName] = scanResult.Package
 			cveStores = append(cveStores, &scanResult.CveStore)
 			if _, ok := scanResultsIdMap[scanResult.ImageScanExecutionHistoryId]; !ok {
 				scanResultsIdMap[scanResult.ImageScanExecutionHistoryId] = scanResult.ImageScanExecutionHistoryId
@@ -259,6 +261,13 @@ func (impl *PolicyServiceImpl) VerifyImage(verifyImageRequest *VerifyImageReques
 				Version:      cve.Version,
 				FixedVersion: cve.FixedVersion,
 			}
+			if packageName, ok := cveNameToScanResultPackageNameMapping[cve.Name]; ok {
+				if len(packageName) > 0 {
+					// fetch package name from image_scan_execution_result table
+					vr.Package = packageName
+				}
+
+			}
 			imageBlockedCves[image] = append(imageBlockedCves[image], vr)
 		}
 	}

pkg/workflow/cd/CdWorkflowRunnerService.go

@@ -22,11 +22,13 @@ import (
 	"github.com/devtron-labs/devtron/pkg/workflow/cd/bean"
 	"github.com/go-pg/pg"
 	"go.uber.org/zap"
+	"time"
 )
 
 type CdWorkflowRunnerService interface {
 	FindWorkflowRunnerById(wfrId int) (*bean.CdWorkflowRunnerDto, error)
 	CheckIfWfrLatest(wfrId, pipelineId int) (isLatest bool, err error)
+	UpdateWfrStatus(dto *bean.CdWorkflowRunnerDto, status string, updatedBy int) error
 }
 
 type CdWorkflowRunnerServiceImpl struct {
@@ -60,3 +62,16 @@ func (impl *CdWorkflowRunnerServiceImpl) CheckIfWfrLatest(wfrId, pipelineId int)
 	}
 	return isLatest, nil
 }
+
+func (impl *CdWorkflowRunnerServiceImpl) UpdateWfrStatus(dto *bean.CdWorkflowRunnerDto, status string, updatedBy int) error {
+	runnerDbObj := adapter.ConvertCdWorkflowRunnerDtoToDbObj(dto)
+	runnerDbObj.Status = status
+	runnerDbObj.UpdatedBy = int32(updatedBy)
+	runnerDbObj.UpdatedOn = time.Now()
+	err := impl.cdWorkflowRepository.UpdateWorkFlowRunner(runnerDbObj)
+	if err != nil {
+		impl.logger.Errorw("error in updating runner status in db", "runnerId", runnerDbObj.Id, "err", err)
+		return err
+	}
+	return nil
+}

pkg/workflow/cd/adapter/adapter.go

@@ -62,3 +62,26 @@ func ConvertCdWorkflowDtoToDbObj(dto *bean.CdWorkflowDto) *pipelineConfig.CdWork
 		},
 	}
 }
+
+func ConvertCdWorkflowRunnerDtoToDbObj(dto *bean.CdWorkflowRunnerDto) *pipelineConfig.CdWorkflowRunner {
+	return &pipelineConfig.CdWorkflowRunner{
+		Id:                      dto.Id,
+		Name:                    dto.Name,
+		WorkflowType:            dto.WorkflowType,
+		ExecutorType:            dto.ExecutorType,
+		Status:                  dto.Status,
+		PodStatus:               dto.PodStatus,
+		Message:                 dto.Message,
+		StartedOn:               dto.StartedOn,
+		FinishedOn:              dto.FinishedOn,
+		Namespace:               dto.Namespace,
+		LogLocation:             dto.LogLocation,
+		TriggeredBy:             dto.TriggeredBy,
+		CdWorkflowId:            dto.CdWorkflowId,
+		PodName:                 dto.PodName,
+		BlobStorageEnabled:      dto.BlobStorageEnabled,
+		RefCdWorkflowRunnerId:   dto.RefCdWorkflowRunnerId,
+		ImagePathReservationIds: dto.ImagePathReservationIds,
+		ReferenceId:             dto.ReferenceId,
+	}
+}

scripts/sql/251_alter_image_scan_result_repository.down.sql

@@ -0,0 +1 @@
+ALTER TABLE "image_scan_execution_result" DROP COLUMN "package";

scripts/sql/251_alter_image_scan_result_repository.up.sql

@@ -0,0 +1 @@
+ALTER TABLE "image_scan_execution_result" ADD COLUMN "package" text;