From 3d637b847bd2ebcba442e04c24d4e5cd66bcd28f Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Tue, 6 Feb 2024 12:08:13 +0530 Subject: [PATCH 1/6] changing material info source --- api/bean/AppView.go | 1 + .../sql/repository/AppListingRepository.go | 3 +- pkg/pipeline/CiHandler.go | 98 ++++++++++--------- 3 files changed, 56 insertions(+), 46 deletions(-) diff --git a/api/bean/AppView.go b/api/bean/AppView.go index 85054ab6b7..a1703f3232 100644 --- a/api/bean/AppView.go +++ b/api/bean/AppView.go @@ -175,6 +175,7 @@ type DeploymentDetailContainer struct { Description string `json:"description" validate:"max=40"` IsVirtualEnvironment bool `json:"isVirtualEnvironment"` HelmReleaseInstallStatus string `json:"-"` + DeployedBy int32 `json:"deployedBy"` } type AppDetailContainer struct { diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index ab7a5fc716..903a51ebe1 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -631,7 +631,8 @@ func (impl AppListingRepositoryImpl) FetchMinDetailOtherEnvironment(appId int) ( func (impl AppListingRepositoryImpl) DeploymentDetailByArtifactId(ciArtifactId int, envId int) (bean.DeploymentDetailContainer, error) { impl.Logger.Debug("reached at AppListingRepository:") var deploymentDetail bean.DeploymentDetailContainer - query := "SELECT env.id AS environment_id, env.environment_name, env.default, pco.created_on as last_deployed_time, a.app_name" + + query := "SELECT env.id AS environment_id, env.environment_name, env.default, pco.created_on as last_deployed_time, a.app_name, " + + " pco.created_by as deployed_by" + " FROM pipeline_config_override pco" + " INNER JOIN pipeline p on p.id = pco.pipeline_id" + " INNER JOIN environment env ON env.id=p.environment_id" + diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index d82e7df1f9..385fc58a5b 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -20,6 +20,7 @@ package pipeline import ( "bufio" "context" + "encoding/json" "errors" "fmt" "io/ioutil" @@ -1478,12 +1479,6 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId return &types.GitTriggerInfoResponse{}, err } - ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(ciPipeline.Id) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return &types.GitTriggerInfoResponse{}, err - } - deployDetail, err := impl.appListingRepository.DeploymentDetailByArtifactId(ciArtifactId, envId) if err != nil { impl.Logger.Errorw("err", "err", err) @@ -1494,67 +1489,80 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId var triggeredByUserEmailId string //check workflow data only for non external builds if !ciPipeline.IsExternal { - var workflow *pipelineConfig.CiWorkflow - if ciArtifact.ParentCiArtifact > 0 { - workflow, err = impl.ciWorkflowRepository.FindLastTriggeredWorkflowByArtifactId(ciArtifact.ParentCiArtifact) - if err != nil { - impl.Logger.Errorw("err", "ciArtifactId", ciArtifact.ParentCiArtifact, "err", err) - return &types.GitTriggerInfoResponse{}, err - } - } else { - workflow, err = impl.ciWorkflowRepository.FindLastTriggeredWorkflowByArtifactId(ciArtifactId) - if err != nil { - impl.Logger.Errorw("err", "ciArtifactId", ciArtifactId, "err", err) - return &types.GitTriggerInfoResponse{}, err - } - } - triggeredByUserEmailId, err = impl.userService.GetEmailById(workflow.TriggeredBy) + triggeredByUserEmailId, err = impl.userService.GetEmailById(deployDetail.DeployedBy) if err != nil && !util.IsErrNoRows(err) { impl.Logger.Errorw("err", "err", err) return &types.GitTriggerInfoResponse{}, err } + var ciMaterials []repository.CiMaterialInfo + err := json.Unmarshal([]byte(ciArtifact.MaterialInfo), &ciMaterials) + if err != nil { + println("material info", ciArtifact.MaterialInfo) + println("unmarshal error for material info", "err", err) + return &types.GitTriggerInfoResponse{}, err + } for _, m := range ciMaterials { + var history []*gitSensor.GitCommit - _gitTrigger := workflow.GitTriggers[m.Id] - // ignore git trigger which have commit and webhook both data nil - if len(_gitTrigger.Commit) == 0 && _gitTrigger.WebhookData.Id == 0 { + var modification repository.Modification + if len(m.Modifications) > 0 { + modification = m.Modifications[0] + } else { continue } - _gitCommit := &gitSensor.GitCommit{ - Message: _gitTrigger.Message, - Author: _gitTrigger.Author, - Date: _gitTrigger.Date, - Changes: _gitTrigger.Changes, - Commit: _gitTrigger.Commit, + if len(modification.Revision) == 0 && modification.WebhookData.Id == 0 { + continue } - // set webhook data - _webhookData := _gitTrigger.WebhookData - if _webhookData.Id > 0 { - _gitCommit.WebhookData = &gitSensor.WebhookData{ - Id: _webhookData.Id, - EventActionType: _webhookData.EventActionType, - Data: _webhookData.Data, + const timeLayout = "2024-02-02 11:04:31.555718+00" + commitDate, err := time.Parse(timeLayout, modification.ModifiedTime) + if err != nil { + impl.Logger.Errorw("error in parsing commit time", "commitTimeString", commitDate, "err", err) + } + gitCommit := &gitSensor.GitCommit{ + Message: modification.Message, + Author: modification.Author, + Date: commitDate, + Commit: modification.Revision, + } + if modification.WebhookData.Id > 0 { + gitCommit.WebhookData = &gitSensor.WebhookData{ + Id: modification.WebhookData.Id, + EventActionType: modification.WebhookData.EventActionType, + Data: modification.WebhookData.Data, } } + history = append(history, gitCommit) + + var url string + if m.Material.Type == "git" { + url = m.Material.GitConfiguration.URL + } else if m.Material.Type == "scm" { + url = m.Material.ScmConfiguration.URL + } else { + continue + } - history = append(history, _gitCommit) + // urlFormat := https://github.com/foo/.git, https://github.com/foo/ + var gitMaterialName string + urlSplit := strings.Split(url, "/") + repoName := urlSplit[len(urlSplit)-1] + repoNameSplit := strings.Split(repoName, ".") + gitMaterialName = repoNameSplit[0] res := pipelineConfig.CiPipelineMaterialResponse{ - Id: m.Id, - GitMaterialId: m.GitMaterialId, - GitMaterialName: m.GitMaterial.Name[strings.Index(m.GitMaterial.Name, "-")+1:], - Type: string(m.Type), - Value: m.Value, - Active: m.Active, - Url: m.GitMaterial.Url, + GitMaterialName: gitMaterialName, + Type: m.Material.Type, + Value: modification.Branch, + Url: url, History: history, } ciMaterialsArr = append(ciMaterialsArr, res) + } } imageTaggingData, err := impl.imageTaggingService.GetTagsData(ciPipeline.Id, ciPipeline.AppId, ciArtifactId, false) From 59b4e8d0245288539467288c28b3fb8d4d1f0eac Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Tue, 6 Feb 2024 15:00:52 +0530 Subject: [PATCH 2/6] changing time layout --- pkg/pipeline/CiHandler.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 385fc58a5b..142b84d8e7 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -163,6 +163,7 @@ const Starting = "Starting" const POD_DELETED_MESSAGE = "pod deleted" const TERMINATE_MESSAGE = "workflow shutdown with strategy: Terminate" const ABORT_MESSAGE_AFTER_STARTING_STAGE = "workflow shutdown with strategy: Force Abort" +const commitTimeLayout = "2006-01-02T15:04:05Z07:00" func (impl *CiHandlerImpl) CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error { @@ -1503,6 +1504,7 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId println("unmarshal error for material info", "err", err) return &types.GitTriggerInfoResponse{}, err } + for _, m := range ciMaterials { var history []*gitSensor.GitCommit @@ -1518,8 +1520,7 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId continue } - const timeLayout = "2024-02-02 11:04:31.555718+00" - commitDate, err := time.Parse(timeLayout, modification.ModifiedTime) + commitDate, err := time.Parse(commitTimeLayout, modification.ModifiedTime) if err != nil { impl.Logger.Errorw("error in parsing commit time", "commitTimeString", commitDate, "err", err) } From bfd374442f88bdecc7e88b62761d7fe027ae7d92 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Thu, 8 Feb 2024 13:08:49 +0530 Subject: [PATCH 3/6] adding for linked ci pipeline also --- pkg/pipeline/CiHandler.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 142b84d8e7..c2e1d73a39 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -1489,7 +1489,7 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId ciMaterialsArr := make([]pipelineConfig.CiPipelineMaterialResponse, 0) var triggeredByUserEmailId string //check workflow data only for non external builds - if !ciPipeline.IsExternal { + if !ciPipeline.IsExternal || ciPipeline.ParentCiPipeline > 0 { triggeredByUserEmailId, err = impl.userService.GetEmailById(deployDetail.DeployedBy) if err != nil && !util.IsErrNoRows(err) { @@ -1500,8 +1500,7 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId var ciMaterials []repository.CiMaterialInfo err := json.Unmarshal([]byte(ciArtifact.MaterialInfo), &ciMaterials) if err != nil { - println("material info", ciArtifact.MaterialInfo) - println("unmarshal error for material info", "err", err) + impl.Logger.Info("error in unmarshalling material info", ciArtifact.MaterialInfo, "err", err) return &types.GitTriggerInfoResponse{}, err } From 760f706699f353e4eb104365b1dbbb476f015f83 Mon Sep 17 00:00:00 2001 From: rajeevranjan17 Date: Tue, 2 Apr 2024 18:15:38 +0530 Subject: [PATCH 4/6] Changed Fetch-Mataerial-Info Logic --- .../configure/BuildPipelineRestHandler.go | 2 +- pkg/pipeline/CiHandler.go | 193 +++++++++++------- pkg/pipeline/ImageTaggingService.go | 9 +- 3 files changed, 127 insertions(+), 77 deletions(-) diff --git a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go index 64f2ad9c53..d0ee880ec2 100644 --- a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go @@ -2011,7 +2011,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetImageTaggingData(w http.Respons } //RBAC - resp, err := handler.imageTaggingService.GetTagsData(ciPipelineId, appId, artifactId, externalCi) + resp, err := handler.imageTaggingService.GetTagsData(ciPipelineId, appId, artifactId, externalCi, 0) if err != nil { handler.Logger.Errorw("error occurred in fetching GetTagsData for artifact ", "err", err, "artifactId", artifactId, "ciPipelineId", ciPipelineId, "externalCi", externalCi, "appId", appId) common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 4c5e0478d7..8e77cf182f 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -20,7 +20,6 @@ package pipeline import ( "bufio" "context" - "encoding/json" "errors" "fmt" "io/ioutil" @@ -1474,12 +1473,6 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId return &types.GitTriggerInfoResponse{}, err } - ciPipeline, err := impl.ciPipelineRepository.FindByIdIncludingInActive(ciArtifact.PipelineId) - if err != nil { - impl.Logger.Errorw("err", "ciArtifactId", ciArtifactId, "err", err) - return &types.GitTriggerInfoResponse{}, err - } - deployDetail, err := impl.appListingRepository.DeploymentDetailByArtifactId(ciArtifactId, envId) if err != nil { impl.Logger.Errorw("err", "err", err) @@ -1489,92 +1482,97 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId ciMaterialsArr := make([]pipelineConfig.CiPipelineMaterialResponse, 0) var triggeredByUserEmailId string //check workflow data only for non external builds - if !ciPipeline.IsExternal || ciPipeline.ParentCiPipeline > 0 { + //if !ciPipeline.IsExternal || ciPipeline.ParentCiPipeline > 0 { - triggeredByUserEmailId, err = impl.userService.GetEmailById(deployDetail.DeployedBy) - if err != nil && !util.IsErrNoRows(err) { - impl.Logger.Errorw("err", "err", err) - return &types.GitTriggerInfoResponse{}, err - } + triggeredByUserEmailId, err = impl.userService.GetEmailById(deployDetail.DeployedBy) + if err != nil && !util.IsErrNoRows(err) { + impl.Logger.Errorw("err", "err", err) + return &types.GitTriggerInfoResponse{}, err + } - var ciMaterials []repository.CiMaterialInfo - err := json.Unmarshal([]byte(ciArtifact.MaterialInfo), &ciMaterials) - if err != nil { - impl.Logger.Info("error in unmarshalling material info", ciArtifact.MaterialInfo, "err", err) - return &types.GitTriggerInfoResponse{}, err - } + ciMaterials, err := repository.GetCiMaterialInfo(ciArtifact.MaterialInfo, ciArtifact.DataSource) + if err != nil { + impl.Logger.Info("error in unmarshalling material info", ciArtifact.MaterialInfo, "err", err) + return &types.GitTriggerInfoResponse{}, err + } - for _, m := range ciMaterials { + for _, m := range ciMaterials { - var history []*gitSensor.GitCommit + var history []*gitSensor.GitCommit - var modification repository.Modification - if len(m.Modifications) > 0 { - modification = m.Modifications[0] - } else { - continue - } + var modification repository.Modification + if len(m.Modifications) > 0 { + modification = m.Modifications[0] + } else { + continue + } - if len(modification.Revision) == 0 && modification.WebhookData.Id == 0 { - continue - } + if len(modification.Revision) == 0 && modification.WebhookData.Id == 0 { + continue + } - commitDate, err := time.Parse(commitTimeLayout, modification.ModifiedTime) - if err != nil { - impl.Logger.Errorw("error in parsing commit time", "commitTimeString", commitDate, "err", err) - } - gitCommit := &gitSensor.GitCommit{ - Message: modification.Message, - Author: modification.Author, - Date: commitDate, - Commit: modification.Revision, - } - if modification.WebhookData.Id > 0 { - gitCommit.WebhookData = &gitSensor.WebhookData{ - Id: modification.WebhookData.Id, - EventActionType: modification.WebhookData.EventActionType, - Data: modification.WebhookData.Data, - } + commitDate, err := time.Parse(commitTimeLayout, modification.ModifiedTime) + if err != nil { + impl.Logger.Errorw("error in parsing commit time", "commitTimeString", commitDate, "err", err) + } + gitCommit := &gitSensor.GitCommit{ + Message: modification.Message, + Author: modification.Author, + Date: commitDate, + Commit: modification.Revision, + } + if modification.WebhookData.Id > 0 { + gitCommit.WebhookData = &gitSensor.WebhookData{ + Id: modification.WebhookData.Id, + EventActionType: modification.WebhookData.EventActionType, + Data: modification.WebhookData.Data, } - history = append(history, gitCommit) + } + history = append(history, gitCommit) - var url string - if m.Material.Type == "git" { - url = m.Material.GitConfiguration.URL - } else if m.Material.Type == "scm" { - url = m.Material.ScmConfiguration.URL - } else { - continue - } + var url string + if m.Material.Type == "git" { + url = m.Material.GitConfiguration.URL + } else if m.Material.Type == "scm" { + url = m.Material.ScmConfiguration.URL + } else { + continue + } - // urlFormat := https://github.com/foo/.git, https://github.com/foo/ - var gitMaterialName string - urlSplit := strings.Split(url, "/") - repoName := urlSplit[len(urlSplit)-1] - repoNameSplit := strings.Split(repoName, ".") - gitMaterialName = repoNameSplit[0] - - res := pipelineConfig.CiPipelineMaterialResponse{ - GitMaterialName: gitMaterialName, - Type: m.Material.Type, - Value: modification.Branch, - Url: url, - History: history, - } - ciMaterialsArr = append(ciMaterialsArr, res) + // urlFormat := https://github.com/foo/.git, https://github.com/foo/ + var gitMaterialName string + urlSplit := strings.Split(url, "/") + repoName := urlSplit[len(urlSplit)-1] + repoNameSplit := strings.Split(repoName, ".") + gitMaterialName = repoNameSplit[0] + res := pipelineConfig.CiPipelineMaterialResponse{ + GitMaterialName: gitMaterialName, + Type: m.Material.Type, + Value: modification.Branch, + Url: url, + History: history, } + ciMaterialsArr = append(ciMaterialsArr, res) + } - imageTaggingData, err := impl.imageTaggingService.GetTagsData(ciPipeline.Id, ciPipeline.AppId, ciArtifactId, false) + //} + ciPipelineId, appId, isExternalCi, cdPipelineId, err := impl.fetchVariablesForImageTagging(ciArtifact.DataSource, ciArtifact.PipelineId, ciArtifact.ExternalCiPipelineId, ciArtifact.ComponentId) if err != nil { - impl.Logger.Errorw("error in fetching imageTaggingData", "err", err, "ciPipelineId", ciPipeline.Id, "appId", ciPipeline.AppId, "ciArtifactId", ciArtifactId) + impl.Logger.Errorw("error in fetching ciPipelineId, appId", "err", err, "PipelineId", ciArtifact.PipelineId, "ciArtifactId", ciArtifactId) return &types.GitTriggerInfoResponse{}, err } + imageTaggingData, err := impl.imageTaggingService.GetTagsData(ciPipelineId, appId, ciArtifactId, isExternalCi, cdPipelineId) + if err != nil { + impl.Logger.Errorw("error in fetching imageTaggingData", "err", err, "ciPipelineId", ciPipelineId, "appId", appId, "ciArtifactId", ciArtifactId) + return &types.GitTriggerInfoResponse{}, err + } + gitTriggerInfoResponse := &types.GitTriggerInfoResponse{ //GitTriggers: workflow.GitTriggers, CiMaterials: ciMaterialsArr, TriggeredByEmail: triggeredByUserEmailId, - AppId: ciPipeline.AppId, + AppId: appId, AppName: deployDetail.AppName, EnvironmentId: deployDetail.EnvironmentId, EnvironmentName: deployDetail.EnvironmentName, @@ -1586,6 +1584,55 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId return gitTriggerInfoResponse, nil } +func (impl *CiHandlerImpl) fetchVariablesForImageTagging(dataSource string, ciPipelineId int, externalCiPipelineId int, componentId int) (int, int, bool, int, error) { + + isExternalCi := false + var appId int + var cdPipelineId int + //CI_RUNNER,EXTERNAL,post_ci,pre_cd,post_cd + + switch dataSource { + case repository.CI_RUNNER: + ciPipeline, err := impl.ciPipelineRepository.FindByIdIncludingInActive(ciPipelineId) + if err != nil { + impl.Logger.Errorw("err in fetching the ciPipeline", "ciPipelineId", ciPipelineId, "err", err) + return 0, 0, isExternalCi, 0, err + } + ciPipelineId = ciPipeline.Id + appId = ciPipeline.AppId + + case repository.POST_CI: + ciPipeline, err := impl.ciPipelineRepository.FindByIdIncludingInActive(componentId) + if err != nil { + impl.Logger.Errorw("err in fetching the ciPipeline", "ciPipelineId", ciPipelineId, "err", err) + return 0, 0, isExternalCi, 0, err + } + ciPipelineId = ciPipeline.Id + appId = ciPipeline.AppId + case repository.PRE_CD, repository.POST_CD: + cdPipeline, err := impl.cdPipelineRepository.FindById(componentId) + if err != nil { + impl.Logger.Errorw("err in fetching the ciPipeline", "ciPipelineId", ciPipelineId, "err", err) + return 0, 0, isExternalCi, 0, err + } + cdPipelineId = cdPipeline.Id + appId = cdPipeline.AppId + case repository.WEBHOOK: + externalCiPipeline, err := impl.ciPipelineRepository.FindExternalCiById(externalCiPipelineId) + if err != nil { + impl.Logger.Errorw("err in fetching the ciPipeline", "ciPipelineId", ciPipelineId, "err", err) + return 0, 0, isExternalCi, 0, err + } + ciPipelineId = externalCiPipeline.Id + appId = externalCiPipeline.AppId + isExternalCi = true + default: + return 0, 0, false, 0, errors.New("invalid data source") + } + return ciPipelineId, appId, isExternalCi, cdPipelineId, nil + +} + func (impl *CiHandlerImpl) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error { ciWorkflows, err := impl.ciWorkflowRepository.FindByStatusesIn([]string{Starting, Running}) if err != nil { diff --git a/pkg/pipeline/ImageTaggingService.go b/pkg/pipeline/ImageTaggingService.go index 1462474fca..c1334904f9 100644 --- a/pkg/pipeline/ImageTaggingService.go +++ b/pkg/pipeline/ImageTaggingService.go @@ -45,7 +45,7 @@ type ImageTaggingService interface { //AppReleaseTags -> all the tags of the given appId, //imageComment -> comment of the given artifactId, // ProdEnvExists -> implies the existence of prod environment in any workflow of given ciPipelineId or its child ciPipelineRequest's - GetTagsData(ciPipelineId, appId, artifactId int, externalCi bool) (*types.ImageTaggingResponseDTO, error) + GetTagsData(ciPipelineId, appId, artifactId int, externalCi bool, cdPipelineId int) (*types.ImageTaggingResponseDTO, error) CreateOrUpdateImageTagging(ciPipelineId, appId, artifactId, userId int, imageTaggingRequest *types.ImageTaggingRequestDTO) (*types.ImageTaggingResponseDTO, error) GetProdEnvFromParentAndLinkedWorkflow(ciPipelineId int) (bool, error) GetProdEnvByCdPipelineId(pipelineId int) (bool, error) @@ -100,7 +100,7 @@ func (impl ImageTaggingServiceImpl) GetImageTaggingServiceConfig() ImageTaggingS // AppReleaseTags -> all the tags of the given appId, // imageComment -> comment of the given artifactId, // ProdEnvExists -> implies the existence of prod environment in any workflow of given ciPipelineId or its child ciPipelineRequest's -func (impl ImageTaggingServiceImpl) GetTagsData(ciPipelineId, appId, artifactId int, externalCi bool) (*types.ImageTaggingResponseDTO, error) { +func (impl ImageTaggingServiceImpl) GetTagsData(ciPipelineId, appId, artifactId int, externalCi bool, cdPipelineId int) (*types.ImageTaggingResponseDTO, error) { resp := &types.ImageTaggingResponseDTO{} imageComment, err := impl.imageTaggingRepo.GetImageComment(artifactId) if err != nil && err != pg.ErrNoRows { @@ -120,9 +120,12 @@ func (impl ImageTaggingServiceImpl) GetTagsData(ciPipelineId, appId, artifactId prodEnvExists := false if externalCi { prodEnvExists, err = impl.FindProdEnvExists(externalCi, []int{ciPipelineId}) + } else if cdPipelineId > 0 { + prodEnvExists, err = impl.GetProdEnvByCdPipelineId(cdPipelineId) } else { prodEnvExists, err = impl.GetProdEnvFromParentAndLinkedWorkflow(ciPipelineId) } + if err != nil { impl.logger.Errorw("error in finding prodEnvExists value", "err", err, "ciPipelineId", ciPipelineId, "externalCi", externalCi) return resp, err @@ -311,7 +314,7 @@ func (impl ImageTaggingServiceImpl) CreateOrUpdateImageTagging(ciPipelineId, app impl.logger.Errorw("error in committing transaction", "err", err, "ciPipelineId", ciPipelineId, "appId", appId, "artifactId", artifactId, "userId", userId, "imageTaggingRequest", imageTaggingRequest) return nil, err } - return impl.GetTagsData(ciPipelineId, appId, artifactId, imageTaggingRequest.ExternalCi) + return impl.GetTagsData(ciPipelineId, appId, artifactId, imageTaggingRequest.ExternalCi, 0) } func (impl ImageTaggingServiceImpl) performTagOperationsAndGetAuditList(tx *pg.Tx, appId, artifactId, userId int, imageTaggingRequest *types.ImageTaggingRequestDTO) ([]*repository.ImageTaggingAudit, error) { From 61ff19171185d0d3db41cc623e9bbbd5787752c3 Mon Sep 17 00:00:00 2001 From: rajeevranjan17 Date: Wed, 3 Apr 2024 11:32:55 +0530 Subject: [PATCH 5/6] removed comments and log errors --- pkg/pipeline/CiHandler.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 8e77cf182f..657473d444 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -1475,14 +1475,12 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId deployDetail, err := impl.appListingRepository.DeploymentDetailByArtifactId(ciArtifactId, envId) if err != nil { - impl.Logger.Errorw("err", "err", err) + impl.Logger.Errorw("error in getting deploy detail", "ciArtifactId", ciArtifactId, "err", err) return &types.GitTriggerInfoResponse{}, err } ciMaterialsArr := make([]pipelineConfig.CiPipelineMaterialResponse, 0) var triggeredByUserEmailId string - //check workflow data only for non external builds - //if !ciPipeline.IsExternal || ciPipeline.ParentCiPipeline > 0 { triggeredByUserEmailId, err = impl.userService.GetEmailById(deployDetail.DeployedBy) if err != nil && !util.IsErrNoRows(err) { @@ -1556,7 +1554,7 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId ciMaterialsArr = append(ciMaterialsArr, res) } - //} + ciPipelineId, appId, isExternalCi, cdPipelineId, err := impl.fetchVariablesForImageTagging(ciArtifact.DataSource, ciArtifact.PipelineId, ciArtifact.ExternalCiPipelineId, ciArtifact.ComponentId) if err != nil { impl.Logger.Errorw("error in fetching ciPipelineId, appId", "err", err, "PipelineId", ciArtifact.PipelineId, "ciArtifactId", ciArtifactId) From 2466664240b0b0deb3bbb7241b1d4b11f9e22393 Mon Sep 17 00:00:00 2001 From: rajeevranjan17 Date: Wed, 3 Apr 2024 13:08:10 +0530 Subject: [PATCH 6/6] made a separate util function for parsing ci material details --- .../sql/repository/CiArtifactRepository.go | 7 +- pkg/pipeline/CiHandler.go | 78 ++++++++++--------- 2 files changed, 46 insertions(+), 39 deletions(-) diff --git a/internal/sql/repository/CiArtifactRepository.go b/internal/sql/repository/CiArtifactRepository.go index 4a805a94f2..3c7b473a2a 100644 --- a/internal/sql/repository/CiArtifactRepository.go +++ b/internal/sql/repository/CiArtifactRepository.go @@ -34,6 +34,7 @@ import ( type credentialsSource = string type ArtifactsSourceType = string +type ciMaterialType = string const ( GLOBAL_CONTAINER_REGISTRY credentialsSource = "global_container_registry" @@ -50,6 +51,8 @@ const ( // deprecated; Handled for backward compatibility EXT ArtifactsSourceType = "ext" // PRE_CI is not a valid DataSource for an artifact + MATERIAL_TYPE_GIT ciMaterialType = "git" + MATERIAL_TYPE_SCM ciMaterialType = "scm" ) type CiArtifactWithExtraData struct { @@ -515,9 +518,9 @@ func (artifact *CiArtifact) ParseMaterialInfo() (map[string]string, error) { scmMap := map[string]string{} for _, material := range ciMaterials { var url string - if material.Material.Type == "git" { + if material.Material.Type == MATERIAL_TYPE_GIT { url = material.Material.GitConfiguration.URL - } else if material.Material.Type == "scm" { + } else if material.Material.Type == MATERIAL_TYPE_SCM { url = material.Material.ScmConfiguration.URL } else { return nil, fmt.Errorf("unknown material type:%s ", material.Material.Type) diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 657473d444..b6db422338 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -1469,22 +1469,20 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId ciArtifact, err := impl.ciArtifactRepository.Get(ciArtifactId) if err != nil { - impl.Logger.Errorw("err", "ciArtifactId", ciArtifactId, "err", err) + impl.Logger.Errorw("error in getting CiArtifact Details", "ciArtifactId", ciArtifactId, "err", err) return &types.GitTriggerInfoResponse{}, err } deployDetail, err := impl.appListingRepository.DeploymentDetailByArtifactId(ciArtifactId, envId) if err != nil { - impl.Logger.Errorw("error in getting deploy detail", "ciArtifactId", ciArtifactId, "err", err) + impl.Logger.Errorw("error in getting deploy detail", "ciArtifactId", ciArtifactId, "envId", envId, "err", err) return &types.GitTriggerInfoResponse{}, err } - ciMaterialsArr := make([]pipelineConfig.CiPipelineMaterialResponse, 0) var triggeredByUserEmailId string - triggeredByUserEmailId, err = impl.userService.GetEmailById(deployDetail.DeployedBy) if err != nil && !util.IsErrNoRows(err) { - impl.Logger.Errorw("err", "err", err) + impl.Logger.Errorw("error in getting the user email Id ", "userId", deployDetail.DeployedBy, "err", err) return &types.GitTriggerInfoResponse{}, err } @@ -1494,6 +1492,37 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId return &types.GitTriggerInfoResponse{}, err } + ciMaterialsArr := impl.parseCiMaterials(ciMaterials) + + ciPipelineId, appId, isExternalCi, cdPipelineId, err := impl.fetchVariablesForImageTagging(ciArtifact.DataSource, ciArtifact.PipelineId, ciArtifact.ExternalCiPipelineId, ciArtifact.ComponentId) + if err != nil { + impl.Logger.Errorw("error in fetching variables for Image Tagging ", "PipelineId", ciArtifact.PipelineId, "ciArtifactId", ciArtifactId, "dataSource", ciArtifact.DataSource, "externalCiPipelineId", ciArtifact.ExternalCiPipelineId, "componentId", ciArtifact.ComponentId, "err", err) + return &types.GitTriggerInfoResponse{}, err + } + imageTaggingData, err := impl.imageTaggingService.GetTagsData(ciPipelineId, appId, ciArtifactId, isExternalCi, cdPipelineId) + if err != nil { + impl.Logger.Errorw("error in fetching imageTaggingData", "ciPipelineId", ciPipelineId, "appId", appId, "ciArtifactId", ciArtifactId, "cdPipelineId", cdPipelineId, "isExternalCi", isExternalCi, "err", err) + return &types.GitTriggerInfoResponse{}, err + } + + gitTriggerInfoResponse := &types.GitTriggerInfoResponse{ + //GitTriggers: workflow.GitTriggers, + CiMaterials: ciMaterialsArr, + TriggeredByEmail: triggeredByUserEmailId, + AppId: appId, + AppName: deployDetail.AppName, + EnvironmentId: deployDetail.EnvironmentId, + EnvironmentName: deployDetail.EnvironmentName, + LastDeployedTime: deployDetail.LastDeployedTime, + Default: deployDetail.Default, + ImageTaggingData: *imageTaggingData, + Image: ciArtifact.Image, + } + return gitTriggerInfoResponse, nil +} + +func (impl *CiHandlerImpl) parseCiMaterials(ciMaterials []repository.CiMaterialInfo) []pipelineConfig.CiPipelineMaterialResponse { + ciMaterialsArr := make([]pipelineConfig.CiPipelineMaterialResponse, 0) for _, m := range ciMaterials { var history []*gitSensor.GitCommit @@ -1529,9 +1558,9 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId history = append(history, gitCommit) var url string - if m.Material.Type == "git" { + if m.Material.Type == repository.MATERIAL_TYPE_GIT { url = m.Material.GitConfiguration.URL - } else if m.Material.Type == "scm" { + } else if m.Material.Type == repository.MATERIAL_TYPE_SCM { url = m.Material.ScmConfiguration.URL } else { continue @@ -1552,34 +1581,9 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId History: history, } ciMaterialsArr = append(ciMaterialsArr, res) - } - ciPipelineId, appId, isExternalCi, cdPipelineId, err := impl.fetchVariablesForImageTagging(ciArtifact.DataSource, ciArtifact.PipelineId, ciArtifact.ExternalCiPipelineId, ciArtifact.ComponentId) - if err != nil { - impl.Logger.Errorw("error in fetching ciPipelineId, appId", "err", err, "PipelineId", ciArtifact.PipelineId, "ciArtifactId", ciArtifactId) - return &types.GitTriggerInfoResponse{}, err - } - imageTaggingData, err := impl.imageTaggingService.GetTagsData(ciPipelineId, appId, ciArtifactId, isExternalCi, cdPipelineId) - if err != nil { - impl.Logger.Errorw("error in fetching imageTaggingData", "err", err, "ciPipelineId", ciPipelineId, "appId", appId, "ciArtifactId", ciArtifactId) - return &types.GitTriggerInfoResponse{}, err - } - - gitTriggerInfoResponse := &types.GitTriggerInfoResponse{ - //GitTriggers: workflow.GitTriggers, - CiMaterials: ciMaterialsArr, - TriggeredByEmail: triggeredByUserEmailId, - AppId: appId, - AppName: deployDetail.AppName, - EnvironmentId: deployDetail.EnvironmentId, - EnvironmentName: deployDetail.EnvironmentName, - LastDeployedTime: deployDetail.LastDeployedTime, - Default: deployDetail.Default, - ImageTaggingData: *imageTaggingData, - Image: ciArtifact.Image, - } - return gitTriggerInfoResponse, nil + return ciMaterialsArr } func (impl *CiHandlerImpl) fetchVariablesForImageTagging(dataSource string, ciPipelineId int, externalCiPipelineId int, componentId int) (int, int, bool, int, error) { @@ -1587,8 +1591,8 @@ func (impl *CiHandlerImpl) fetchVariablesForImageTagging(dataSource string, ciPi isExternalCi := false var appId int var cdPipelineId int - //CI_RUNNER,EXTERNAL,post_ci,pre_cd,post_cd + //CI_RUNNER,EXTERNAL,post_ci,pre_cd,post_cd switch dataSource { case repository.CI_RUNNER: ciPipeline, err := impl.ciPipelineRepository.FindByIdIncludingInActive(ciPipelineId) @@ -1602,7 +1606,7 @@ func (impl *CiHandlerImpl) fetchVariablesForImageTagging(dataSource string, ciPi case repository.POST_CI: ciPipeline, err := impl.ciPipelineRepository.FindByIdIncludingInActive(componentId) if err != nil { - impl.Logger.Errorw("err in fetching the ciPipeline", "ciPipelineId", ciPipelineId, "err", err) + impl.Logger.Errorw("err in fetching the ciPipeline", "ciPipelineId", ciPipelineId, "componentId", componentId, "err", err) return 0, 0, isExternalCi, 0, err } ciPipelineId = ciPipeline.Id @@ -1610,7 +1614,7 @@ func (impl *CiHandlerImpl) fetchVariablesForImageTagging(dataSource string, ciPi case repository.PRE_CD, repository.POST_CD: cdPipeline, err := impl.cdPipelineRepository.FindById(componentId) if err != nil { - impl.Logger.Errorw("err in fetching the ciPipeline", "ciPipelineId", ciPipelineId, "err", err) + impl.Logger.Errorw("err in fetching the cdPipeline", "ciPipelineId", ciPipelineId, "componentId", componentId, "err", err) return 0, 0, isExternalCi, 0, err } cdPipelineId = cdPipeline.Id @@ -1618,7 +1622,7 @@ func (impl *CiHandlerImpl) fetchVariablesForImageTagging(dataSource string, ciPi case repository.WEBHOOK: externalCiPipeline, err := impl.ciPipelineRepository.FindExternalCiById(externalCiPipelineId) if err != nil { - impl.Logger.Errorw("err in fetching the ciPipeline", "ciPipelineId", ciPipelineId, "err", err) + impl.Logger.Errorw("err in fetching the external ciPipeline", "ciPipelineId", ciPipelineId, "externalCiPipelineId", externalCiPipelineId, "err", err) return 0, 0, isExternalCi, 0, err } ciPipelineId = externalCiPipeline.Id