feat: gitops async with timeline (#5286)

* feat: replace user_deployment_request.status with pipeline_status_timelines

* updated schema migration script

* fix: migration script

* update readme to include multi arch flag (#4998)

* updated trigger event func and timeline names

* review comment incorporation

* fix: GetAllInCompleteRequests query

* fix: GetAllInCompleteRequests query

* fixed: UpdatePreviousQueuedRunnerStatus error

* fix: loogers

* doc: Added enhancements to security doc (#5203)

* Rough Draft Initiated

* Created Security Enhancement Docs + Other Adhoc Fixes

* Incorporated PM Feedback

* Modified Portforward Section to Kubectl Section (#5236)

* feat: cd pipeline deployment history refactoring (#5200)

* searchableKey service move

* migrated cd deployment history code

* idempotency of triggerPipeline func and minor fixes

* timeline fixes

* updated: NewTriggerEvent func

* updated ErrorDeploymentSuperseded err handling

* updated: GetArgoPipelinesHavingLatestTriggerStuckInNonTerminalStatuses statuses

* fix: timeline status

* refactored: UpdatePreviousQueuedRunnerStatus

* fix: GetPreviousQueuedRunners query

* fix: UpdateRunnerStatusToFailedForIds query

* fix: buildTriggerEventForOverrideRequest func

* fix: UpdateRunnerStatusToFailedForIds query

* fix: UpdateRunnerStatusToFailedForIds query

* updated: handleAsyncTriggerReleaseError func

* fix: concurrent case handlings

* updated folder name

* updated GetLastStatusPublishedTimeForWfrId func

* feat: removed audit timeline and error handlings

* fix: FetchTimelinesForWfrIdExcludingStatuses query

* fix: deleted pipeline flow

* timeline handling

* timeline handling fixed

* removed unused const

* chore: updated go-bitbucket version

* removed unnecessary alais

---------

Co-authored-by: ayu-devtron <167413063+ayu-devtron@users.noreply.github.com>
Co-authored-by: ashokdevtron <141001279+ashokdevtron@users.noreply.github.com>
Co-authored-by: kartik-579 <84493919+kartik-579@users.noreply.github.com>

Author: 4 people

.gitbook.yaml

@@ -77,3 +77,4 @@ redirects:
     global-configurations/api-token: user-guide/global-configurations/authorization/api-tokens.md
     user-guide/creating-application/workflow/ci-pipeline2: user-guide/creating-application/workflow/ci-pipeline.md
     user-guide/clusters: user-guide/resource-browser.md
+    usage/clusters: user-guide/resource-browser.md

README.md

@@ -160,6 +160,10 @@ helm install devtron devtron/devtron-operator --create-namespace --namespace dev
 
 ```
 
+### Install Multi-Architecture Nodes (ARM and AMD)
+
+To install Devtron on clusters with the multi-architecture nodes (ARM and AMD), append the Devtron installation command with ```--set installer.arch=multi-arch```
+
 ## :blue_heart: Technology
 
 Devtron is built on some of the most trusted and loved technologies:

Wire.go

@@ -1,3 +1,6 @@
+//go:build wireinject
+// +build wireinject
+
 /*
  * Copyright (c) 2024. Devtron Inc.
  *
@@ -14,9 +17,6 @@
  * limitations under the License.
  */
 
-//go:build wireinject
-// +build wireinject
-
 package main
 
 import (
@@ -39,6 +39,7 @@ import (
 	"github.com/devtron-labs/devtron/api/connector"
 	"github.com/devtron-labs/devtron/api/dashboardEvent"
 	"github.com/devtron-labs/devtron/api/deployment"
+	"github.com/devtron-labs/devtron/api/devtronResource"
 	"github.com/devtron-labs/devtron/api/externalLink"
 	client "github.com/devtron-labs/devtron/api/helm-app"
 	"github.com/devtron-labs/devtron/api/infraConfig"
@@ -121,9 +122,8 @@ import (
 	delete2 "github.com/devtron-labs/devtron/pkg/delete"
 	deployment2 "github.com/devtron-labs/devtron/pkg/deployment"
 	git2 "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git"
+	"github.com/devtron-labs/devtron/pkg/deployment/manifest/publish"
 	"github.com/devtron-labs/devtron/pkg/deploymentGroup"
-	"github.com/devtron-labs/devtron/pkg/devtronResource"
-	repository9 "github.com/devtron-labs/devtron/pkg/devtronResource/repository"
 	"github.com/devtron-labs/devtron/pkg/dockerRegistry"
 	"github.com/devtron-labs/devtron/pkg/eventProcessor"
 	"github.com/devtron-labs/devtron/pkg/generateManifest"
@@ -196,6 +196,9 @@ func InitializeApp() (*App, error) {
 
 		eventProcessor.EventProcessorWireSet,
 		workflow3.WorkflowWireSet,
+
+		devtronResource.DevtronResourceWireSet,
+
 		// -------wireset end ----------
 		// -------
 		gitSensor.GetConfig,
@@ -924,8 +927,8 @@ func InitializeApp() (*App, error) {
 		wire.Bind(new(executors.SystemWorkflowExecutor), new(*executors.SystemWorkflowExecutorImpl)),
 		repository5.NewManifestPushConfigRepository,
 		wire.Bind(new(repository5.ManifestPushConfigRepository), new(*repository5.ManifestPushConfigRepositoryImpl)),
-		app.NewGitOpsManifestPushServiceImpl,
-		wire.Bind(new(app.GitOpsPushService), new(*app.GitOpsManifestPushServiceImpl)),
+		publish.NewGitOpsManifestPushServiceImpl,
+		wire.Bind(new(publish.GitOpsPushService), new(*publish.GitOpsManifestPushServiceImpl)),
 
 		// start: docker registry wire set injection
 		router.NewDockerRegRouterImpl,
@@ -951,12 +954,6 @@ func InitializeApp() (*App, error) {
 		resourceQualifiers.NewQualifierMappingServiceImpl,
 		wire.Bind(new(resourceQualifiers.QualifierMappingService), new(*resourceQualifiers.QualifierMappingServiceImpl)),
 
-		repository9.NewDevtronResourceSearchableKeyRepositoryImpl,
-		wire.Bind(new(repository9.DevtronResourceSearchableKeyRepository), new(*repository9.DevtronResourceSearchableKeyRepositoryImpl)),
-
-		devtronResource.NewDevtronResourceSearchableKeyServiceImpl,
-		wire.Bind(new(devtronResource.DevtronResourceSearchableKeyService), new(*devtronResource.DevtronResourceSearchableKeyServiceImpl)),
-
 		argocdServer.NewArgoClientWrapperServiceImpl,
 		wire.Bind(new(argocdServer.ArgoClientWrapperService), new(*argocdServer.ArgoClientWrapperServiceImpl)),
 

api/bean/ValuesOverrideRequest.go

@@ -49,6 +49,14 @@ func (workflowType WorkflowType) WorkflowTypeToStageType() repository.PipelineSt
 	}
 }
 
+func (workflowType WorkflowType) IsStageTypeDeploy() bool {
+	switch workflowType {
+	case CD_WORKFLOW_TYPE_DEPLOY:
+		return true
+	}
+	return false
+}
+
 type ValuesOverrideRequest struct {
 	PipelineId                            int                         `json:"pipelineId" validate:"required"`
 	AppId                                 int                         `json:"appId" validate:"required"`

api/devtronResource/DevtronResourceHistoryRestHandler.go

@@ -0,0 +1,125 @@
+package devtronResource
+
+import (
+	"fmt"
+	apiBean "github.com/devtron-labs/devtron/api/devtronResource/bean"
+	"github.com/devtron-labs/devtron/api/restHandler/common"
+	"github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin"
+	"github.com/devtron-labs/devtron/pkg/devtronResource"
+	"github.com/devtron-labs/devtron/pkg/devtronResource/adapter"
+	"github.com/devtron-labs/devtron/pkg/devtronResource/bean"
+	"github.com/devtron-labs/devtron/pkg/devtronResource/history/deployment/cdPipeline"
+	"github.com/devtron-labs/devtron/util/rbac"
+	"github.com/gorilla/schema"
+	"go.uber.org/zap"
+	"net/http"
+)
+
+type HistoryRestHandler interface {
+	GetDeploymentHistory(w http.ResponseWriter, r *http.Request)
+	GetDeploymentHistoryConfigList(w http.ResponseWriter, r *http.Request)
+}
+
+type HistoryRestHandlerImpl struct {
+	logger                   *zap.SugaredLogger
+	enforcer                 casbin.Enforcer
+	deploymentHistoryService cdPipeline.DeploymentHistoryService
+	apiReqDecoderService     devtronResource.APIReqDecoderService
+	enforcerUtil             rbac.EnforcerUtil
+}
+
+func NewHistoryRestHandlerImpl(logger *zap.SugaredLogger,
+	enforcer casbin.Enforcer,
+	deploymentHistoryService cdPipeline.DeploymentHistoryService,
+	apiReqDecoderService devtronResource.APIReqDecoderService,
+	enforcerUtil rbac.EnforcerUtil) *HistoryRestHandlerImpl {
+	return &HistoryRestHandlerImpl{
+		logger:                   logger,
+		enforcer:                 enforcer,
+		deploymentHistoryService: deploymentHistoryService,
+		apiReqDecoderService:     apiReqDecoderService,
+		enforcerUtil:             enforcerUtil,
+	}
+}
+
+func (handler *HistoryRestHandlerImpl) GetDeploymentHistory(w http.ResponseWriter, r *http.Request) {
+	kind, _, _, caughtError := getKindSubKindVersion(w, r)
+	if caughtError || kind != bean.DevtronResourceCdPipeline.ToString() {
+		common.WriteJsonResp(w, fmt.Errorf(apiBean.RequestInvalidKindVersionErrMessage), nil, http.StatusBadRequest)
+		return
+	}
+	v := r.URL.Query()
+	var decoder = schema.NewDecoder()
+	decoder.IgnoreUnknownKeys(true)
+	queryParams := apiBean.GetHistoryQueryParams{}
+	err := decoder.Decode(&queryParams, v)
+	if err != nil {
+		common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
+		return
+	}
+	decodedReqBean, err := handler.apiReqDecoderService.GetFilterCriteriaParamsForDeploymentHistory(queryParams.FilterCriteria)
+	if err != nil {
+		handler.logger.Errorw("error in getting filter criteria params", "err", err, "filterCriteria", queryParams.FilterCriteria)
+		common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
+		return
+	}
+
+	// RBAC START
+	token := r.Header.Get("token")
+	resourceName := handler.enforcerUtil.GetAppRBACNameByAppId(decodedReqBean.AppId)
+	if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionGet, resourceName); !ok {
+		common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden)
+		return
+	}
+	// RBAC END
+
+	resp, err := handler.deploymentHistoryService.
+		GetCdPipelineDeploymentHistory(adapter.GetCDDeploymentHistoryListReq(&queryParams, decodedReqBean))
+	if err != nil {
+		handler.logger.Errorw("service error, GetCdPipelineDeploymentHistory", "err", err)
+		common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)
+		return
+	}
+	common.WriteJsonResp(w, err, resp, http.StatusOK)
+	return
+}
+
+func (handler *HistoryRestHandlerImpl) GetDeploymentHistoryConfigList(w http.ResponseWriter, r *http.Request) {
+	kind, _, _, caughtError := getKindSubKindVersion(w, r)
+	if caughtError || kind != bean.DevtronResourceCdPipeline.ToString() {
+		common.WriteJsonResp(w, fmt.Errorf(apiBean.RequestInvalidKindVersionErrMessage), nil, http.StatusBadRequest)
+		return
+	}
+	v := r.URL.Query()
+	var decoder = schema.NewDecoder()
+	decoder.IgnoreUnknownKeys(true)
+	queryParams := apiBean.GetHistoryConfigQueryParams{}
+	err := decoder.Decode(&queryParams, v)
+	if err != nil {
+		common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
+		return
+	}
+	decodedReqBean, err := handler.apiReqDecoderService.GetFilterCriteriaParamsForDeploymentHistory(queryParams.FilterCriteria)
+	if err != nil {
+		handler.logger.Errorw("error in getting filter criteria params", "err", err, "filterCriteria", queryParams.FilterCriteria)
+		common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
+		return
+	}
+	//RBAC START
+	token := r.Header.Get("token")
+	resourceName := handler.enforcerUtil.GetAppRBACNameByAppId(decodedReqBean.AppId)
+	if isValidated := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionGet, resourceName); !isValidated {
+		common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden)
+		return
+	}
+	//RBAC END
+	resp, err := handler.deploymentHistoryService.
+		GetCdPipelineDeploymentHistoryConfigList(adapter.GetCDDeploymentHistoryConfigListReq(&queryParams, decodedReqBean))
+	if err != nil {
+		handler.logger.Errorw("service error, GetCdPipelineDeploymentHistoryConfigList", "err", err)
+		common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)
+		return
+	}
+	common.WriteJsonResp(w, err, resp, http.StatusOK)
+	return
+}

api/devtronResource/DevtronResourceHistoryRouter.go

@@ -0,0 +1,23 @@
+package devtronResource
+
+import "github.com/gorilla/mux"
+
+type HistoryRouter interface {
+	InitDtResourceHistoryRouter(devtronResourceRouter *mux.Router)
+}
+
+type HistoryRouterImpl struct {
+	dtResourceHistoryRestHandler HistoryRestHandler
+}
+
+func NewHistoryRouterImpl(dtResourceHistoryRestHandler HistoryRestHandler) *HistoryRouterImpl {
+	return &HistoryRouterImpl{dtResourceHistoryRestHandler: dtResourceHistoryRestHandler}
+}
+
+func (router *HistoryRouterImpl) InitDtResourceHistoryRouter(historyRouter *mux.Router) {
+	historyRouter.Path("/deployment/config/{kind:[a-zA-Z0-9/-]+}/{version:[a-zA-Z0-9]+}").
+		HandlerFunc(router.dtResourceHistoryRestHandler.GetDeploymentHistoryConfigList).Methods("GET")
+
+	historyRouter.Path("/deployment/{kind:[a-zA-Z0-9/-]+}/{version:[a-zA-Z0-9]+}").
+		HandlerFunc(router.dtResourceHistoryRestHandler.GetDeploymentHistory).Methods("GET")
+}

api/devtronResource/DevtronResourceRestHandler.go

@@ -0,0 +1,31 @@
+package devtronResource
+
+import (
+	"fmt"
+	apiBean "github.com/devtron-labs/devtron/api/devtronResource/bean"
+	"github.com/devtron-labs/devtron/api/restHandler/common"
+	"github.com/devtron-labs/devtron/pkg/devtronResource/helper"
+	"github.com/gorilla/mux"
+	"net/http"
+)
+
+func getKindSubKindVersion(w http.ResponseWriter, r *http.Request) (kind string, subKind string, version string, caughtError bool) {
+	vars := mux.Vars(r)
+	kindVar := vars[apiBean.PathParamKind]
+	versionVar := vars[apiBean.PathParamVersion]
+	kind, subKind, statusCode, err := resolveKindSubKindValues(kindVar)
+	if err != nil {
+		common.WriteJsonResp(w, err, nil, statusCode)
+		caughtError = true
+	}
+	return kind, subKind, versionVar, caughtError
+}
+
+func resolveKindSubKindValues(kindVar string) (kind, subKind string, statusCode int, err error) {
+	kind, subKind, err = helper.GetKindAndSubKindFrom(kindVar)
+	if err != nil {
+		err = fmt.Errorf("invalid parameter: kind")
+		statusCode = http.StatusBadRequest
+	}
+	return kind, subKind, statusCode, err
+}

api/devtronResource/DevtronResourceRouter.go

@@ -0,0 +1,22 @@
+package devtronResource
+
+import "github.com/gorilla/mux"
+
+type DevtronResourceRouter interface {
+	InitDevtronResourceRouter(devtronResourceRouter *mux.Router)
+}
+
+type DevtronResourceRouterImpl struct {
+	historyRouter HistoryRouter
+}
+
+func NewDevtronResourceRouterImpl(historyRouter HistoryRouter) *DevtronResourceRouterImpl {
+	return &DevtronResourceRouterImpl{
+		historyRouter: historyRouter,
+	}
+}
+
+func (router *DevtronResourceRouterImpl) InitDevtronResourceRouter(devtronResourceRouter *mux.Router) {
+	historyRouter := devtronResourceRouter.PathPrefix("/history").Subrouter()
+	router.historyRouter.InitDtResourceHistoryRouter(historyRouter)
+}

api/devtronResource/bean/bean.go

@@ -0,0 +1,20 @@
+package bean
+
+type GetHistoryQueryParams struct {
+	FilterCriteria []string `schema:"filterCriteria"`
+	OffSet         int      `schema:"offSet"`
+	Limit          int      `schema:"limit"`
+}
+
+type GetHistoryConfigQueryParams struct {
+	BaseConfigurationId  int      `schema:"baseConfigurationId"`
+	HistoryComponent     string   `schema:"historyComponent"`
+	HistoryComponentName string   `schema:"historyComponentName"`
+	FilterCriteria       []string `schema:"filterCriteria"`
+}
+
+const (
+	RequestInvalidKindVersionErrMessage = "Invalid kind and version! Implementation not supported."
+	PathParamKind                       = "kind"
+	PathParamVersion                    = "version"
+)

api/devtronResource/wire_devtronResource.go

@@ -0,0 +1,30 @@
+package devtronResource
+
+import (
+	"github.com/devtron-labs/devtron/pkg/devtronResource"
+	"github.com/devtron-labs/devtron/pkg/devtronResource/history/deployment/cdPipeline"
+	"github.com/devtron-labs/devtron/pkg/devtronResource/read"
+	"github.com/devtron-labs/devtron/pkg/devtronResource/repository"
+	"github.com/google/wire"
+)
+
+var DevtronResourceWireSet = wire.NewSet(
+	//old bindings, migrated from wire.go
+	read.NewDevtronResourceSearchableKeyServiceImpl,
+	wire.Bind(new(read.DevtronResourceSearchableKeyService), new(*read.DevtronResourceSearchableKeyServiceImpl)),
+	repository.NewDevtronResourceSearchableKeyRepositoryImpl,
+	wire.Bind(new(repository.DevtronResourceSearchableKeyRepository), new(*repository.DevtronResourceSearchableKeyRepositoryImpl)),
+
+	NewDevtronResourceRouterImpl,
+	wire.Bind(new(DevtronResourceRouter), new(*DevtronResourceRouterImpl)),
+
+	NewHistoryRouterImpl,
+	wire.Bind(new(HistoryRouter), new(*HistoryRouterImpl)),
+	NewHistoryRestHandlerImpl,
+	wire.Bind(new(HistoryRestHandler), new(*HistoryRestHandlerImpl)),
+	cdPipeline.NewDeploymentHistoryServiceImpl,
+	wire.Bind(new(cdPipeline.DeploymentHistoryService), new(*cdPipeline.DeploymentHistoryServiceImpl)),
+
+	devtronResource.NewAPIReqDecoderServiceImpl,
+	wire.Bind(new(devtronResource.APIReqDecoderService), new(*devtronResource.APIReqDecoderServiceImpl)),
+)