feat: gitops support for oci repositories (#5577)

* parsing requirements.yaml for oci

* wip

* fixing hard coded path in reference charts

* repo secret

* wip

* wip

* wire fix

* wip

* reverting ref chart changes

* wip: pr review changes

* refactoring

* adding back argocd assets

* refactoring changes

* common lib update

* wip

* fix after sanity

* common lib update

Author: iamayushm

Wire.go

@@ -115,6 +115,7 @@ import (
 	"github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange"
 	"github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/FullMode/resource"
 	"github.com/devtron-labs/devtron/pkg/appWorkflow"
+	"github.com/devtron-labs/devtron/pkg/argoRepositoryCreds"
 	"github.com/devtron-labs/devtron/pkg/asyncProvider"
 	"github.com/devtron-labs/devtron/pkg/attributes"
 	"github.com/devtron-labs/devtron/pkg/build"
@@ -989,6 +990,9 @@ func InitializeApp() (*App, error) {
 		common.NewDeploymentConfigServiceImpl,
 		wire.Bind(new(common.DeploymentConfigService), new(*common.DeploymentConfigServiceImpl)),
 
+		argoRepositoryCreds.NewRepositorySecret,
+		wire.Bind(new(argoRepositoryCreds.RepositorySecret), new(*argoRepositoryCreds.RepositorySecretImpl)),
+
 		repocreds.NewServiceClientImpl,
 		wire.Bind(new(repocreds.ServiceClient), new(*repocreds.ServiceClientImpl)),
 	)

cmd/external-app/wire.go

@@ -66,6 +66,7 @@ import (
 	repository4 "github.com/devtron-labs/devtron/pkg/appStore/chartGroup/repository"
 	"github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/EAMode"
 	"github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/FullMode/deployment"
+	"github.com/devtron-labs/devtron/pkg/argoRepositoryCreds"
 	"github.com/devtron-labs/devtron/pkg/attributes"
 	delete2 "github.com/devtron-labs/devtron/pkg/delete"
 	"github.com/devtron-labs/devtron/pkg/deployment/common"
@@ -244,6 +245,11 @@ func InitializeApp() (*App, error) {
 
 		common.NewDeploymentConfigServiceImpl,
 		wire.Bind(new(common.DeploymentConfigService), new(*common.DeploymentConfigServiceImpl)),
+
+		wire.Bind(new(util4.K8sService), new(*util4.K8sServiceImpl)),
+
+		argoRepositoryCreds.NewRepositorySecret,
+		wire.Bind(new(argoRepositoryCreds.RepositorySecret), new(*argoRepositoryCreds.RepositorySecretImpl)),
 	)
 	return &App{}, nil
 }

cmd/external-app/wire_gen.go

@@ -22,7 +22,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1
 	github.com/deckarep/golang-set v1.8.0
 	github.com/devtron-labs/authenticator v0.4.35-0.20240607135426-c86e868ecee1
-	github.com/devtron-labs/common-lib v0.0.25-0.20240802103040-a6b975ffa69e
+	github.com/devtron-labs/common-lib v0.0.25-0.20240808104611-3301505b68a7
 	github.com/devtron-labs/go-bitbucket v0.9.60-beta
 	github.com/devtron-labs/protos v0.0.3-0.20240802105333-92ee9bb85d80
 	github.com/evanphx/json-patch v5.7.0+incompatible

go.mod

@@ -197,8 +197,8 @@ github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzq
 github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/devtron-labs/authenticator v0.4.35-0.20240607135426-c86e868ecee1 h1:qdkpTAo2Kr0ZicZIVXfNwsGSshpc9OB9j9RzmKYdIwY=
 github.com/devtron-labs/authenticator v0.4.35-0.20240607135426-c86e868ecee1/go.mod h1:IkKPPEfgLCMR29he5yv2OCC6iM2R7K5/0AA3k8b9XNc=
-github.com/devtron-labs/common-lib v0.0.25-0.20240802103040-a6b975ffa69e h1:oC1KJ4jeIebSRWtBarETQPmSVhbK06EWAE49g9VukEY=
-github.com/devtron-labs/common-lib v0.0.25-0.20240802103040-a6b975ffa69e/go.mod h1:3GN9TABx4D+hVuF69vGYUUx+H8/WelcKw0lUt8aELok=
+github.com/devtron-labs/common-lib v0.0.25-0.20240808104611-3301505b68a7 h1:No5VWdDCPhX9rJP5FfKkJz2hMO64GddeEldSDqYE3WU=
+github.com/devtron-labs/common-lib v0.0.25-0.20240808104611-3301505b68a7/go.mod h1:3GN9TABx4D+hVuF69vGYUUx+H8/WelcKw0lUt8aELok=
 github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU=
 github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y=
 github.com/devtron-labs/protos v0.0.3-0.20240802105333-92ee9bb85d80 h1:xwbTeijNTf4/j1v+tSfwVqwLVnReas/NqEKeQHvSTys=

go.sum

@@ -963,7 +963,17 @@ func (impl *ChartGroupServiceImpl) performDeployStageOnAcd(installedAppVersion *
 		installedAppVersion.Status == appStoreBean.GIT_ERROR {
 		//step 2 git operation pull push
 		//TODO: save git Timeline here
-		appStoreGitOpsResponse, err := impl.fullModeDeploymentService.GenerateManifestAndPerformGitOperations(installedAppVersion)
+		appStoreAppVersion, err := impl.appStoreApplicationVersionRepository.FindById(installedAppVersion.AppStoreVersion)
+		if err != nil {
+			impl.logger.Errorw("fetching error", "err", err)
+			return nil, err
+		}
+		err = impl.fullModeDeploymentService.CreateArgoRepoSecretIfNeeded(appStoreAppVersion)
+		if err != nil {
+			impl.logger.Errorw("error in creating argo app repository secret", "appStoreApplicationVersionId", appStoreAppVersion.Id, "err", err)
+			return nil, err
+		}
+		appStoreGitOpsResponse, err := impl.fullModeDeploymentService.GenerateManifestAndPerformGitOperations(installedAppVersion, appStoreAppVersion)
 		if err != nil {
 			impl.logger.Errorw(" error", "err", err)
 			_, err = impl.appStoreDeploymentDBService.AppStoreDeployOperationStatusUpdate(installedAppVersion.InstalledAppId, appStoreBean.GIT_ERROR)

pkg/appStore/chartGroup/ChartGroupService.go

@@ -623,11 +623,6 @@ func (impl *AppStoreDeploymentDBServiceImpl) createAppForAppStore(createRequest
 func (impl *AppStoreDeploymentDBServiceImpl) validateAndGetOverrideDeploymentAppType(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, isGitOpsConfigured bool) (overrideDeploymentType string, err error) {
 	// initialise OverrideDeploymentType to the given DeploymentType
 	overrideDeploymentType = installAppVersionRequest.DeploymentAppType
-	appStoreAppVersion, err := impl.appStoreApplicationVersionRepository.FindById(installAppVersionRequest.AppStoreVersion)
-	if err != nil {
-		impl.logger.Errorw("error in fetching app store application version", "err", err)
-		return overrideDeploymentType, err
-	}
 
 	// virtual environments only supports Manifest Download
 	if installAppVersionRequest.Environment.IsVirtualEnvironment && util.IsManifestPush(installAppVersionRequest.DeploymentAppType) {
@@ -641,8 +636,7 @@ func (impl *AppStoreDeploymentDBServiceImpl) validateAndGetOverrideDeploymentApp
 	}
 
 	// OCI chart currently supports HELM installation only
-	isOCIRepo := appStoreAppVersion.AppStore.DockerArtifactStore != nil
-	if isOCIRepo || getAppInstallationMode(installAppVersionRequest.AppOfferingMode) == globalUtil.SERVER_MODE_HYPERION {
+	if getAppInstallationMode(installAppVersionRequest.AppOfferingMode) == globalUtil.SERVER_MODE_HYPERION {
 		overrideDeploymentType = util.PIPELINE_DEPLOYMENT_TYPE_HELM
 	} else {
 		overrideDeploymentType, err = impl.deploymentTypeOverrideService.ValidateAndOverrideDeploymentAppType(overrideDeploymentType, isGitOpsConfigured, installAppVersionRequest.EnvironmentId)

pkg/appStore/installedApp/service/AppStoreDeploymentDBService.go

@@ -164,11 +164,21 @@ func (impl *AppStoreDeploymentServiceImpl) InstallApp(installAppVersionRequest *
 
 	var gitOpsResponse *bean2.AppStoreGitOpsResponse
 	if installedAppDeploymentAction.PerformGitOps {
-		manifest, err := impl.fullModeDeploymentService.GenerateManifest(installAppVersionRequest)
+		appStoreAppVersion, err := impl.appStoreApplicationVersionRepository.FindById(installAppVersionRequest.AppStoreVersion)
+		if err != nil {
+			impl.logger.Errorw("fetching error", "err", err)
+			return nil, err
+		}
+		manifest, err := impl.fullModeDeploymentService.GenerateManifest(installAppVersionRequest, appStoreAppVersion)
 		if err != nil {
 			impl.logger.Errorw("error in performing manifest and git operations", "err", err)
 			return nil, err
 		}
+		err = impl.fullModeDeploymentService.CreateArgoRepoSecretIfNeeded(appStoreAppVersion)
+		if err != nil {
+			impl.logger.Errorw("error in creating argo app repository secret", "appStoreApplicationVersionId", appStoreAppVersion.Id, "err", err)
+			return nil, err
+		}
 		gitOpsResponse, err = impl.fullModeDeploymentService.GitOpsOperations(manifest, installAppVersionRequest)
 		if err != nil {
 			impl.logger.Errorw("error in doing gitops operation", "err", err)
@@ -729,12 +739,17 @@ func (impl *AppStoreDeploymentServiceImpl) UpdateInstalledApp(ctx context.Contex
 		// manifest contains ChartRepoName where the valuesConfig and requirementConfig files will get committed
 		// and that gitOpsRepoUrl is extracted from db inside GenerateManifest func and not from the current
 		// orchestrator cm prefix and appName.
-		manifest, err := impl.fullModeDeploymentService.GenerateManifest(upgradeAppRequest)
+		manifest, err := impl.fullModeDeploymentService.GenerateManifest(upgradeAppRequest, appStoreAppVersion)
 		if err != nil {
 			impl.logger.Errorw("error in generating manifest for helm apps", "err", err)
 			_ = impl.appStoreDeploymentDBService.UpdateInstalledAppVersionHistoryStatus(upgradeAppRequest.InstalledAppVersionHistoryId, pipelineConfig.WorkflowFailed)
 			return nil, err
 		}
+		err = impl.fullModeDeploymentService.CreateArgoRepoSecretIfNeeded(appStoreAppVersion)
+		if err != nil {
+			impl.logger.Errorw("error in creating argo app repository secret", "appStoreApplicationVersionId", appStoreAppVersion.Id, "err", err)
+			return nil, err
+		}
 		// required if gitOps repo name is changed, gitOps repo name will change if env variable which we use as suffix changes
 		monoRepoMigrationRequired = impl.checkIfMonoRepoMigrationRequired(installedApp, deploymentConfig)
 		argocdAppName := installedApp.App.AppName + "-" + installedApp.Environment.Name

pkg/appStore/installedApp/service/AppStoreDeploymentService.go

@@ -414,11 +414,11 @@ func (impl *EAModeDeploymentServiceImpl) GitOpsOperations(manifestResponse *bean
 	return nil, errors.New("this is not implemented")
 }
 
-func (impl *EAModeDeploymentServiceImpl) GenerateManifestAndPerformGitOperations(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*bean.AppStoreGitOpsResponse, error) {
+func (impl *EAModeDeploymentServiceImpl) GenerateManifestAndPerformGitOperations(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, appStoreApplicationVersion *appStoreDiscoverRepository.AppStoreApplicationVersion) (*bean.AppStoreGitOpsResponse, error) {
 	return nil, errors.New("this is not implemented")
 }
 
-func (impl *EAModeDeploymentServiceImpl) GenerateManifest(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (manifestResponse *bean.AppStoreManifestResponse, err error) {
+func (impl *EAModeDeploymentServiceImpl) GenerateManifest(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, appStoreApplicationVersion *appStoreDiscoverRepository.AppStoreApplicationVersion) (manifestResponse *bean.AppStoreManifestResponse, err error) {
 	return nil, errors.New("this is not implemented")
 }
 
@@ -445,3 +445,7 @@ func (impl *EAModeDeploymentServiceImpl) DeleteACD(acdAppName string, ctx contex
 func (impl *EAModeDeploymentServiceImpl) GetAcdAppGitOpsRepoURL(appName string, environmentName string) (string, error) {
 	return "", errors.New("this is not implemented")
 }
+
+func (impl *EAModeDeploymentServiceImpl) CreateArgoRepoSecretIfNeeded(appStoreApplicationVersion *appStoreDiscoverRepository.AppStoreApplicationVersion) error {
+	return errors.New("this is not implemented")
+}

pkg/appStore/installedApp/service/EAMode/EAModeDeploymentService.go

@@ -25,6 +25,7 @@ import (
 	client "github.com/devtron-labs/devtron/api/helm-app/service"
 	"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus"
 	"github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/common"
+	"github.com/devtron-labs/devtron/pkg/argoRepositoryCreds"
 	repository5 "github.com/devtron-labs/devtron/pkg/cluster/repository"
 	"github.com/devtron-labs/devtron/pkg/deployment/common"
 	commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean"
@@ -102,6 +103,7 @@ type FullModeDeploymentServiceImpl struct {
 	environmentRepository                repository5.EnvironmentRepository
 	deploymentConfigService              common.DeploymentConfigService
 	chartTemplateService                 util.ChartTemplateService
+	RepositorySecretService              argoRepositoryCreds.RepositorySecret
 }
 
 func NewFullModeDeploymentServiceImpl(
@@ -127,7 +129,8 @@ func NewFullModeDeploymentServiceImpl(
 	gitOpsValidationService validation.GitOpsValidationService,
 	environmentRepository repository5.EnvironmentRepository,
 	deploymentConfigService common.DeploymentConfigService,
-	chartTemplateService util.ChartTemplateService) *FullModeDeploymentServiceImpl {
+	chartTemplateService util.ChartTemplateService,
+	RepositorySecretService argoRepositoryCreds.RepositorySecret) *FullModeDeploymentServiceImpl {
 	return &FullModeDeploymentServiceImpl{
 		Logger:                               logger,
 		acdClient:                            acdClient,
@@ -152,6 +155,7 @@ func NewFullModeDeploymentServiceImpl(
 		environmentRepository:                environmentRepository,
 		deploymentConfigService:              deploymentConfigService,
 		chartTemplateService:                 chartTemplateService,
+		RepositorySecretService:              RepositorySecretService,
 	}
 }