Add feature: restore

This commit provides a feature: restore - which will be helpful to
restore data from a backup.
Some part of restoring solution require kubectl binary to be installed
and configured .kube/config file as well.
With that way, we don't need to copy the restore file/dir to the pod or
create new pod with new PV or mount same PVC into new pod, which might be
rejected by some PV drivers. Also mounting local host directory
to the OpenShift cluster might be prohibited in some
deployments (especially in public deployments where user is not an admin), so
that is not a good idea to use.

Change-Id: I7c44bb18dcbd55895df30d9ee5add1d6c42a7625

Author: danpawlik

cli/cmd/backup.go

@@ -23,6 +23,7 @@ package cmd
 import (
 	"errors"
 	"os"
+	"path/filepath"
 
 	cliutils "github.com/softwarefactory-project/sf-operator/cli/cmd/utils"
 	controllers "github.com/softwarefactory-project/sf-operator/controllers"
@@ -33,8 +34,11 @@ import (
 )
 
 const (
-	zuulBackupPod = "zuul-scheduler-0"
-	dbBackupPod   = "mariadb-0"
+	zuulBackupPod     = "zuul-scheduler-0"
+	dbBackupPod       = "mariadb-0"
+	DBBackupPath      = "mariadb/db-zuul.sql"
+	ZuulBackupPath    = "zuul/zuul.keys"
+	SecretsBackupPath = "secrets/"
 )
 
 // Short legend what to backup
@@ -49,17 +53,15 @@ const (
 //                This key is added as authorized keys on external system
 // - zuul-keystore-password - this is the key used to encrypt/decrypt key pairs stored into zookeeper
 // - zuul-auth-secret - this contains the secret for the zuul-client connection
-// - mariadb-root-password - this contains MariaDB root password
 
-var secretsToBackup = []string{
+var SecretsToBackup = []string{
 	"ca-cert",
 	"zookeeper-client-tls",
 	"zookeeper-server-tls",
 	"nodepool-builder-ssh-key",
 	"zuul-ssh-key",
 	"zuul-keystore-password",
 	"zuul-auth-secret",
-	"mariadb-root-password",
 }
 
 func prepareBackup(kmd *cobra.Command, backupDir string) (string, *kubernetes.Clientset, string) {
@@ -80,10 +82,10 @@ func prepareBackup(kmd *cobra.Command, backupDir string) (string, *kubernetes.Cl
 func createSecretBackup(ns string, backupDir string, kubeClientSet *kubernetes.Clientset) {
 	ctrl.Log.Info("Creating secrets backup...")
 
-	secretsDir := backupDir + "/secrets"
+	secretsDir := backupDir + "/" + SecretsBackupPath
 	cliutils.CreateDirectory(secretsDir, 0755)
 
-	for _, sec := range secretsToBackup {
+	for _, sec := range SecretsToBackup {
 		secret := cliutils.GetSecretByName(sec, ns, kubeClientSet)
 
 		// convert secret content to string (was bytes)
@@ -120,10 +122,12 @@ func createZuulKeypairBackup(ns string, backupDir string, kubeClientSet *kuberne
 
 	pod := cliutils.GetPodByName(zuulBackupPod, ns, kubeClientSet)
 
-	zuulBackupDir := backupDir + "/zuul/"
+	// https://zuul-ci.org/docs/zuul/latest/client.html
+	zuulBackupPath := backupDir + "/" + ZuulBackupPath
+	zuulBackupDir := filepath.Dir(zuulBackupPath)
 	cliutils.CreateDirectory(zuulBackupDir, 0755)
 	backupZuulCMD := []string{
-		"zuul",
+		"zuul-admin",
 		"export-keys",
 		"/tmp/zuul-backup",
 	}
@@ -143,7 +147,7 @@ func createZuulKeypairBackup(ns string, backupDir string, kubeClientSet *kuberne
 	commandBuffer := cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.ZuulSchedulerIdent, backupZuulPrintCMD)
 
 	// write stdout to file
-	cliutils.WriteContentToFile(zuulBackupDir+"zuul.keys", commandBuffer.Bytes(), 0640)
+	cliutils.WriteContentToFile(zuulBackupPath, commandBuffer.Bytes(), 0640)
 
 	// Remove key file from the pod
 	cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.ZuulSchedulerIdent, backupZuulRemoveCMD)
@@ -156,7 +160,9 @@ func createMySQLBackup(ns string, backupDir string, kubeClientSet *kubernetes.Cl
 	ctrl.Log.Info("Doing DB backup...")
 
 	// create MariaDB dir
-	mariaDBBackupDir := backupDir + "/mariadb/"
+	mariadbBackupPath := backupDir + "/" + DBBackupPath
+	mariaDBBackupDir := filepath.Dir(mariadbBackupPath)
+
 	cliutils.CreateDirectory(mariaDBBackupDir, 0755)
 
 	pod := cliutils.GetPodByName(dbBackupPod, ns, kubeClientSet)
@@ -174,7 +180,7 @@ func createMySQLBackup(ns string, backupDir string, kubeClientSet *kubernetes.Cl
 	commandBuffer := cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.MariaDBIdent, backupZuulCMD)
 
 	// write stdout to file
-	cliutils.WriteContentToFile(mariaDBBackupDir+"db-zuul.sql", commandBuffer.Bytes(), 0640)
+	cliutils.WriteContentToFile(mariadbBackupPath, commandBuffer.Bytes(), 0640)
 	ctrl.Log.Info("Finished doing DBs backup!")
 }
 

cli/cmd/restore.go

@@ -21,29 +21,190 @@ package cmd
 */
 
 import (
+	"context"
 	"errors"
+	"fmt"
 	"os"
+	"path/filepath"
+
+	cliutils "github.com/softwarefactory-project/sf-operator/cli/cmd/utils"
+	controllers "github.com/softwarefactory-project/sf-operator/controllers"
 
 	"github.com/spf13/cobra"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/kubernetes"
 	ctrl "sigs.k8s.io/controller-runtime"
 )
 
+func prepareRestore(kmd *cobra.Command) (string, *kubernetes.Clientset, string) {
+
+	cliCtx, err := cliutils.GetCLIContext(kmd)
+	if err != nil {
+		ctrl.Log.Error(err, "Error initializing CLI:")
+		os.Exit(1)
+	}
+
+	kubeContext := cliCtx.KubeContext
+	_, kubeClientSet := cliutils.GetClientset(kubeContext)
+	return cliCtx.Namespace, kubeClientSet, kubeContext
+}
+
+func restoreSecret(ns string, backupDir string, kubeContext string) {
+	ctrl.Log.Info("Restoring secrets...")
+
+	env := cliutils.ENV{
+		Cli: cliutils.CreateKubernetesClientOrDie(kubeContext),
+		Ctx: context.TODO(),
+		Ns:  ns,
+	}
+
+	for _, sec := range SecretsToBackup {
+		pathToSecret := backupDir + "/" + SecretsBackupPath + "/" + sec + ".yaml"
+		secretContent := cliutils.ReadYAMLToMapOrDie(pathToSecret)
+
+		var secret corev1.Secret
+		if cliutils.GetMOrDie(&env, sec, &secret) {
+			secretMap := secretContent["data"].(map[string]interface{})
+			for key, value := range secretMap {
+				stringValue, ok := value.(string)
+				if !ok {
+					ctrl.Log.Error(errors.New("can not convert secret data value to string"),
+						"Can not restore secret"+sec)
+					os.Exit(1)
+				}
+				secret.Data[key] = []byte(stringValue)
+			}
+		} else {
+			ctrl.Log.Error(errors.New("the secret does not exist"),
+				"The secret: "+sec+" should be available before continuing restore")
+			os.Exit(1)
+		}
+
+		cliutils.UpdateROrDie(&env, &secret)
+	}
+
+}
+
+func restoreDB(ns string, backupDir string, kubeClientSet *kubernetes.Clientset, kubeContext string) {
+	ctrl.Log.Info("Restoring DB...")
+	pod := cliutils.GetPodByName(dbBackupPod, ns, kubeClientSet)
+
+	kubectlPath := cliutils.GetKubectlPath()
+	dropDBCMD := []string{
+		"mysql",
+		"-e DROP DATABASE zuul;",
+	}
+	cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.MariaDBIdent, dropDBCMD)
+
+	mariadbBackupPath := backupDir + "/" + DBBackupPath
+
+	// Below command is executing something like:
+	//     cat backup/mariadb/db-zuul.sql | kubectl -n sf exec -it mariadb-0 -c mariadb -- sh -c "mysql -h0"
+	// but in that case, we need to do it via system kubernetes client.
+	executeCommand := fmt.Sprintf(
+		"cat %s | %s -n %s exec -it %s -c %s -- sh -c \"mysql -h0\"",
+		mariadbBackupPath, kubectlPath, ns, pod.Name, controllers.MariaDBIdent,
+	)
+
+	cliutils.ExecuteKubectlClient(ns, pod.Name, controllers.MariaDBIdent, executeCommand)
+
+	ctrl.Log.Info("Finished restoring DB from backup!")
+}
+func restoreZuul(ns string, backupDir string, kubeClientSet *kubernetes.Clientset, kubeContext string) {
+	ctrl.Log.Info("Restoring Zuul...")
+	pod := cliutils.GetPodByName(zuulBackupPod, ns, kubeClientSet)
+
+	// ensure that pod does not have any restore file
+	restoreZuulRemoveCMD := []string{
+		"rm",
+		"-rf",
+		"/tmp/zuul-import",
+	}
+	cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.ZuulSchedulerIdent, restoreZuulRemoveCMD)
+
+	// create empty directory for future restore
+	restoreZuulCreateDirCMD := []string{
+		"mkdir",
+		"-p",
+		"/tmp/zuul-import",
+	}
+	cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.ZuulSchedulerIdent, restoreZuulCreateDirCMD)
+
+	// copy the Zuul private keys backup to pod
+	// tar cf - -C /tmp/backup/zuul zuul.keys | /usr/bin/kubectl exec -i -n sf zuul-scheduler-0 -c zuul-scheduler -- tar xf -  -C /tmp
+	kubectlPath := cliutils.GetKubectlPath()
+	basePath := filepath.Dir(backupDir + "/" + ZuulBackupPath)
+	baseFile := filepath.Base(ZuulBackupPath)
+	executeCommand := fmt.Sprintf(
+		"tar cf - -C %s %s | %s exec -i -n %s %s -c %s -- tar xf - -C /tmp/zuul-import",
+		basePath, baseFile, kubectlPath, ns, pod.Name, controllers.ZuulSchedulerIdent,
+	)
+	ctrl.Log.Info("Executing " + executeCommand)
+
+	cliutils.ExecuteKubectlClient(ns, pod.Name, controllers.ZuulSchedulerIdent, executeCommand)
+
+	// https://zuul-ci.org/docs/zuul/latest/client.html
+	restoreZuulCMD := []string{
+		"zuul-admin",
+		"import-keys",
+		"--force",
+		"/tmp/zuul-import/" + baseFile,
+	}
+
+	// Execute command for restore
+	cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.ZuulSchedulerIdent, restoreZuulCMD)
+
+	// remove after all
+	cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.ZuulSchedulerIdent, restoreZuulRemoveCMD)
+
+	ctrl.Log.Info("Finished doing Zuul private keys restore!")
+
+}
+
 func restoreCmd(kmd *cobra.Command, args []string) {
-	err := errors.New("backup is not supported yet")
-	ctrl.Log.Error(err, "Command error")
-	os.Exit(1)
+
+	// NOTE: Solution for restoring DB and Zuul require kubectl binary to be installed and configured .kube/config
+	// file as well.
+	// With that way, we don't need to copy the restore file/dir to the pod or create new pod with new PV or
+	// mount same PVC into new pod, which might be rejected by some PV drivers. Also mounting local host directory
+	// to the OpenShift cluster might be prohibited in some deployments (especially in public deployments where
+	// user is not an admin), so that is not a good idea to use.
+
+	backupDir, _ := kmd.Flags().GetString("backup_dir")
+
+	if backupDir == "" {
+		ctrl.Log.Error(errors.New("not enough parameters"),
+			"The '--backup-dir' parameter needs to be set")
+		os.Exit(1)
+
+	}
+
+	// prepare to make restore
+	ns, kubeClientSet, kubeContext := prepareRestore(kmd)
+
+	if ns == "" {
+		ctrl.Log.Info("You did not specify the namespace!")
+		os.Exit(1)
+	}
+
+	restoreZuul(ns, backupDir, kubeClientSet, kubeContext)
+	restoreSecret(ns, backupDir, kubeContext)
+	restoreDB(ns, backupDir, kubeClientSet, kubeContext)
+
 }
 
 func MkRestoreCmd() *cobra.Command {
 
 	var (
+		backupDir  string
 		restoreCmd = &cobra.Command{
 			Use:   "restore",
 			Short: "Restore a deployment to a previous backup",
-			Long:  `This isn't implemented yet, this subcommand is a placeholder.`,
 			Run:   restoreCmd,
 		}
 	)
+	restoreCmd.Flags().StringVar(&backupDir, "backup_dir", "", "The path to the dir where backup is located")
 
 	return restoreCmd
 }

cli/cmd/utils/utils.go

@@ -22,6 +22,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"gopkg.in/yaml.v3"
 	"io/fs"
 	"os"
 	"os/exec"
@@ -381,29 +382,6 @@ func ConvertMapOfBytesToMapOfStrings(contentMap map[string][]byte) map[string]st
 	return strMap
 }
 
-func getSecrets(ns string, kubeClientSet *kubernetes.Clientset) *apiv1.SecretList {
-	secrets, err := kubeClientSet.CoreV1().Secrets(ns).List(context.TODO(), metav1.ListOptions{})
-	if err != nil {
-		ctrl.Log.Error(err, "Can not get secrets!")
-		os.Exit(1)
-	}
-	return secrets
-}
-
-func GetSecretValue(ns string, kubeClientSet *kubernetes.Clientset, secretName string) *string {
-	secrets := getSecrets(ns, kubeClientSet)
-	if secrets != nil && len(secrets.Items) > 0 {
-		for _, secret := range secrets.Items {
-			if secret.ObjectMeta.Name == secretName {
-				strMap := ConvertMapOfBytesToMapOfStrings(secret.Data)
-				secretValue := strMap[secretName]
-				return &secretValue
-			}
-		}
-	}
-	return nil
-}
-
 func GetClientset(kubeContext string) (*rest.Config, *kubernetes.Clientset) {
 	restConfig := controllers.GetConfigContextOrDie(kubeContext)
 	kubeClientset, err := kubernetes.NewForConfig(restConfig)
@@ -459,3 +437,40 @@ func GetSecretByName(secretName string, ns string, kubeClientSet *kubernetes.Cli
 	}
 	return secret
 }
+
+func ReadYAMLToMapOrDie(filePath string) map[string]interface{} {
+	readFile, _ := GetFileContent(filePath)
+	secretContent := make(map[string]interface{})
+	err := yaml.Unmarshal(readFile, &secretContent)
+	if err != nil {
+		ctrl.Log.Error(err, "Problem on reading the file content")
+	}
+	if len(secretContent) == 0 {
+		ctrl.Log.Error(errors.New("file is empty"), "The file is empty or it does not exist!")
+		os.Exit(1)
+	}
+	return secretContent
+}
+
+func GetKubectlPath() string {
+	kubectlPath, err := exec.LookPath("kubectl")
+	if err != nil {
+		ctrl.Log.Error(errors.New("no kubectl binary"),
+			"No 'kubectl' binary found. Please install the 'kubectl' binary before attempting a restore")
+		os.Exit(1)
+	}
+	return kubectlPath
+}
+
+func ExecuteKubectlClient(ns string, podName string, containerName string, executeCommand string) {
+	cmd := exec.Command("sh", "-c", executeCommand)
+	cmd.Stdin = os.Stdin
+	cmd.Stdout = os.Stdout
+
+	err := cmd.Run()
+	if err != nil {
+		ctrl.Log.Error(err, "There is an issue on executing command: "+executeCommand)
+		os.Exit(1)
+	}
+
+}

doc/operator/getting_started.md

@@ -14,7 +14,7 @@ In order to install the SF Operator on OpenShift, you will need:
 1. [OLM](https://olm.operatorframework.io/) running on your cluster. For most flavors of OpenShift [this is already the case](https://docs.openshift.com/container-platform/4.13/operators/understanding/olm/olm-understanding-olm.html#olm-overview_olm-understanding-olm).
 1. The community operators CatalogSource, to handle operator dependencies for SF-Operator. For most standard installations of OLM, [this CatalogSource is already installed](https://operatorhub.io/how-to-install-an-operator#How-do-I-get-Operator-Lifecycle-Manager?).
 1. A valid kubeconfig file, for a user with enough permissions to create a CatalogSource and a Subscription Custom Resources, on the `olm` and `operators` namespaces respectively.
-1. The [kubectl utility](https://kubernetes.io/docs/tasks/tools/#kubectl), to apply and create new resources on the OpenShift cluster.
+1. The [kubectl utility](https://kubernetes.io/docs/tasks/tools/#kubectl), to apply and create new resources on the OpenShift cluster. It is also required to use `restore` functionality.
 
 ## Installing the operator
 
@@ -97,4 +97,4 @@ TBD
 
 ## Next steps
 
- Your next step is to [deploy a Zuul-based CI with the operator](../deployment/getting_started.md).
+ Your next step is to [deploy a Zuul-based CI with the operator](../deployment/getting_started.md).