Feat: Provide the ability to exclude dqlite and clickhouse (timeseries) data collection

Author: mrajagopal

cmd/nginx-supportpkg.go

@@ -33,6 +33,8 @@ func Execute() {
 
 	var namespaces []string
 	var product string
+	var excludeDBData bool
+	var excludeTimeSeriesData bool
 	var jobList []jobs.Job
 
 	var rootCmd = &cobra.Command{
@@ -47,6 +49,13 @@ func Execute() {
 				os.Exit(1)
 			}
 
+			if excludeDBData {
+				collector.ExcludeDBData = true
+			}
+			if excludeTimeSeriesData {
+				collector.ExcludeTimeSeriesData = true
+			}
+
 			collector.Logger.Printf("Starting kubectl-nginx-supportpkg - version: %s - build: %s", version.Version, version.Build)
 			collector.Logger.Printf("Input args are %v", os.Args)
 
@@ -68,8 +77,10 @@ func Execute() {
 				failedJobs := 0
 				for _, job := range jobList {
 					fmt.Printf("Running job %s...", job.Name)
-					err = job.Collect(collector)
-					if err != nil {
+					err, Skipped := job.Collect(collector)
+					if Skipped {
+						fmt.Print(" SKIPPED\n")
+					} else if err != nil {
 						fmt.Printf(" Error: %s\n", err)
 						failedJobs++
 					} else {
@@ -108,6 +119,9 @@ func Execute() {
 		os.Exit(1)
 	}
 
+	rootCmd.Flags().BoolVarP(&excludeDBData, "exclude-db-data", "d", false, "exclude DB data collection")
+	rootCmd.Flags().BoolVarP(&excludeTimeSeriesData, "exclude-time-series-data", "t", false, "exclude time series data collection")
+
 	versionStr := "nginx-supportpkg - version: " + version.Version + " - build: " + version.Build + "\n"
 	rootCmd.SetVersionTemplate(versionStr)
 	rootCmd.Version = versionStr
@@ -118,7 +132,8 @@ func Execute() {
 			"\n nginx-supportpkg -h|--help" +
 			"\n nginx-supportpkg -v|--version" +
 			"\n nginx-supportpkg [-n|--namespace] ns1 [-n|--namespace] ns2 [-p|--product] [nic,ngf,ngx,nim]" +
-			"\n nginx-supportpkg [-n|--namespace] ns1,ns2 [-p|--product] [nic,ngf,ngx,nim] \n")
+			"\n nginx-supportpkg [-n|--namespace] ns1,ns2 [-p|--product] [nic,ngf,ngx,nim]" +
+			"\n nginx-supportpkg [-n|--namespace] ns1 [-n|--namespace] ns2 [-p|--product] [nim] [-d|--exclude-db-data] [-t|--exclude-time-series-data] \n")
 
 	if err := rootCmd.Execute(); err != nil {
 		fmt.Println(err)

pkg/data_collector/data_collector.go

@@ -47,15 +47,17 @@ import (
 )
 
 type DataCollector struct {
-	BaseDir             string
-	Namespaces          []string
-	Logger              *log.Logger
-	LogFile             *os.File
-	K8sRestConfig       *rest.Config
-	K8sCoreClientSet    *kubernetes.Clientset
-	K8sCrdClientSet     *crdClient.Clientset
-	K8sMetricsClientSet *metricsClient.Clientset
-	K8sHelmClientSet    map[string]helmClient.Client
+	BaseDir               string
+	Namespaces            []string
+	Logger                *log.Logger
+	LogFile               *os.File
+	K8sRestConfig         *rest.Config
+	K8sCoreClientSet      *kubernetes.Clientset
+	K8sCrdClientSet       *crdClient.Clientset
+	K8sMetricsClientSet   *metricsClient.Clientset
+	K8sHelmClientSet      map[string]helmClient.Client
+	ExcludeDBData         bool
+	ExcludeTimeSeriesData bool
 }
 
 func NewDataCollector(namespaces ...string) (*DataCollector, error) {

pkg/jobs/job.go

@@ -20,12 +20,12 @@ package jobs
 
 import (
 	"context"
-	"errors"
 	"fmt"
-	"github.com/nginxinc/nginx-k8s-supportpkg/pkg/data_collector"
 	"os"
 	"path/filepath"
 	"time"
+
+	"github.com/nginxinc/nginx-k8s-supportpkg/pkg/data_collector"
 )
 
 type Job struct {
@@ -35,11 +35,12 @@ type Job struct {
 }
 
 type JobResult struct {
-	Files map[string][]byte
-	Error error
+	Files   map[string][]byte
+	Error   error
+	Skipped bool
 }
 
-func (j Job) Collect(dc *data_collector.DataCollector) error {
+func (j Job) Collect(dc *data_collector.DataCollector) (error, bool) {
 	ch := make(chan JobResult, 1)
 
 	ctx, cancel := context.WithTimeout(context.Background(), j.Timeout)
@@ -51,28 +52,29 @@ func (j Job) Collect(dc *data_collector.DataCollector) error {
 	select {
 	case <-ctx.Done():
 		dc.Logger.Printf("\tJob %s has timed out: %s\n---\n", j.Name, ctx.Err())
-		return errors.New(fmt.Sprintf("Context cancelled: %v", ctx.Err()))
+		err := fmt.Errorf("Context cancelled: %v", ctx.Err())
+		return err, false
 
 	case jobResults := <-ch:
 		if jobResults.Error != nil {
 			dc.Logger.Printf("\tJob %s has failed: %s\n", j.Name, jobResults.Error)
-			return jobResults.Error
+			return jobResults.Error, jobResults.Skipped
 		}
 
 		for fileName, fileValue := range jobResults.Files {
 			err := os.MkdirAll(filepath.Dir(fileName), os.ModePerm)
 			if err != nil {
-				return fmt.Errorf("MkdirAll failed: %v", err)
+				return fmt.Errorf("MkdirAll failed: %v", err), jobResults.Skipped
 			}
 			file, _ := os.Create(fileName)
 			_, err = file.Write(fileValue)
 			if err != nil {
-				return fmt.Errorf("Write failed: %v", err)
+				return fmt.Errorf("Write failed: %v", err), jobResults.Skipped
 			}
 			_ = file.Close()
 			dc.Logger.Printf("\tJob %s wrote %d bytes to %s\n", j.Name, len(fileValue), fileName)
 		}
 		dc.Logger.Printf("\tJob %s completed successfully\n---\n", j.Name)
-		return nil
+		return nil, jobResults.Skipped
 	}
 }

pkg/jobs/nim_job_list.go

@@ -151,7 +151,14 @@ func NIMJobList() []Job {
 			Timeout: time.Second * 600,
 			Execute: func(dc *data_collector.DataCollector, ctx context.Context, ch chan JobResult) {
 				jobResult := JobResult{Files: make(map[string][]byte), Error: nil}
-				// command := []string{"clickhouse-client", "--database", "nms", "-q", "SHOW CREATE TABLE nms.events"}
+
+				if dc.ExcludeTimeSeriesData {
+					dc.Logger.Printf("\tSkipping clickhouse data dump as ExcludeTimeSeriesData is set to true\n")
+					jobResult.Skipped = true
+					ch <- jobResult
+					return
+				}
+
 				for _, namespace := range dc.Namespaces {
 					pods, err := dc.K8sCoreClientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
 					if err != nil {
@@ -191,6 +198,14 @@ func NIMJobList() []Job {
 			Execute: func(dc *data_collector.DataCollector, ctx context.Context, ch chan JobResult) {
 				jobResult := JobResult{Files: make(map[string][]byte), Error: nil}
 
+				if dc.ExcludeDBData {
+					dc.Logger.Printf("\tSkipping dqlite dump as ExcludeDBData is set to true\n")
+					jobResult.Skipped = true
+					ch <- jobResult
+					return
+				}
+				// Check if ExcludeDBData is set to true, skip dump if so
+				// (already handled above with dc.ExcludeDBData)
 				dbConfigs := []struct {
 					dbName        string
 					containerName string