Nim integration

cmd/nginx-supportpkg.go

@@ -31,17 +31,17 @@ import (
 
 func Execute() {
 
-	var namespaces []string
 	var product string
 	var jobList []jobs.Job
+	collector := data_collector.DataCollector{}
 
 	var rootCmd = &cobra.Command{
 		Use:   "nginx-supportpkg",
 		Short: "nginx-supportpkg - a tool to create Ingress Controller diagnostics package",
 		Long:  `nginx-supportpkg - a tool to create Ingress Controller diagnostics package`,
 		Run: func(cmd *cobra.Command, args []string) {
 
-			collector, err := data_collector.NewDataCollector(namespaces...)
+			err := data_collector.NewDataCollector(&collector)
 			if err != nil {
 				fmt.Println(fmt.Errorf("unable to start data collector: %s", err))
 				os.Exit(1)
@@ -57,21 +57,25 @@ func Execute() {
 				jobList = slices.Concat(jobs.CommonJobList(), jobs.NGFJobList())
 			case "ngx":
 				jobList = slices.Concat(jobs.CommonJobList(), jobs.NGXJobList())
+			case "nim":
+				jobList = slices.Concat(jobs.CommonJobList(), jobs.NIMJobList())
 			default:
-				fmt.Printf("Error: product must be in the following list: [nic, ngf, ngx]\n")
+				fmt.Printf("Error: product must be in the following list: [nic, ngf, ngx, nim]\n")
 				os.Exit(1)
 			}
 
 			if collector.AllNamespacesExist() {
 				failedJobs := 0
 				for _, job := range jobList {
 					fmt.Printf("Running job %s...", job.Name)
-					err = job.Collect(collector)
-					if err != nil {
-						fmt.Printf(" Error: %s\n", err)
+					err, Skipped := job.Collect(&collector)
+					if Skipped {
+						fmt.Print(" SKIPPED\n")
+					} else if err != nil {
+						fmt.Printf(" FAILED: %s\n", err)
 						failedJobs++
 					} else {
-						fmt.Print(" OK\n")
+						fmt.Print(" COMPLETED\n")
 					}
 				}
 
@@ -94,7 +98,7 @@ func Execute() {
 		},
 	}
 
-	rootCmd.Flags().StringSliceVarP(&namespaces, "namespace", "n", []string{}, "list of namespaces to collect information from")
+	rootCmd.Flags().StringSliceVarP(&collector.Namespaces, "namespace", "n", []string{}, "list of namespaces to collect information from")
 	if err := rootCmd.MarkFlagRequired("namespace"); err != nil {
 		fmt.Println(err)
 		os.Exit(1)
@@ -106,6 +110,9 @@ func Execute() {
 		os.Exit(1)
 	}
 
+	rootCmd.Flags().BoolVarP(&collector.ExcludeDBData, "exclude-db-data", "d", false, "exclude DB data collection")
+	rootCmd.Flags().BoolVarP(&collector.ExcludeTimeSeriesData, "exclude-time-series-data", "t", false, "exclude time series data collection")
+
 	versionStr := "nginx-supportpkg - version: " + version.Version + " - build: " + version.Build + "\n"
 	rootCmd.SetVersionTemplate(versionStr)
 	rootCmd.Version = versionStr
@@ -115,8 +122,9 @@ func Execute() {
 			"Usage:" +
 			"\n nginx-supportpkg -h|--help" +
 			"\n nginx-supportpkg -v|--version" +
-			"\n nginx-supportpkg [-n|--namespace] ns1 [-n|--namespace] ns2 [-p|--product] [nic,ngf,ngx]" +
-			"\n nginx-supportpkg [-n|--namespace] ns1,ns2 [-p|--product] [nic,ngf,ngx] \n")
+			"\n nginx-supportpkg [-n|--namespace] ns1 [-n|--namespace] ns2 [-p|--product] [nic,ngf,ngx,nim]" +
+			"\n nginx-supportpkg [-n|--namespace] ns1,ns2 [-p|--product] [nic,ngf,ngx,nim]" +
+			"\n nginx-supportpkg [-n|--namespace] ns1 [-n|--namespace] ns2 [-p|--product] [nim] [-d|--exclude-db-data] [-t|--exclude-time-series-data] \n")
 
 	if err := rootCmd.Execute(); err != nil {
 		fmt.Println(err)

pkg/data_collector/data_collector.go

@@ -24,9 +24,15 @@ import (
 	"compress/gzip"
 	"context"
 	"fmt"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"strconv"
+	"time"
+
 	helmClient "github.com/mittwald/go-helm-client"
 	"github.com/nginxinc/nginx-k8s-supportpkg/pkg/crds"
-	"io"
 	corev1 "k8s.io/api/core/v1"
 	crdClient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -38,35 +44,32 @@ import (
 	"k8s.io/client-go/tools/remotecommand"
 	"k8s.io/client-go/util/homedir"
 	metricsClient "k8s.io/metrics/pkg/client/clientset/versioned"
-	"log"
-	"os"
-	"path/filepath"
-	"strconv"
-	"time"
 )
 
 type DataCollector struct {
-	BaseDir             string
-	Namespaces          []string
-	Logger              *log.Logger
-	LogFile             *os.File
-	K8sRestConfig       *rest.Config
-	K8sCoreClientSet    *kubernetes.Clientset
-	K8sCrdClientSet     *crdClient.Clientset
-	K8sMetricsClientSet *metricsClient.Clientset
-	K8sHelmClientSet    map[string]helmClient.Client
+	BaseDir               string
+	Namespaces            []string
+	Logger                *log.Logger
+	LogFile               *os.File
+	K8sRestConfig         *rest.Config
+	K8sCoreClientSet      *kubernetes.Clientset
+	K8sCrdClientSet       *crdClient.Clientset
+	K8sMetricsClientSet   *metricsClient.Clientset
+	K8sHelmClientSet      map[string]helmClient.Client
+	ExcludeDBData         bool
+	ExcludeTimeSeriesData bool
 }
 
-func NewDataCollector(namespaces ...string) (*DataCollector, error) {
+func NewDataCollector(collector *DataCollector) error {
 
 	tmpDir, err := os.MkdirTemp("", "-pkg-diag")
 	if err != nil {
-		return nil, fmt.Errorf("unable to create temp directory: %s", err)
+		return fmt.Errorf("unable to create temp directory: %s", err)
 	}
 
 	logFile, err := os.OpenFile(filepath.Join(tmpDir, "supportpkg.log"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
 	if err != nil {
-		return nil, fmt.Errorf("unable to create log file: %s", err)
+		return fmt.Errorf("unable to create log file: %s", err)
 	}
 
 	// Find config
@@ -77,30 +80,27 @@ func NewDataCollector(namespaces ...string) (*DataCollector, error) {
 	config, err := clientcmd.BuildConfigFromFlags("", kubeConfig)
 
 	if err != nil {
-		return nil, fmt.Errorf("unable to connect to k8s using file %s: %s", kubeConfig, err)
-	}
-
-	dc := DataCollector{
-		BaseDir:          tmpDir,
-		Namespaces:       namespaces,
-		LogFile:          logFile,
-		Logger:           log.New(logFile, "", log.LstdFlags|log.LUTC|log.Lmicroseconds|log.Lshortfile),
-		K8sHelmClientSet: make(map[string]helmClient.Client),
+		return fmt.Errorf("unable to connect to k8s using file %s: %s", kubeConfig, err)
 	}
+	// Set up the DataCollector options
+	collector.BaseDir = tmpDir
+	collector.LogFile = logFile
+	collector.Logger = log.New(logFile, "", log.LstdFlags|log.LUTC|log.Lmicroseconds|log.Lshortfile)
+	collector.K8sHelmClientSet = make(map[string]helmClient.Client)
 
 	//Initialize clients
-	dc.K8sRestConfig = config
-	dc.K8sCoreClientSet, _ = kubernetes.NewForConfig(config)
-	dc.K8sCrdClientSet, _ = crdClient.NewForConfig(config)
-	dc.K8sMetricsClientSet, _ = metricsClient.NewForConfig(config)
-	for _, namespace := range dc.Namespaces {
-		dc.K8sHelmClientSet[namespace], _ = helmClient.NewClientFromRestConf(&helmClient.RestConfClientOptions{
+	collector.K8sRestConfig = config
+	collector.K8sCoreClientSet, _ = kubernetes.NewForConfig(config)
+	collector.K8sCrdClientSet, _ = crdClient.NewForConfig(config)
+	collector.K8sMetricsClientSet, _ = metricsClient.NewForConfig(config)
+	for _, namespace := range collector.Namespaces {
+		collector.K8sHelmClientSet[namespace], _ = helmClient.NewClientFromRestConf(&helmClient.RestConfClientOptions{
 			Options:    &helmClient.Options{Namespace: namespace},
 			RestConfig: config,
 		})
 	}
 
-	return &dc, nil
+	return nil
 }
 
 func (c *DataCollector) WrapUp(product string) (string, error) {
@@ -266,3 +266,68 @@ func (c *DataCollector) AllNamespacesExist() bool {
 
 	return allExist
 }
+
+// CopyFileFromPod copies a file from a pod's container to the local filesystem.
+func (c *DataCollector) CopyFileFromPod(namespace, pod, container, srcPath, destPath string, ctx context.Context) error {
+	cmd := []string{"tar", "cf", "-", "-C", filepath.Dir(srcPath), filepath.Base(srcPath)}
+	req := c.K8sCoreClientSet.CoreV1().RESTClient().Post().
+		Namespace(namespace).
+		Resource("pods").
+		Name(pod).
+		SubResource("exec").
+		VersionedParams(&corev1.PodExecOptions{
+			Container: container,
+			Command:   cmd,
+			Stdin:     false,
+			Stdout:    true,
+			Stderr:    true,
+			TTY:       false,
+		}, scheme.ParameterCodec)
+
+	exec, err := remotecommand.NewSPDYExecutor(c.K8sRestConfig, "POST", req.URL())
+	if err != nil {
+		return err
+	}
+
+	// Stream the data from the Pod
+	var stdout, stderr bytes.Buffer
+	err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{
+		Stdout: &stdout,
+		Stderr: &stderr,
+	})
+	if err != nil {
+		return err
+	}
+
+	// Create a local file to save the output
+	localFile, err := os.Create(destPath)
+	if err != nil {
+		// return fmt.Errorf("failed to create local file: %w", err)
+		return err
+	}
+	defer localFile.Close()
+
+	// Untar the stream and write the content to the local file
+	tarReader := tar.NewReader(&stdout)
+	for {
+		header, err := tarReader.Next()
+
+		if err == io.EOF {
+			break // End of tar archive
+		}
+		if err != nil {
+			return err
+		}
+
+		// Ensure the tar file matches the expected file path
+		if header.Name == filepath.Base(srcPath) {
+			_, err = io.Copy(localFile, tarReader)
+			if err != nil {
+				return fmt.Errorf("failed to write to local file: %w", err)
+			}
+			break
+		}
+	}
+
+	return nil
+}

pkg/jobs/common_job_list.go

@@ -23,12 +23,13 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/nginxinc/nginx-k8s-supportpkg/pkg/data_collector"
 	"io"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"path/filepath"
 	"time"
+
+	"github.com/nginxinc/nginx-k8s-supportpkg/pkg/data_collector"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 func CommonJobList() []Job {
@@ -88,6 +89,91 @@ func CommonJobList() []Job {
 				ch <- jobResult
 			},
 		},
+		{
+			Name:    "pv-list",
+			Timeout: time.Second * 10,
+			Execute: func(dc *data_collector.DataCollector, ctx context.Context, ch chan JobResult) {
+				jobResult := JobResult{Files: make(map[string][]byte), Error: nil}
+				for _, namespace := range dc.Namespaces {
+					result, err := dc.K8sCoreClientSet.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{})
+					if err != nil {
+						dc.Logger.Printf("\tCould not retrieve persistent volumes list %s: %v\n", namespace, err)
+					} else {
+						jsonResult, _ := json.MarshalIndent(result, "", "  ")
+						jobResult.Files[filepath.Join(dc.BaseDir, "resources", namespace, "persistentvolumes.json")] = jsonResult
+					}
+				}
+				ch <- jobResult
+			},
+		},
+		{
+			Name:    "pvc-list",
+			Timeout: time.Second * 10,
+			Execute: func(dc *data_collector.DataCollector, ctx context.Context, ch chan JobResult) {
+				jobResult := JobResult{Files: make(map[string][]byte), Error: nil}
+				for _, namespace := range dc.Namespaces {
+					result, err := dc.K8sCoreClientSet.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{})
+					if err != nil {
+						dc.Logger.Printf("\tCould not retrieve persistent volume claims list %s: %v\n", namespace, err)
+					} else {
+						jsonResult, _ := json.MarshalIndent(result, "", "  ")
+						jobResult.Files[filepath.Join(dc.BaseDir, "resources", namespace, "persistentvolumeclaims.json")] = jsonResult
+					}
+				}
+				ch <- jobResult
+			},
+		},
+		{
+			Name:    "sc-list",
+			Timeout: time.Second * 10,
+			Execute: func(dc *data_collector.DataCollector, ctx context.Context, ch chan JobResult) {
+				jobResult := JobResult{Files: make(map[string][]byte), Error: nil}
+				for _, namespace := range dc.Namespaces {
+					result, err := dc.K8sCoreClientSet.StorageV1().StorageClasses().List(ctx, metav1.ListOptions{})
+					if err != nil {
+						dc.Logger.Printf("\tCould not retrieve storage classes list %s: %v\n", namespace, err)
+					} else {
+						jsonResult, _ := json.MarshalIndent(result, "", "  ")
+						jobResult.Files[filepath.Join(dc.BaseDir, "resources", namespace, "storageclasses.json")] = jsonResult
+					}
+				}
+				ch <- jobResult
+			},
+		},
+		{
+			Name:    "apiresources-list",
+			Timeout: time.Second * 10,
+			Execute: func(dc *data_collector.DataCollector, ctx context.Context, ch chan JobResult) {
+				jobResult := JobResult{Files: make(map[string][]byte), Error: nil}
+				for _, namespace := range dc.Namespaces {
+					result, err := dc.K8sCoreClientSet.DiscoveryClient.ServerPreferredResources()
+					if err != nil {
+						dc.Logger.Printf("\tCould not retrieve API resources list %s: %v\n", namespace, err)
+					} else {
+						jsonResult, _ := json.MarshalIndent(result, "", "  ")
+						jobResult.Files[filepath.Join(dc.BaseDir, "resources", namespace, "apiresources.json")] = jsonResult
+					}
+				}
+				ch <- jobResult
+			},
+		},
+		{
+			Name:    "apiversions-list",
+			Timeout: time.Second * 10,
+			Execute: func(dc *data_collector.DataCollector, ctx context.Context, ch chan JobResult) {
+				jobResult := JobResult{Files: make(map[string][]byte), Error: nil}
+				for _, namespace := range dc.Namespaces {
+					result, err := dc.K8sCoreClientSet.DiscoveryClient.ServerGroups()
+					if err != nil {
+						dc.Logger.Printf("\tCould not retrieve API versions list %s: %v\n", namespace, err)
+					} else {
+						jsonResult, _ := json.MarshalIndent(result, "", "  ")
+						jobResult.Files[filepath.Join(dc.BaseDir, "resources", namespace, "apiversions.json")] = jsonResult
+					}
+				}
+				ch <- jobResult
+			},
+		},
 		{
 			Name:    "events-list",
 			Timeout: time.Second * 10,