nydusify chunkdict generate --sources

Add the 'nydus-image chunkdict save' command
with the "--sources" followed by the nydus image of registry
(e.g.,'registry.com/busybox:nydus-v1,registry.com/busybox:nydus-v2')

Signed-off-by: Zhao Yuan <1627990440@qq.com>

Author: newthifans

contrib/nydusify/cmd/nydusify.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/checker"
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/checker/rule"
+	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/chunkdict/generator"
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/converter"
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/packer"
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/provider"
@@ -639,6 +640,68 @@ func main() {
 				return checker.Check(context.Background())
 			},
 		},
+		{
+			Name:  "chunkdict",
+			Usage: "Deduplicate chunk for Nydus image (experimental)",
+			Subcommands: []*cli.Command{
+				{
+					Name:  "generate",
+					Usage: "Save chunk and blob information of Multi-image into the database (experimental)",
+					Flags: []cli.Flag{
+						&cli.StringSliceFlag{
+							Name:     "sources",
+							Required: true,
+							Usage:    "One or more Nydus image reference(Multiple images should be split by commas)",
+							EnvVars:  []string{"SOURCES"},
+						},
+						&cli.BoolFlag{
+							Name:     "source-insecure",
+							Required: false,
+							Usage:    "Skip verifying server certs for HTTPS source registry",
+							EnvVars:  []string{"SOURCE_INSECURE"},
+						},
+						&cli.StringFlag{
+							Name:    "work-dir",
+							Value:   "./output",
+							Usage:   "Working directory for generating chunkdict image",
+							EnvVars: []string{"WORK_DIR"},
+						},
+						&cli.StringFlag{
+							Name:    "nydus-image",
+							Value:   "nydus-image",
+							Usage:   "Path to the nydus-image binary, default to search in PATH",
+							EnvVars: []string{"NYDUS_IMAGE"},
+						},
+						&cli.StringFlag{
+							Name:  "platform",
+							Value: "linux/" + runtime.GOARCH,
+							Usage: "Specify platform identifier to choose image manifest, possible values: 'linux/amd64' and 'linux/arm64'",
+						},
+					},
+					Action: func(c *cli.Context) error {
+						setupLogLevel(c)
+
+						_, arch, err := provider.ExtractOsArch(c.String("platform"))
+						if err != nil {
+							return err
+						}
+
+						generator, err := generator.New(generator.Opt{
+							WorkDir:        c.String("work-dir"),
+							Sources:        c.StringSlice("sources"),
+							SourceInsecure: c.Bool("source-insecure"),
+							NydusImagePath: c.String("nydus-image"),
+							ExpectedArch:   arch,
+						})
+						if err != nil {
+							return err
+						}
+
+						return generator.Generate(context.Background())
+					},
+				},
+			},
+		},
 		{
 			Name:    "mount",
 			Aliases: []string{"view"},

contrib/nydusify/pkg/build/builder.go

@@ -1,4 +1,4 @@
-// Copyright 2020 Ant Group. All rights reserved.
+// Copyright 2023 Nydus Developers. All rights reserved.
 //
 // SPDX-License-Identifier: Apache-2.0
 
@@ -41,6 +41,10 @@ type CompactOption struct {
 	CompactConfigPath   string
 }
 
+type SaveOption struct {
+	BootstrapPath string
+}
+
 type Builder struct {
 	binaryPath string
 	stdout     io.Writer
@@ -143,3 +147,16 @@ func (builder *Builder) Run(option BuilderOption) error {
 
 	return builder.run(args, option.PrefetchPatterns)
 }
+
+// Save calls `nydus-image chunkdict save` to parse Nydus bootstrap
+func (builder *Builder) Save(option SaveOption) error {
+	args := []string{
+		"chunkdict",
+		"save",
+		"--log-level",
+		"warn",
+		"--bootstrap",
+		option.BootstrapPath,
+	}
+	return builder.run(args, "")
+}

contrib/nydusify/pkg/chunkdict/generator/generator.go

@@ -0,0 +1,106 @@
+package generator
+
+import (
+	"context"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+
+	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/build"
+	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/parser"
+	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/provider"
+	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils"
+)
+
+// Opt defines Chunkdict generate options.
+// Note: sources is one or more Nydus image references.
+type Opt struct {
+	WorkDir        string
+	Sources        []string
+	SourceInsecure bool
+	NydusImagePath string
+	ExpectedArch   string
+}
+
+// Generator generates chunkdict by deduplicating multiple nydus images
+// invoking "nydus-image chunkdict save" to save image information into database.
+type Generator struct {
+	Opt
+	sourcesParser []*parser.Parser
+}
+
+// New creates Generator instance.
+func New(opt Opt) (*Generator, error) {
+	// TODO: support sources image resolver
+	var sourcesParser []*parser.Parser
+	for _, source := range opt.Sources {
+		sourcesRemote, err := provider.DefaultRemote(source, opt.SourceInsecure)
+		if err != nil {
+			return nil, errors.Wrap(err, "Init source image parser")
+		}
+		sourceParser, err := parser.New(sourcesRemote, opt.ExpectedArch)
+		sourcesParser = append(sourcesParser, sourceParser)
+		if err != nil {
+			return nil, errors.Wrap(err, "Failed to create parser")
+		}
+	}
+
+	generator := &Generator{
+		Opt:           opt,
+		sourcesParser: sourcesParser,
+	}
+
+	return generator, nil
+}
+
+// Generate saves multiple Nydus bootstraps into the database one by one.
+func (generator *Generator) Generate(ctx context.Context) error {
+	for index := range generator.Sources {
+		if err := generator.save(ctx, index); err != nil {
+			if utils.RetryWithHTTP(err) {
+				generator.sourcesParser[index].Remote.MaybeWithHTTP(err)
+			}
+			if err := generator.save(ctx, index); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// "save" stores information of chunk and blob of a Nydus Image in the database
+func (generator *Generator) save(ctx context.Context, index int) error {
+	sourceParsed, err := generator.sourcesParser[index].Parse(ctx)
+	if err != nil {
+		return errors.Wrap(err, "parse Nydus image")
+	}
+
+	// Create a directory to store the image bootstrap
+	nydusImageName := strings.Replace(generator.Sources[index], "/", ":", -1)
+	folderPath := filepath.Join(generator.WorkDir, nydusImageName)
+	if err := os.MkdirAll(folderPath, fs.ModePerm); err != nil {
+		return errors.Wrap(err, "creat work directory")
+	}
+	if err := generator.Output(ctx, sourceParsed, folderPath, index); err != nil {
+		return errors.Wrap(err, "output image information")
+	}
+
+	// Invoke "nydus-image save" command
+	builder := build.NewBuilder(generator.NydusImagePath)
+	if err := builder.Save(build.SaveOption{
+		BootstrapPath: filepath.Join(folderPath, "nydus_bootstrap"),
+	}); err != nil {
+		return errors.Wrap(err, "invalid nydus bootstrap format")
+	}
+
+	logrus.Infof("Save chunk information from image %s", generator.sourcesParser[index].Remote.Ref)
+
+	if err := os.RemoveAll(folderPath); err != nil {
+		return errors.Wrap(err, "remove work directory")
+	}
+	return nil
+}

contrib/nydusify/pkg/chunkdict/generator/output.go

@@ -0,0 +1,66 @@
+package generator
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+
+	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/parser"
+	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils"
+)
+
+func prettyDump(obj interface{}, name string) error {
+	bytes, err := json.MarshalIndent(obj, "", "  ")
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(name, bytes, 0644)
+}
+
+// Output outputs Nydus image nydus_bootstrap file and manifest, config to JSON file.
+func (generator *Generator) Output(
+	ctx context.Context, sourceParsed *parser.Parsed, outputPath string, index int,
+) error {
+	if sourceParsed.Index != nil {
+		if err := prettyDump(
+			sourceParsed.Index,
+			filepath.Join(outputPath, "nydus_index.json"),
+		); err != nil {
+			return errors.Wrap(err, "output nydus index file")
+		}
+	}
+	if sourceParsed.NydusImage != nil {
+		if err := prettyDump(
+			sourceParsed.NydusImage.Manifest,
+			filepath.Join(outputPath, "nydus_manifest.json"),
+		); err != nil {
+			return errors.Wrap(err, "output Nydus manifest file")
+		}
+		if err := prettyDump(
+			sourceParsed.NydusImage.Config,
+			filepath.Join(outputPath, "nydus_config.json"),
+		); err != nil {
+			return errors.Wrap(err, "output Nydus config file")
+		}
+		source := filepath.Join(outputPath, "nydus_bootstrap")
+		logrus.Infof("Pulling Nydus bootstrap to %s", source)
+		bootstrapReader, err := generator.sourcesParser[index].PullNydusBootstrap(ctx, sourceParsed.NydusImage)
+		if err != nil {
+			return errors.Wrap(err, "pull Nydus bootstrap layer")
+		}
+		defer bootstrapReader.Close()
+
+		if err := utils.UnpackFile(bootstrapReader, utils.BootstrapFileNameInLayer, source); err != nil {
+			return errors.Wrap(err, "unpack Nydus bootstrap layer")
+		}
+	} else {
+		err := fmt.Errorf("the %s is not a Nydus image", generator.sourcesParser[index].Remote.Ref)
+		return err
+	}
+	return nil
+}