Merge pull request #399 from AkihiroSuda/fix-build-output

build: support --output=(oci|tar|docker|image), --quiet, --cache-from, --cache-to

Author: AkihiroSuda

README.md

@@ -593,11 +593,20 @@ Flags:
 - :whale: `--target`: Set the target build stage to build
 - :whale: `--build-arg`: Set build-time variables
 - :whale: `--no-cache`: Do not use cache when building the image
+- :whale: `--output=OUTPUT`: Output destination (format: type=local,dest=path)
+  - :whale: `type=local,dest=path/to/output-dir`: Local directory
+  - :whale: `type=oci[,dest=path/to/output.tar]`: Docker/OCI dual-format tar ball (compatible with `docker buildx build`)
+  - :whale: `type=docker[,dest=path/to/output.tar]`: Docker format tar ball (compatible with `docker buildx build`)
+  - :whale: `type=tar[,dest=path/to/output.tar]`: Raw tar ball
+  - :whale: `type=image,name=example.com/image,push=true`: Push to a registry (see [`buildctl build`](https://github.com/moby/buildkit/tree/v0.9.0#imageregistry) documentation)
 - :whale: `--progress=(auto|plain|tty)`: Set type of progress output (auto, plain, tty). Use plain to show container output
 - :whale: `--secret`: Secret file to expose to the build: id=mysecret,src=/local/secret
 - :whale: `--ssh`: SSH agent socket or keys to expose to the build (format: `default|<id>[=<socket>|<key>[,<key>]]`)
+- :whale: `-q, --quiet`: Suppress the build output and print image ID on success
+- :whale: `--cache-from=CACHE`: External cache sources (eg. user/app:cache, type=local,src=path/to/dir) (compatible with `docker buildx build`)
+- :whale: `--cache-to=CACHE`: Cache export destinations (eg. user/app:cache, type=local,dest=path/to/dir) (compatible with `docker buildx build`)
 
-Unimplemented `docker build` flags: `--add-host`, `--cache-from`, `--iidfile`, `--label`, `--network`, `--platform`, `--quiet`, `--squash`
+Unimplemented `docker build` flags: `--add-host`, `--iidfile`, `--label`, `--network`, `--platform`, `--squash`
 
 ### :whale: nerdctl commit
 Create a new image from a container's changes

cmd/nerdctl/build.go

@@ -17,9 +17,10 @@
 package main
 
 import (
-	"fmt"
+	"io"
 	"os"
 	"os/exec"
+	"strconv"
 	"strings"
 
 	"path/filepath"
@@ -69,7 +70,7 @@ var buildCommand = &cli.Command{
 		&cli.StringFlag{
 			Name:    "output",
 			Aliases: []string{"o"},
-			Value:   "type=docker",
+			Usage:   "Output destination (format: type=local,dest=path)",
 		},
 		&cli.StringFlag{
 			Name:  "progress",
@@ -84,6 +85,19 @@ var buildCommand = &cli.Command{
 			Name:  "ssh",
 			Usage: "SSH agent socket or keys to expose to the build (format: default|<id>[=<socket>|<key>[,<key>]])",
 		},
+		&cli.BoolFlag{
+			Name:    "quiet",
+			Aliases: []string{"q"},
+			Usage:   "Suppress the build output and print image ID on success",
+		},
+		&cli.StringSliceFlag{
+			Name:  "cache-from",
+			Usage: "External cache sources (eg. user/app:cache, type=local,src=path/to/dir)",
+		},
+		&cli.StringSliceFlag{
+			Name:  "cache-to",
+			Usage: "Cache export destinations (eg. user/app:cache, type=local,dest=path/to/dir)",
+		},
 	},
 }
 
@@ -93,31 +107,37 @@ func buildAction(clicontext *cli.Context) error {
 		return err
 	}
 
-	buildctlBinary, buildctlArgs, err := generateBuildctlArgs(clicontext)
+	buildctlBinary, buildctlArgs, needsLoading, err := generateBuildctlArgs(clicontext)
 	if err != nil {
 		return err
 	}
 
+	quiet := clicontext.Bool("quiet")
+
 	logrus.Debugf("running %s %v", buildctlBinary, buildctlArgs)
 	buildctlCmd := exec.Command(buildctlBinary, buildctlArgs...)
 	buildctlCmd.Env = os.Environ()
 
-	buildctlStdout, err := buildctlCmd.StdoutPipe()
-	if err != nil {
-		return err
+	var buildctlStdout io.Reader
+	if needsLoading {
+		buildctlStdout, err = buildctlCmd.StdoutPipe()
+		if err != nil {
+			return err
+		}
+	} else {
+		buildctlCmd.Stdout = clicontext.App.Writer
 	}
-	buildctlCmd.Stderr = clicontext.App.ErrWriter
 
-	if err := buildctlCmd.Start(); err != nil {
-		return err
+	if !quiet {
+		buildctlCmd.Stderr = clicontext.App.ErrWriter
 	}
 
-	localBuild, err := isLocalBuild(clicontext)
-	if err != nil {
+	if err := buildctlCmd.Start(); err != nil {
 		return err
 	}
-	if !localBuild {
-		if err = loadImage(buildctlStdout, clicontext); err != nil {
+
+	if needsLoading {
+		if err = loadImage(buildctlStdout, clicontext, quiet); err != nil {
 			return err
 		}
 	}
@@ -129,24 +149,29 @@ func buildAction(clicontext *cli.Context) error {
 	return nil
 }
 
-func generateBuildctlArgs(clicontext *cli.Context) (string, []string, error) {
+func generateBuildctlArgs(clicontext *cli.Context) (string, []string, bool, error) {
+	var needsLoading bool
 	if clicontext.NArg() < 1 {
-		return "", nil, errors.New("context needs to be specified")
+		return "", nil, false, errors.New("context needs to be specified")
 	}
 	buildContext := clicontext.Args().First()
 	if buildContext == "-" || strings.Contains(buildContext, "://") {
-		return "", nil, errors.Errorf("unsupported build context: %q", buildContext)
+		return "", nil, false, errors.Errorf("unsupported build context: %q", buildContext)
 	}
 
 	buildctlBinary, err := buildkitutil.BuildctlBinary()
 	if err != nil {
-		return "", nil, err
+		return "", nil, false, err
 	}
 
-	output := fmt.Sprintf("--output=%s", clicontext.String("output"))
+	output := clicontext.String("output")
+	if output == "" {
+		output = "type=docker"
+		needsLoading = true
+	}
 	if tagSlice := strutil.DedupeStrSlice(clicontext.StringSlice("tag")); len(tagSlice) > 0 {
 		if len(tagSlice) > 1 {
-			return "", nil, errors.Errorf("specifying multiple -t is not supported yet")
+			return "", nil, false, errors.Errorf("specifying multiple -t is not supported yet")
 		}
 		output += ",name=" + tagSlice[0]
 	}
@@ -159,7 +184,7 @@ func generateBuildctlArgs(clicontext *cli.Context) (string, []string, error) {
 		"--frontend=dockerfile.v0",
 		"--local=context=" + buildContext,
 		"--local=dockerfile=" + buildContext,
-		output,
+		"--output=" + output,
 	}...)
 
 	if filename := clicontext.String("file"); filename != "" {
@@ -176,6 +201,20 @@ func generateBuildctlArgs(clicontext *cli.Context) (string, []string, error) {
 
 	for _, ba := range strutil.DedupeStrSlice(clicontext.StringSlice("build-arg")) {
 		buildctlArgs = append(buildctlArgs, "--opt=build-arg:"+ba)
+
+		// Support `--build-arg BUILDKIT_INLINE_CACHE=1` for compatibility with `docker buildx build`
+		// https://github.com/docker/buildx/blob/v0.6.3/docs/reference/buildx_build.md#-export-build-cache-to-an-external-cache-destination---cache-to
+		if strings.HasPrefix(ba, "BUILDKIT_INLINE_CACHE=") {
+			bic := strings.TrimPrefix(ba, "BUILDKIT_INLINE_CACHE=")
+			bicParsed, err := strconv.ParseBool(bic)
+			if err == nil {
+				if bicParsed {
+					buildctlArgs = append(buildctlArgs, "--export-cache=type=inline")
+				}
+			} else {
+				logrus.WithError(err).Warnf("invalid BUILDKIT_INLINE_CACHE: %q", bic)
+			}
+		}
 	}
 
 	if clicontext.Bool("no-cache") {
@@ -190,18 +229,19 @@ func generateBuildctlArgs(clicontext *cli.Context) (string, []string, error) {
 		buildctlArgs = append(buildctlArgs, "--ssh="+s)
 	}
 
-	return buildctlBinary, buildctlArgs, nil
-}
-
-func isLocalBuild(clicontext *cli.Context) (bool, error) {
-	opts, err := strutil.ParseCSVMap(clicontext.String("output"))
-	if err != nil {
-		return false, err
+	for _, s := range strutil.DedupeStrSlice(clicontext.StringSlice("cache-from")) {
+		if !strings.Contains(s, "type=") {
+			s = "type=registry,ref=" + s
+		}
+		buildctlArgs = append(buildctlArgs, "--import-cache="+s)
 	}
-	if v, ok := opts["type"]; ok {
-		if strings.TrimSpace(strings.ToLower(v)) == "local" {
-			return true, nil
+
+	for _, s := range strutil.DedupeStrSlice(clicontext.StringSlice("cache-to")) {
+		if !strings.Contains(s, "type=") {
+			s = "type=registry,ref=" + s
 		}
+		buildctlArgs = append(buildctlArgs, "--export-cache="+s)
 	}
-	return false, nil
+
+	return buildctlBinary, buildctlArgs, needsLoading, nil
 }

cmd/nerdctl/load.go

@@ -55,10 +55,10 @@ func loadAction(clicontext *cli.Context) error {
 		defer f.Close()
 		in = f
 	}
-	return loadImage(in, clicontext)
+	return loadImage(in, clicontext, false)
 }
 
-func loadImage(in io.Reader, clicontext *cli.Context) error {
+func loadImage(in io.Reader, clicontext *cli.Context, quiet bool) error {
 	client, ctx, cancel, err := newClient(clicontext, containerd.WithDefaultPlatform(platforms.DefaultStrict()))
 	if err != nil {
 		return err
@@ -74,12 +74,18 @@ func loadImage(in io.Reader, clicontext *cli.Context) error {
 		image := containerd.NewImage(client, img)
 
 		// TODO: Show unpack status
-		fmt.Fprintf(clicontext.App.Writer, "unpacking %s (%s)...", img.Name, img.Target.Digest)
+		if !quiet {
+			fmt.Fprintf(clicontext.App.Writer, "unpacking %s (%s)...", img.Name, img.Target.Digest)
+		}
 		err = image.Unpack(ctx, sn)
 		if err != nil {
 			return err
 		}
-		fmt.Fprintf(clicontext.App.Writer, "done\n")
+		if quiet {
+			fmt.Fprintln(clicontext.App.Writer, img.Target.Digest)
+		} else {
+			fmt.Fprintf(clicontext.App.Writer, "done\n")
+		}
 	}
 
 	return nil

pkg/composer/serviceparser/build.go

@@ -29,7 +29,7 @@ import (
 
 func parseBuildConfig(c *types.BuildConfig, project *types.Project, imageName string) (*Build, error) {
 	if unknown := reflectutil.UnknownNonEmptyFields(c,
-		"Context", "Dockerfile", "Args", "Target",
+		"Context", "Dockerfile", "Args", "CacheFrom", "Target",
 	); len(unknown) > 0 {
 		logrus.Warnf("Ignoring: build: %+v", unknown)
 	}
@@ -66,6 +66,10 @@ func parseBuildConfig(c *types.BuildConfig, project *types.Project, imageName st
 		}
 	}
 
+	for _, s := range c.CacheFrom {
+		b.BuildArgs = append(b.BuildArgs, "--cache-from="+s)
+	}
+
 	if c.Target != "" {
 		b.BuildArgs = append(b.BuildArgs, "--target="+c.Target)
 	}