Support zstdchunked conversion

Signed-off-by: abushwang <abushwangs@gmail.com>

Author: wswsmao

.github/workflows/integration-test.yml

@@ -93,6 +93,10 @@ jobs:
       run: |
         ./script/integration/estargz/test.sh ${{ env.OCI_IMAGE_NAME }}
 
+    - name: Test eStargz Driver (zstdchunked)
+      run: |
+        ./script/integration/estargz/test-zstdchunked.sh ${{ env.OCI_IMAGE_NAME }}
+
     - name: Test one-time mode
       run: |
         ./script/integration/one-time.sh ${{ env.OCI_IMAGE_NAME }}

README.md

@@ -1,7 +1,7 @@
 # Acceleration Service
 
 Acceleration Service provides a general service to Harbor with the ability to automatically convert user images to accelerated images. When a user does something such as artifact push, Harbor will request the service to complete the corresponding image conversion through its integrated [Nydus](https://github.com/dragonflyoss/image-service),
-[eStargz](https://github.com/containerd/stargz-snapshotter), etc. drivers.
+[eStargz, zstdchunked](https://github.com/containerd/stargz-snapshotter) etc. drivers.
 
 [![Release Version](https://img.shields.io/github/v/release/goharbor/acceleration-service?style=flat)](https://github.com/goharbor/acceleration-service/releases)
 [![Docker Pulls](https://img.shields.io/docker/pulls/goharbor/harbor-acceld.svg)](https://hub.docker.com/r/goharbor/harbor-acceld/)

misc/config/config.estargz.zstdchunked.yaml

@@ -0,0 +1,52 @@
+# Configuration file of Harbor Acceleration Service
+
+# http related config
+server:
+  name: API
+  # listened host for http
+  host: 0.0.0.0
+  # port for http
+  port: 2077
+
+metric:
+  # export metrics on `/metrics` endpoint
+  enabled: true
+
+provider:
+  source:
+    # hostname of harbor service
+    hub.harbor.com:
+      # base64 encoded `<robot-name>:<robot-secret>` for robot
+      # account created in harbor
+      # auth: YTpiCg==
+      # skip verifying server certs for HTTPS source registry
+      insecure: false
+      webhook:
+        # webhook request auth header configured in harbor
+        auth_header: header
+    localhost:
+      # If auth is not provided, it will attempt to read from docker config
+      # auth: YWRtaW46SGFyYm9yMTIzNDU=
+  # work directory of acceld
+  work_dir: /tmp
+  gcpolicy:
+      # size threshold that triggers GC, the oldest used blobs will be reclaimed if exceeds the size.
+      threshold: 1000MB
+
+converter:
+  # number of worker for executing conversion task
+  worker: 5
+  # enable to add harbor specified annotations to converted image for tracking.
+  harbor_annotation: true
+  # only convert images for specific platforms, leave empty for all platforms.
+  # platforms: linux/amd64,linux/arm64
+  driver:
+    # accelerator driver type: `estargz`
+    type: estargz
+    config:
+      docker2oci: true
+      # compress format: `zstdchunked`
+      compression: zstdchunked
+  rules:
+    # add suffix to tag of source image reference as target image reference
+    - tag_suffix: -estargz-zstdchunked

pkg/driver/estargz/estargz.go

@@ -16,17 +16,35 @@ package estargz
 
 import (
 	"context"
+	"fmt"
 	"strconv"
 
 	"github.com/containerd/containerd/images/converter"
 	"github.com/containerd/platforms"
 	"github.com/containerd/stargz-snapshotter/estargz"
 	estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz"
+	zstdchunkedconvert "github.com/containerd/stargz-snapshotter/nativeconverter/zstdchunked"
 	"github.com/goharbor/acceleration-service/pkg/content"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 )
 
+const (
+	DefaultCompressFormat = "estargz"
+	ZstdChunkedFormat     = "zstdchunked"
+)
+
+type layerConverter func([]estargz.Option) converter.ConvertFunc
+
+var compressFormatConverters = map[string]layerConverter{
+	DefaultCompressFormat: func(opts []estargz.Option) converter.ConvertFunc {
+		return estargzconvert.LayerConvertFunc(opts...)
+	},
+	ZstdChunkedFormat: func(opts []estargz.Option) converter.ConvertFunc {
+		return zstdchunkedconvert.LayerConvertFunc(opts...)
+	},
+}
+
 type Driver struct {
 	cfg        map[string]string
 	platformMC platforms.MatchComparer
@@ -37,15 +55,22 @@ func New(cfg map[string]string, platformMC platforms.MatchComparer) (*Driver, er
 }
 
 func (d *Driver) Convert(ctx context.Context, p content.Provider, ref string) (*ocispec.Descriptor, error) {
-	opts, docker2oci, err := getESGZConvertOpts(d.cfg)
+	opts, docker2oci, compressFormat, err := getConvertOpts(d.cfg)
 	if err != nil {
-		return nil, errors.Wrap(err, "parse estargz conversion options")
+		return nil, errors.Wrap(err, "parse conversion options")
 	}
+
+	layerConv, ok := compressFormatConverters[compressFormat]
+	if !ok {
+		return nil, fmt.Errorf("unsupported compress format: %s", compressFormat)
+	}
+
 	image, err := p.Image(ctx, ref)
 	if err != nil {
 		return nil, errors.Wrap(err, "get source image")
 	}
-	return converter.DefaultIndexConvertFunc(estargzconvert.LayerConvertFunc(opts...), docker2oci, d.platformMC)(
+
+	return converter.DefaultIndexConvertFunc(layerConv(opts), docker2oci, d.platformMC)(
 		ctx, p.ContentStore(), *image)
 }
 
@@ -57,11 +82,16 @@ func (d *Driver) Version() string {
 	return ""
 }
 
-func getESGZConvertOpts(cfg map[string]string) (opts []estargz.Option, docker2oci bool, err error) {
+func getConvertOpts(cfg map[string]string) (opts []estargz.Option, docker2oci bool, compressFormat string, err error) {
+	compressFormat = DefaultCompressFormat
+	if format, ok := cfg["compression"]; ok && format != "" {
+		compressFormat = format
+	}
+
 	if s, ok := cfg["docker2oci"]; ok {
 		b, err := strconv.ParseBool(s)
 		if err != nil {
-			return nil, false, err
+			return nil, false, "", err
 		}
 		docker2oci = b
 	}

script/integration/estargz/test-zstdchunked.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -euo pipefail
+
+OCI_IMAGE_NAME=$1
+
+# Start acceld service
+sudo nohup ./acceld --config ./misc/config/config.estargz.zstdchunked.yaml &> acceld.log &
+sleep 1
+
+# Convert image by accelctl
+sudo ./accelctl task create --sync localhost/library/$OCI_IMAGE_NAME:latest
+
+# Verify filesystem consistency for converted image
+# TODO
+
+# Gracefully exit acceld
+sudo pkill -SIGINT acceld