Merge pull request #4061 from lengrongfu/master

[Feat] Add commit compression type support

Author: fahedouch

cmd/nerdctl/container/container_commit.go

@@ -17,6 +17,8 @@
 package container
 
 import (
+	"errors"
+
 	"github.com/spf13/cobra"
 
 	"github.com/containerd/nerdctl/v2/cmd/nerdctl/completion"
@@ -40,6 +42,7 @@ func CommitCommand() *cobra.Command {
 	cmd.Flags().StringP("message", "m", "", "Commit message")
 	cmd.Flags().StringArrayP("change", "c", nil, "Apply Dockerfile instruction to the created image (supported directives: [CMD, ENTRYPOINT])")
 	cmd.Flags().BoolP("pause", "p", true, "Pause container during commit")
+	cmd.Flags().StringP("compression", "", "gzip", "commit compression algorithm (zstd or gzip)")
 	return cmd
 }
 
@@ -66,23 +69,29 @@ func commitOptions(cmd *cobra.Command) (types.ContainerCommitOptions, error) {
 		return types.ContainerCommitOptions{}, err
 	}
 
+	com, err := cmd.Flags().GetString("compression")
+	if err != nil {
+		return types.ContainerCommitOptions{}, err
+	}
+	if com != string(types.Zstd) && com != string(types.Gzip) {
+		return types.ContainerCommitOptions{}, errors.New("--compression param only supports zstd or gzip")
+	}
 	return types.ContainerCommitOptions{
-		Stdout:   cmd.OutOrStdout(),
-		GOptions: globalOptions,
-		Author:   author,
-		Message:  message,
-		Pause:    pause,
-		Change:   change,
+		Stdout:      cmd.OutOrStdout(),
+		GOptions:    globalOptions,
+		Author:      author,
+		Message:     message,
+		Pause:       pause,
+		Change:      change,
+		Compression: types.CompressionType(com),
 	}, nil
-
 }
 
 func commitAction(cmd *cobra.Command, args []string) error {
 	options, err := commitOptions(cmd)
 	if err != nil {
 		return err
 	}
-
 	client, ctx, cancel, err := clientutil.NewClient(cmd.Context(), options.GOptions.Namespace, options.GOptions.Address)
 	if err != nil {
 		return err

cmd/nerdctl/container/container_commit_test.go

@@ -19,10 +19,14 @@ package container
 import (
 	"testing"
 
+	"gotest.tools/v3/assert"
+
 	"github.com/containerd/nerdctl/mod/tigron/expect"
 	"github.com/containerd/nerdctl/mod/tigron/require"
 	"github.com/containerd/nerdctl/mod/tigron/test"
+	"github.com/containerd/nerdctl/mod/tigron/tig"
 
+	"github.com/containerd/nerdctl/v2/pkg/inspecttypes/native"
 	"github.com/containerd/nerdctl/v2/pkg/testutil"
 	"github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest"
 )
@@ -86,3 +90,52 @@ func TestCommit(t *testing.T) {
 
 	testCase.Run(t)
 }
+
+func TestZstdCommit(t *testing.T) {
+	testCase := nerdtest.Setup()
+	testCase.Require = require.All(
+		// FIXME: Docker  does not support compression
+		require.Not(nerdtest.Docker),
+		nerdtest.ContainerdVersion("2.0.0"),
+		nerdtest.CGroup,
+	)
+	testCase.Cleanup = func(data test.Data, helpers test.Helpers) {
+		helpers.Anyhow("rm", "-f", data.Identifier())
+		helpers.Anyhow("rmi", "-f", data.Identifier("image"))
+	}
+	testCase.Setup = func(data test.Data, helpers test.Helpers) {
+		identifier := data.Identifier()
+		helpers.Ensure("run", "-d", "--name", identifier, testutil.CommonImage, "sleep", nerdtest.Infinity)
+		nerdtest.EnsureContainerStarted(helpers, identifier)
+		helpers.Ensure("exec", identifier, "sh", "-euxc", `echo hello-test-commit > /foo`)
+		helpers.Ensure("commit", identifier, data.Identifier("image"), "--compression=zstd")
+		data.Labels().Set("image", data.Identifier("image"))
+	}
+
+	testCase.SubTests = []*test.Case{
+		{
+			Description: "verify zstd has been used",
+			Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
+				return helpers.Command("image", "inspect", "--mode=native", data.Labels().Get("image"))
+			},
+			Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
+				return &test.Expected{
+					ExitCode: 0,
+					Output: expect.JSON([]native.Image{}, func(images []native.Image, s string, t tig.T) {
+						assert.Equal(t, len(images), 1)
+						assert.Equal(helpers.T(), images[0].Manifest.Layers[len(images[0].Manifest.Layers)-1].MediaType, "application/vnd.docker.image.rootfs.diff.tar.zstd")
+					}),
+				}
+			},
+		},
+		{
+			Description: "verify the image is working",
+			Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
+				return helpers.Command("run", "--rm", data.Labels().Get("image"), "sh", "-c", "--", "cat /foo")
+			},
+			Expected: test.Expects(0, nil, expect.Equals("hello-test-commit\n")),
+		},
+	}
+
+	testCase.Run(t)
+}

docs/command-reference.md

@@ -766,6 +766,7 @@ Flags:
 - :whale: `-m, --message`: Commit message
 - :whale: `-c, --change`: Apply Dockerfile instruction to the created image (supported directives: [CMD, ENTRYPOINT])
 - :whale: `-p, --pause`: Pause container during commit (default: true)
+- :nerd_face: `--compression`: Commit compression algorithm (supported values: zstd or gzip) (default: gzip) (zstd is generally better for compression ratio but might not be as widely supported)
 
 ## Image management
 

pkg/api/types/container_types.go

@@ -385,8 +385,17 @@ type ContainerCommitOptions struct {
 	Change []string
 	// Pause container during commit
 	Pause bool
+	// Compression is set commit compression algorithm
+	Compression CompressionType
 }
 
+type CompressionType string
+
+const (
+	Zstd CompressionType = "zstd"
+	Gzip CompressionType = "gzip"
+)
+
 // ContainerDiffOptions specifies options for `nerdctl (container) diff`.
 type ContainerDiffOptions struct {
 	Stdout io.Writer

pkg/cmd/container/commit.go

@@ -44,11 +44,12 @@ func Commit(ctx context.Context, client *containerd.Client, rawRef string, req s
 	}
 
 	opts := &commit.Opts{
-		Author:  options.Author,
-		Message: options.Message,
-		Ref:     parsedReference.String(),
-		Pause:   options.Pause,
-		Changes: changes,
+		Author:      options.Author,
+		Message:     options.Message,
+		Ref:         parsedReference.String(),
+		Pause:       options.Pause,
+		Changes:     changes,
+		Compression: options.Compression,
 	}
 
 	walker := &containerwalker.ContainerWalker{

pkg/imgutil/commit/commit.go

@@ -57,11 +57,12 @@ type Changes struct {
 }
 
 type Opts struct {
-	Author  string
-	Message string
-	Ref     string
-	Pause   bool
-	Changes Changes
+	Author      string
+	Message     string
+	Ref         string
+	Pause       bool
+	Changes     Changes
+	Compression types.CompressionType
 }
 
 var (
@@ -176,7 +177,7 @@ func Commit(ctx context.Context, client *containerd.Client, container containerd
 	// Sync filesystem to make sure that all the data writes in container could be persisted to disk.
 	Sync()
 
-	diffLayerDesc, diffID, err := createDiff(ctx, id, sn, client.ContentStore(), differ)
+	diffLayerDesc, diffID, err := createDiff(ctx, id, sn, client.ContentStore(), differ, opts.Compression)
 	if err != nil {
 		return emptyDigest, fmt.Errorf("failed to export layer: %w", err)
 	}
@@ -356,8 +357,14 @@ func writeContentsForImage(ctx context.Context, snName string, baseImg container
 }
 
 // createDiff creates a layer diff into containerd's content store.
-func createDiff(ctx context.Context, name string, sn snapshots.Snapshotter, cs content.Store, comparer diff.Comparer) (ocispec.Descriptor, digest.Digest, error) {
-	newDesc, err := rootfs.CreateDiff(ctx, name, sn, comparer)
+func createDiff(ctx context.Context, name string, sn snapshots.Snapshotter, cs content.Store, comparer diff.Comparer, compression types.CompressionType) (ocispec.Descriptor, digest.Digest, error) {
+	opts := make([]diff.Opt, 0)
+	mediaType := images.MediaTypeDockerSchema2LayerGzip
+	if compression == types.Zstd {
+		opts = append(opts, diff.WithMediaType(ocispec.MediaTypeImageLayerZstd))
+		mediaType = images.MediaTypeDockerSchema2LayerZstd
+	}
+	newDesc, err := rootfs.CreateDiff(ctx, name, sn, comparer, opts...)
 	if err != nil {
 		return ocispec.Descriptor{}, digest.Digest(""), err
 	}
@@ -378,7 +385,7 @@ func createDiff(ctx context.Context, name string, sn snapshots.Snapshotter, cs c
 	}
 
 	return ocispec.Descriptor{
-		MediaType: images.MediaTypeDockerSchema2LayerGzip,
+		MediaType: mediaType,
 		Digest:    newDesc.Digest,
 		Size:      info.Size,
 	}, diffID, nil

pkg/testutil/nerdtest/requirements.go

@@ -24,6 +24,7 @@ import (
 	"os/exec"
 	"strings"
 
+	"github.com/Masterminds/semver/v3"
 	"gotest.tools/v3/assert"
 
 	"github.com/containerd/containerd/v2/defaults"
@@ -32,6 +33,7 @@ import (
 
 	"github.com/containerd/nerdctl/v2/pkg/buildkitutil"
 	"github.com/containerd/nerdctl/v2/pkg/clientutil"
+	"github.com/containerd/nerdctl/v2/pkg/infoutil"
 	"github.com/containerd/nerdctl/v2/pkg/inspecttypes/dockercompat"
 	"github.com/containerd/nerdctl/v2/pkg/rootlessutil"
 	"github.com/containerd/nerdctl/v2/pkg/testutil"
@@ -416,3 +418,24 @@ var RemapIDs = &test.Requirement{
 		return false, "snapshotter does not support ID remapping"
 	},
 }
+
+func ContainerdVersion(v string) *test.Requirement {
+	return &test.Requirement{
+		Check: func(data test.Data, helpers test.Helpers) (bool, string) {
+			ctx := context.Background()
+			namespace := defaultNamespace
+			address := defaults.DefaultAddress
+			client, ctx, cancel, err := clientutil.NewClient(ctx, namespace, address)
+			if err != nil {
+				return false, fmt.Sprintf("failed to create client: %v", err)
+			}
+			defer cancel()
+			if sv, err := infoutil.ServerSemVer(ctx, client); err != nil {
+				return false, err.Error()
+			} else if sv.LessThan(semver.MustParse(v)) {
+				return false, fmt.Sprintf("`nerdctl commit --compression expects containerd %s or later, got containerd %v", v, sv)
+			}
+			return true, ""
+		},
+	}
+}