feature(main): add load tar files (#1)

* feature(main): add tar load feature

Signed-off-by: cuisongliu <[email protected]>

* feature(main): add tar load feature

Signed-off-by: cuisongliu <[email protected]>

* feature(main): add tar load feature

Signed-off-by: cuisongliu <[email protected]>

* feature(main): add tar main cmd

Signed-off-by: cuisongliu <[email protected]>

* feature(main): add tar main cmd

Signed-off-by: cuisongliu <[email protected]>

* feature(main): add tar main cmd

Signed-off-by: cuisongliu <[email protected]>

---------

Signed-off-by: cuisongliu <[email protected]>

.goreleaser.yml

@@ -14,6 +14,7 @@ builds:
       - -X github.com/labring/sreg/pkg/version.buildDate={{.Date}}
     tags:
       - containers_image_openpgp
+      - exclude_graphdriver_devicemapper
 
 checksum:
   name_template: 'checksums.txt'

pkg/buildimage/constants.go

@@ -17,8 +17,10 @@ limitations under the License.
 package buildimage
 
 const (
-	ChartsDirName    = "charts"
-	ManifestsDirName = "manifests"
-	ImagesDirName    = "images"
-	ImageShimDirName = "shim"
+	ChartsDirName      = "charts"
+	ManifestsDirName   = "manifests"
+	ImagesDirName      = "images"
+	ImageShimDirName   = "shim"
+	ImageSkopeoDirName = "skopeo"
+	ImageTarConfigName = "tar.txt"
 )

pkg/buildimage/tar_images.go

@@ -0,0 +1,62 @@
+// Copyright © 2021 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buildimage
+
+import (
+	"fmt"
+	"github.com/labring/sreg/pkg/utils/file"
+	"github.com/labring/sreg/pkg/utils/logger"
+	"path"
+	"strings"
+)
+
+func TarList(dir string) ([]string, error) {
+	wrapGetImageErr := func(err error, s string) error {
+		return fmt.Errorf("failed to get images in %s: %w", s, err)
+	}
+	tarDir := path.Join(dir, ImagesDirName, ImageSkopeoDirName)
+	if !file.IsExist(path.Join(tarDir, ImageTarConfigName)) {
+		logger.Warn("image tar config %s is not exists,skip", path.Join(tarDir, ImageTarConfigName))
+		return nil, nil
+	}
+
+	images, err := file.ReadLines(path.Join(tarDir, ImageTarConfigName))
+	if err != nil {
+		return nil, wrapGetImageErr(err, tarDir)
+	}
+	for i, image := range images {
+		parts := strings.SplitN(image, "@", 2)
+		if len(parts) != 2 {
+			return nil, fmt.Errorf("invalid image format: %s", image)
+		}
+		if strings.HasPrefix(parts[0], "docker-archive:") || strings.HasPrefix(parts[0], "oci-archive:") {
+			partOne := parts[0]
+
+			partOneSpilt := strings.SplitN(partOne, ":", 2)
+			if len(partOneSpilt) == 2 {
+				originalPath := partOneSpilt[1]
+				// 将原始路径和路径部分组合
+				modifiedPath := path.Join(tarDir, originalPath)
+				// 重新构建 partOne
+				partOne = partOneSpilt[0] + ":" + modifiedPath
+			}
+			parts[0] = partOne
+			images[i] = strings.Join(parts, "@")
+		} else {
+			return nil, fmt.Errorf("invalid image format: %s , must prefix in docker-archive or oci-archive", image)
+		}
+	}
+	return images, nil
+}

pkg/buildimage/tar_images_test.go

@@ -0,0 +1,29 @@
+/*
+Copyright 2023 [email protected].
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildimage
+
+import (
+	"testing"
+)
+
+func TestTarList(t *testing.T) {
+	data, err := TarList("test/testregistrytar")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(data)
+}

pkg/buildimage/test/testregistrytar/images/shim/image.txt

@@ -0,0 +1 @@
+docker.io/library/busybox:latest

pkg/buildimage/test/testregistrytar/images/skopeo/tar.txt

@@ -0,0 +1 @@
+docker-archive:config_main.tar@library/config_main

pkg/registry/commands/copy.go

@@ -105,7 +105,7 @@ func runCopy(cmd *cobra.Command, source, dst string, opts globalOptions) error {
 	if err != nil {
 		return err
 	}
-	if err := sync.ToImage(ctx, sys, srcRef, dep, imageListSelection); err != nil {
+	if err := sync.RegistryToImage(ctx, sys, srcRef, dep, imageListSelection); err != nil {
 		return err
 	}
 	fmt.Fprintln(out, "Copy image completed")

pkg/registry/commands/save.go

@@ -50,6 +50,21 @@ func NewRegistryImageSaveCmd(examplePrefix string) *cobra.Command {
 				return err
 			}
 			logger.Info("images pulled: %+v", outImages)
+			if args[0] != "" {
+				tarIs := save.NewImageTarSaver(context.Background(), flagsResults.registryPullMaxPullProcs)
+				tars, err := buildimage.TarList(args[0])
+				if err != nil {
+					return err
+				}
+				if len(tars) != 0 {
+					outTars, err := tarIs.SaveImages(tars, flagsResults.registryPullRegistryDir, v1.Platform{OS: "linux", Architecture: flagsResults.registryPullArch})
+					if err != nil {
+						return err
+					}
+					logger.Info("images tar saved: %+v", outTars)
+				}
+			}
+
 			return nil
 		},
 		PreRunE: func(cmd *cobra.Command, args []string) error {

pkg/registry/save/registry_save.go

@@ -17,6 +17,7 @@ package save
 import (
 	"context"
 	"fmt"
+	"github.com/containers/image/v5/transports/alltransports"
 	"strings"
 	stdsync "sync"
 	"time"
@@ -73,11 +74,20 @@ func (is *tmpRegistryImage) SaveImages(images []string, dir string, platform v1.
 				<-numCh
 				mu.Unlock()
 			}()
-			srcRef, err := sync.ImageNameToReference(sys, img, is.auths)
-			if err != nil {
-				return err
+			var srcRef itype.ImageReference
+			if strings.HasPrefix(img, "docker-daemon") {
+				logger.Info("Using containers-storage or docker-daemon as image transport")
+				srcRef, err = alltransports.ParseImageName(img)
+				if err != nil {
+					return fmt.Errorf("invalid source name %s: %v", img, err)
+				}
+			} else {
+				srcRef, err = sync.ImageNameToReference(sys, img, is.auths)
+				if err != nil {
+					return err
+				}
 			}
-			err = sync.ToImage(is.ctx, sys, srcRef, ep, copy.CopySystemImage)
+			err = sync.RegistryToImage(is.ctx, sys, srcRef, ep, copy.CopySystemImage)
 			if err != nil {
 				return fmt.Errorf("save image %s: %w", img, err)
 			}

pkg/registry/save/registry_tars_save.go

@@ -0,0 +1,106 @@
+// Copyright © 2021 sealos.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package save
+
+import (
+	"context"
+	"fmt"
+	"github.com/containers/image/v5/transports/alltransports"
+	"strings"
+	stdsync "sync"
+	"time"
+
+	"github.com/containers/image/v5/copy"
+	itype "github.com/containers/image/v5/types"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+	"golang.org/x/sync/errgroup"
+
+	"github.com/labring/sreg/pkg/registry/handler"
+	"github.com/labring/sreg/pkg/registry/sync"
+	httputils "github.com/labring/sreg/pkg/utils/http"
+	"github.com/labring/sreg/pkg/utils/logger"
+)
+
+func NewImageTarSaver(ctx context.Context, maxPullProcs int) Registry {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	return &tmpTarRegistryImage{
+		ctx:          ctx,
+		maxPullProcs: maxPullProcs,
+	}
+}
+
+type tmpTarRegistryImage struct {
+	ctx          context.Context
+	maxPullProcs int
+}
+
+func (is *tmpTarRegistryImage) SaveImages(images []string, dir string, platform v1.Platform) ([]string, error) {
+	logger.Debug("trying to save images: %+v for platform: %s", images,
+		strings.Join([]string{platform.OS, platform.Architecture, platform.Variant}, ","))
+	config, err := handler.NewConfig(dir, 0)
+	if err != nil {
+		return nil, err
+	}
+	config.Log.AccessLog.Disabled = true
+	errCh := handler.Run(is.ctx, config)
+
+	probeCtx, cancel := context.WithTimeout(is.ctx, time.Second*3)
+	defer cancel()
+	ep := sync.ParseRegistryAddress(localhost, config.HTTP.Addr)
+	if err = httputils.WaitUntilEndpointAlive(probeCtx, "http://"+ep); err != nil {
+		return nil, err
+	}
+
+	if platform.OS == "" {
+		platform.OS = "linux"
+	}
+	sys := &itype.SystemContext{
+		ArchitectureChoice:          platform.Architecture,
+		OSChoice:                    platform.OS,
+		VariantChoice:               platform.Variant,
+		DockerInsecureSkipTLSVerify: itype.OptionalBoolTrue,
+	}
+	eg, _ := errgroup.WithContext(is.ctx)
+	numCh := make(chan struct{}, is.maxPullProcs)
+	var outImages []string
+	var mu stdsync.Mutex
+	for index := range images {
+		img := images[index]
+		numCh <- struct{}{}
+		eg.Go(func() error {
+			mu.Lock()
+			defer func() {
+				<-numCh
+				mu.Unlock()
+			}()
+			allImage := strings.Split(img, "@")
+			srcRef, err := alltransports.ParseImageName(allImage[0])
+			if err != nil {
+				return fmt.Errorf("invalid source name %s: %v", allImage[0], err)
+			}
+			err = sync.ArchiveToImage(is.ctx, sys, srcRef, fmt.Sprintf("%s/%s", ep, allImage[1]), copy.CopySystemImage)
+			if err != nil {
+				return fmt.Errorf("save image %s: %w", img, err)
+			}
+			outImages = append(outImages, img)
+			return nil
+		})
+	}
+	err = eg.Wait()
+	errCh <- err
+	return outImages, err
+}

pkg/registry/save/save_test.go

@@ -44,3 +44,22 @@ func TestSaveTmpImages(t *testing.T) {
 		t.Logf("images: %+v", imgs)
 	})
 }
+
+func TestSaveTarImages(t *testing.T) {
+	defer func() {
+		_ = os.RemoveAll("testdata/registry")
+	}()
+	save := NewImageTarSaver(context.TODO(), 5)
+	t.Run("no credentials found", func(t *testing.T) {
+		_ = os.Mkdir("testdata/registry", 0755)
+		imgs, err := save.SaveImages([]string{"docker-archive:testregistrytar/images/skopeo/config_main.tar@library/config_main"}, "testdata/registry", v1.Platform{
+			Architecture: "amd64",
+			OS:           "linux",
+		})
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		t.Logf("images: %+v", imgs)
+	})
+}

pkg/registry/save/testregistrytar/images/shim/image.txt

@@ -0,0 +1,5 @@
+registry.k8s.io/autoscaling/vpa-admission-controller:0.13.0
+registry.k8s.io/autoscaling/vpa-recommender:0.13.0
+registry.k8s.io/autoscaling/vpa-updater:0.13.0
+registry.k8s.io/redis:e2e
+registry.k8s.io/ubuntu-slim:0.1

pkg/registry/save/testregistrytar/images/skopeo/tar.txt

@@ -0,0 +1 @@
+docker-archive:config_main.tar@library/config_main

pkg/registry/sync/sync.go

@@ -152,7 +152,7 @@ func ImageNameToReference(sys *types.SystemContext, img string, auth map[string]
 	return srcRef, nil
 }
 
-func ToImage(ctx context.Context, sys *types.SystemContext, src types.ImageReference, dst string, selection copy.ImageListSelection) error {
+func RegistryToImage(ctx context.Context, sys *types.SystemContext, src types.ImageReference, dst string, selection copy.ImageListSelection) error {
 	allSrcImage := src.DockerReference().String()
 	var repo string
 	parts := strings.SplitN(allSrcImage, "/", 2)
@@ -185,6 +185,27 @@ func ToImage(ctx context.Context, sys *types.SystemContext, src types.ImageRefer
 	}, getRetryOptions())
 }
 
+func ArchiveToImage(ctx context.Context, sys *types.SystemContext, src types.ImageReference, dst string, selection copy.ImageListSelection) error {
+	logger.Debug("syncing image from %s to %s", src.Transport().Name(), dst)
+	destRef, err := alltransports.ParseImageName(fmt.Sprintf("docker://%s", dst))
+	if err != nil {
+		return fmt.Errorf("invalid destination name %s: %v", dst, err)
+	}
+	policyContext, err := getPolicyContext()
+	if err != nil {
+		return err
+	}
+	return retry.RetryIfNecessary(ctx, func() error {
+		_, err = copy.Image(ctx, policyContext, destRef, src, &copy.Options{
+			SourceCtx:          sys,
+			DestinationCtx:     sys,
+			ImageListSelection: selection,
+			ReportWriter:       os.Stdout,
+		})
+		return err
+	}, getRetryOptions())
+}
+
 func getPolicyContext() (*signature.PolicyContext, error) {
 	policy := &signature.Policy{
 		Default: []signature.PolicyRequirement{