Skip to content

Commit 4485eae

Browse files
qweeahqweeah
authored
feat: support copying referrers for multi-arch images (#1122)
Signed-off-by: Billy Zha <[email protected]>
1 parent 0efe794 commit 4485eae

File tree

8 files changed

+143
-36
lines changed

8 files changed

+143
-36
lines changed

‎cmd/oras/root/cp.go

+51-17
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ package root
1717

1818
import (
1919
"context"
20+
"encoding/json"
2021
"fmt"
2122
"strings"
2223
"sync"
@@ -28,6 +29,7 @@ import (
2829
"oras.land/oras-go/v2/content"
2930
"oras.land/oras/cmd/oras/internal/display"
3031
"oras.land/oras/cmd/oras/internal/option"
32+
"oras.land/oras/internal/docker"
3133
"oras.land/oras/internal/graph"
3234
)
3335

@@ -137,36 +139,27 @@ func runCopy(ctx context.Context, opts copyOptions) error {
137139
var desc ocispec.Descriptor
138140
rOpts := oras.DefaultResolveOptions
139141
rOpts.TargetPlatform = opts.Platform.Platform
140-
if dstRef := opts.To.Reference; dstRef == "" {
142+
if opts.recursive {
141143
desc, err = oras.Resolve(ctx, src, opts.From.Reference, rOpts)
142144
if err != nil {
143145
return fmt.Errorf("failed to resolve %s: %w", opts.From.Reference, err)
144146
}
145-
if opts.recursive {
146-
err = oras.ExtendedCopyGraph(ctx, src, dst, desc, extendedCopyOptions.ExtendedCopyGraphOptions)
147-
} else {
148-
err = oras.CopyGraph(ctx, src, dst, desc, extendedCopyOptions.CopyGraphOptions)
149-
}
147+
err = recursiveCopy(ctx, src, dst, opts.To.Reference, desc, extendedCopyOptions)
150148
} else {
151-
if opts.recursive {
152-
srcRef := opts.From.Reference
153-
if rOpts.TargetPlatform != nil {
154-
// resolve source reference to specified platform
155-
desc, err := oras.Resolve(ctx, src, opts.From.Reference, rOpts)
156-
if err != nil {
157-
return fmt.Errorf("failed to resolve %s: %w", opts.From.Reference, err)
158-
}
159-
srcRef = desc.Digest.String()
149+
if opts.To.Reference == "" {
150+
desc, err = oras.Resolve(ctx, src, opts.From.Reference, rOpts)
151+
if err != nil {
152+
return fmt.Errorf("failed to resolve %s: %w", opts.From.Reference, err)
160153
}
161-
desc, err = oras.ExtendedCopy(ctx, src, srcRef, dst, dstRef, extendedCopyOptions)
154+
err = oras.CopyGraph(ctx, src, dst, desc, extendedCopyOptions.CopyGraphOptions)
162155
} else {
163156
copyOptions := oras.CopyOptions{
164157
CopyGraphOptions: extendedCopyOptions.CopyGraphOptions,
165158
}
166159
if opts.Platform.Platform != nil {
167160
copyOptions.WithTargetPlatform(opts.Platform.Platform)
168161
}
169-
desc, err = oras.Copy(ctx, src, opts.From.Reference, dst, dstRef, copyOptions)
162+
desc, err = oras.Copy(ctx, src, opts.From.Reference, dst, opts.To.Reference, copyOptions)
170163
}
171164
}
172165
if err != nil {
@@ -191,3 +184,44 @@ func runCopy(ctx context.Context, opts copyOptions) error {
191184

192185
return nil
193186
}
187+
188+
// recursiveCopy copies an artifact and its referrers from one target to another.
189+
// If the artifact is a manifest list or index, referrers of its manifests are copied as well.
190+
func recursiveCopy(ctx context.Context, src oras.ReadOnlyGraphTarget, dst oras.Target, dstRef string, root ocispec.Descriptor, opts oras.ExtendedCopyOptions) error {
191+
if root.MediaType == ocispec.MediaTypeImageIndex || root.MediaType == docker.MediaTypeManifestList {
192+
fetched, err := content.FetchAll(ctx, src, root)
193+
if err != nil {
194+
return err
195+
}
196+
var index ocispec.Index
197+
if err = json.Unmarshal(fetched, &index); err != nil {
198+
return nil
199+
}
200+
201+
referrers, err := graph.FindPredecessors(ctx, src, index.Manifests, opts)
202+
if err != nil {
203+
return err
204+
}
205+
206+
findPredecessor := opts.FindPredecessors
207+
opts.FindPredecessors = func(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
208+
descs, err := findPredecessor(ctx, src, desc)
209+
if err != nil {
210+
return nil, err
211+
}
212+
if content.Equal(desc, root) {
213+
// make sure referrers of child manifests are copied by pointing them to root
214+
descs = append(descs, referrers...)
215+
}
216+
return descs, nil
217+
}
218+
}
219+
220+
var err error
221+
if dstRef == "" || dstRef == root.Digest.String() {
222+
err = oras.ExtendedCopyGraph(ctx, src, dst, root, opts.ExtendedCopyGraphOptions)
223+
} else {
224+
_, err = oras.ExtendedCopy(ctx, src, root.Digest.String(), dst, dstRef, opts)
225+
}
226+
return err
227+
}

‎internal/docker/mediatype.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,6 @@ package docker
1717

1818
// docker media types
1919
const (
20-
MediaTypeManifest = "application/vnd.docker.distribution.manifest.v2+json"
20+
MediaTypeManifest = "application/vnd.docker.distribution.manifest.v2+json"
21+
MediaTypeManifestList = "application/vnd.docker.distribution.manifest.list.v2+json"
2122
)

‎internal/graph/graph.go

+31
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,11 @@ package graph
1818
import (
1919
"context"
2020
"encoding/json"
21+
"sync"
2122

2223
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
24+
"golang.org/x/sync/errgroup"
25+
"oras.land/oras-go/v2"
2326
"oras.land/oras-go/v2/content"
2427
"oras.land/oras-go/v2/registry"
2528
"oras.land/oras/internal/docker"
@@ -188,3 +191,31 @@ func fetchBytes(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descr
188191
defer rc.Close()
189192
return content.ReadAll(rc, desc)
190193
}
194+
195+
// FindPredecessors returns all predecessors of descs in src concurrently.
196+
func FindPredecessors(ctx context.Context, src oras.ReadOnlyGraphTarget, descs []ocispec.Descriptor, opts oras.ExtendedCopyOptions) ([]ocispec.Descriptor, error) {
197+
var referrers []ocispec.Descriptor
198+
g, ctx := errgroup.WithContext(ctx)
199+
var m sync.Mutex
200+
if opts.Concurrency != 0 {
201+
g.SetLimit(opts.Concurrency)
202+
}
203+
for _, desc := range descs {
204+
g.Go(func(node ocispec.Descriptor) func() error {
205+
return func() error {
206+
descs, err := opts.FindPredecessors(ctx, src, node)
207+
if err != nil {
208+
return err
209+
}
210+
m.Lock()
211+
defer m.Unlock()
212+
referrers = append(referrers, descs...)
213+
return nil
214+
}
215+
}(desc))
216+
}
217+
if err := g.Wait(); err != nil {
218+
return nil, err
219+
}
220+
return referrers, nil
221+
}

‎test/e2e/internal/testdata/multi_arch/const.go

+1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323

2424
var (
2525
Tag = "multi"
26+
EmptyTag = "empty_index"
2627
Digest = "sha256:e2bfc9cc6a84ec2d7365b5a28c6bc5806b7fa581c9ad7883be955a64e3cc034f"
2728
Manifest = `{"mediaType":"application/vnd.oci.image.index.v1+json","schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:9d84a5716c66a1d1b9c13f8ed157ba7d1edfe7f9b8766728b8a1f25c0d9c14c1","size":458,"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:4f93460061882467e6fb3b772dc6ab72130d9ac1906aed2fc7589a5cd145433c","size":458,"platform":{"architecture":"arm64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:58efe73e78fe043ca31b89007a025c594ce12aa7e6da27d21c7b14b50112e255","size":458,"platform":{"architecture":"arm","os":"linux","variant":"v7"}}]}`
2829
Descriptor = `{"mediaType":"application/vnd.oci.image.index.v1+json","digest":"sha256:e2bfc9cc6a84ec2d7365b5a28c6bc5806b7fa581c9ad7883be955a64e3cc034f","size":706}`

‎test/e2e/suite/command/cp.go

+48-17
Original file line numberDiff line numberDiff line change
@@ -115,15 +115,15 @@ var _ = Describe("1.1 registry users:", func() {
115115
})
116116

117117
It("should copy an image and its referrers to a new repository", func() {
118-
stateKeys := append(append(foobarStates, foobar.ImageReferrersStateKeys...), foobar.ImageReferrerConfigStateKeys...)
118+
stateKeys := append(append(foobar.ImageLayerStateKeys, foobar.ManifestStateKey, foobar.ImageReferrerConfigStateKeys[0]), foobar.ImageReferrersStateKeys...)
119119
src := RegistryRef(ZOTHost, ArtifactRepo, foobar.Tag)
120120
dst := RegistryRef(ZOTHost, cpTestRepo("referrers"), foobar.Digest)
121121
ORAS("cp", "-r", src, dst, "-v").MatchStatus(stateKeys, true, len(stateKeys)).Exec()
122122
CompareRef(src, dst)
123123
})
124124

125125
It("should copy a multi-arch image and its referrers to a new repository via tag", func() {
126-
stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.IndexReferrerConfigStateKey)
126+
stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.LinuxAMD64ReferrerConfigStateKey)
127127
src := RegistryRef(ZOTHost, ArtifactRepo, ma.Tag)
128128
dstRepo := cpTestRepo("index-referrers")
129129
dst := RegistryRef(ZOTHost, dstRepo, "copiedTag")
@@ -142,13 +142,47 @@ var _ = Describe("1.1 registry users:", func() {
142142
Expect(len(index.Manifests)).To(Equal(1))
143143
Expect(index.Manifests[0].Digest.String()).To(Equal(ma.IndexReferrerDigest))
144144
ORAS("manifest", "fetch", RegistryRef(ZOTHost, dstRepo, ma.LinuxAMD64Referrer.Digest.String())).
145-
WithDescription("not copy referrer of successor").
146-
ExpectFailure().
145+
WithDescription("copy referrer of successor").
146+
Exec()
147+
})
148+
149+
It("should copy a multi-arch image and its referrers without concurrency limitation", func() {
150+
stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.LinuxAMD64ReferrerConfigStateKey)
151+
src := RegistryRef(ZOTHost, ArtifactRepo, ma.Tag)
152+
dstRepo := cpTestRepo("index-referrers-concurrent")
153+
dst := RegistryRef(ZOTHost, dstRepo, "copiedTag")
154+
// test
155+
ORAS("cp", src, dst, "-r", "-v", "--concurrency", "0").
156+
MatchStatus(stateKeys, true, len(stateKeys)).
157+
MatchKeyWords("Digest: " + ma.Digest).
158+
Exec()
159+
// validate
160+
CompareRef(RegistryRef(ZOTHost, ImageRepo, ma.Digest), dst)
161+
var index ocispec.Index
162+
bytes := ORAS("discover", dst, "-o", "json", "--artifact-type", ma.IndexReferrerConfigStateKey.Name).
163+
MatchKeyWords(ma.IndexReferrerDigest).
164+
WithDescription("copy image referrer").
165+
Exec().Out.Contents()
166+
Expect(json.Unmarshal(bytes, &index)).ShouldNot(HaveOccurred())
167+
Expect(len(index.Manifests)).To(Equal(1))
168+
Expect(index.Manifests[0].Digest.String()).To(Equal(ma.IndexReferrerDigest))
169+
ORAS("manifest", "fetch", RegistryRef(ZOTHost, dstRepo, ma.LinuxAMD64Referrer.Digest.String())).
170+
WithDescription("copy referrer of successor").
147171
Exec()
148172
})
149173

174+
It("should copy an empty index", func() {
175+
src := RegistryRef(ZOTHost, ImageRepo, ma.EmptyTag)
176+
dstRepo := cpTestRepo("empty-index")
177+
dst := RegistryRef(ZOTHost, dstRepo, "copiedTag")
178+
// test
179+
ORAS("cp", src, dst, "-r", "-v", "--concurrency", "0").Exec()
180+
// validate
181+
CompareRef(RegistryRef(ZOTHost, ImageRepo, ma.EmptyTag), dst)
182+
})
183+
150184
It("should copy a multi-arch image and its referrers to a new repository via digest", func() {
151-
stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.IndexReferrerConfigStateKey)
185+
stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.LinuxAMD64ReferrerConfigStateKey)
152186
src := RegistryRef(ZOTHost, ArtifactRepo, ma.Tag)
153187
dstRepo := cpTestRepo("index-referrers-digest")
154188
dst := RegistryRef(ZOTHost, dstRepo, ma.Digest)
@@ -168,7 +202,6 @@ var _ = Describe("1.1 registry users:", func() {
168202
Expect(index.Manifests[0].Digest.String()).To(Equal(ma.IndexReferrerDigest))
169203
ORAS("manifest", "fetch", RegistryRef(ZOTHost, dstRepo, ma.LinuxAMD64Referrer.Digest.String())).
170204
WithDescription("not copy referrer of successor").
171-
ExpectFailure().
172205
Exec()
173206
})
174207

@@ -270,7 +303,7 @@ var _ = Describe("OCI spec 1.0 registry users:", func() {
270303
When("running `cp`", func() {
271304
It("should copy an image artifact and its referrers from a registry to a fallback registry", func() {
272305
repo := cpTestRepo("to-fallback")
273-
stateKeys := append(append(foobarStates, foobar.ImageReferrersStateKeys...), foobar.ImageReferrerConfigStateKeys...)
306+
stateKeys := append(append(foobar.ImageLayerStateKeys, foobar.ManifestStateKey, foobar.ImageReferrerConfigStateKeys[0]), foobar.ImageReferrersStateKeys...)
274307
src := RegistryRef(ZOTHost, ArtifactRepo, foobar.SignatureImageReferrer.Digest.String())
275308
dst := RegistryRef(FallbackHost, repo, "")
276309
ORAS("cp", "-r", src, dst, "-v").MatchStatus(stateKeys, true, len(stateKeys)).Exec()
@@ -280,7 +313,7 @@ var _ = Describe("OCI spec 1.0 registry users:", func() {
280313
})
281314
It("should copy an image artifact and its referrers from a fallback registry to a registry", func() {
282315
repo := cpTestRepo("from-fallback")
283-
stateKeys := append(append(foobarStates, foobar.ImageReferrersStateKeys...), foobar.ImageReferrerConfigStateKeys...)
316+
stateKeys := append(append(foobar.ImageLayerStateKeys, foobar.ManifestStateKey, foobar.ImageReferrerConfigStateKeys[0]), foobar.ImageReferrersStateKeys...)
284317
src := RegistryRef(FallbackHost, ArtifactRepo, foobar.SBOMImageReferrer.Digest.String())
285318
dst := RegistryRef(ZOTHost, repo, "")
286319
ORAS("cp", "-r", src, dst, "-v").MatchStatus(stateKeys, true, len(stateKeys)).Exec()
@@ -439,7 +472,7 @@ var _ = Describe("OCI layout users:", func() {
439472
})
440473

441474
It("should copy a tagged image and its referrers from a registry to an OCI image layout", func() {
442-
stateKeys := append(append(foobarStates, foobar.ImageReferrersStateKeys...), foobar.ImageReferrerConfigStateKeys...)
475+
stateKeys := append(append(foobar.ImageLayerStateKeys, foobar.ManifestStateKey, foobar.ImageReferrerConfigStateKeys[0]), foobar.ImageReferrersStateKeys...)
443476
dst := LayoutRef(GinkgoT().TempDir(), "copied")
444477
src := RegistryRef(ZOTHost, ArtifactRepo, foobar.Tag)
445478
// test
@@ -451,7 +484,7 @@ var _ = Describe("OCI layout users:", func() {
451484
})
452485

453486
It("should copy a image and its referrers from a registry to an OCI image layout via digest", func() {
454-
stateKeys := append(append(foobarStates, foobar.ImageReferrersStateKeys...), foobar.ImageReferrerConfigStateKeys...)
487+
stateKeys := append(append(foobar.ImageLayerStateKeys, foobar.ManifestStateKey, foobar.ImageReferrerConfigStateKeys[0]), foobar.ImageReferrersStateKeys...)
455488
toDir := GinkgoT().TempDir()
456489
src := RegistryRef(ZOTHost, ArtifactRepo, foobar.Digest)
457490
// test
@@ -463,7 +496,7 @@ var _ = Describe("OCI layout users:", func() {
463496
})
464497

465498
It("should copy a multi-arch image and its referrers from a registry to an OCI image layout a via tag", func() {
466-
stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.IndexReferrerConfigStateKey)
499+
stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.LinuxAMD64ReferrerConfigStateKey)
467500
src := RegistryRef(ZOTHost, ArtifactRepo, ma.Tag)
468501
toDir := GinkgoT().TempDir()
469502
dst := LayoutRef(toDir, "copied")
@@ -485,13 +518,12 @@ var _ = Describe("OCI layout users:", func() {
485518
Expect(len(index.Manifests)).To(Equal(1))
486519
Expect(index.Manifests[0].Digest.String()).To(Equal(ma.IndexReferrerDigest))
487520
ORAS("manifest", "fetch", Flags.Layout, LayoutRef(toDir, ma.LinuxAMD64Referrer.Digest.String())).
488-
WithDescription("not copy referrer of successor").
489-
ExpectFailure().
521+
WithDescription("copy referrer of successor").
490522
Exec()
491523
})
492524

493525
It("should copy a multi-arch image and its referrers from an OCI image layout to a registry via digest", func() {
494-
stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.IndexReferrerConfigStateKey)
526+
stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.LinuxAMD64ReferrerConfigStateKey)
495527
fromDir := GinkgoT().TempDir()
496528
src := LayoutRef(fromDir, ma.Tag)
497529
dst := RegistryRef(ZOTHost, cpTestRepo("recursive-from-layout"), "copied")
@@ -514,9 +546,8 @@ var _ = Describe("OCI layout users:", func() {
514546
Expect(json.Unmarshal(bytes, &index)).ShouldNot(HaveOccurred())
515547
Expect(len(index.Manifests)).To(Equal(1))
516548
Expect(index.Manifests[0].Digest.String()).To(Equal(ma.IndexReferrerDigest))
517-
ORAS("manifest", "fetch", LayoutRef(fromDir, ma.LinuxAMD64Referrer.Digest.String())).
518-
WithDescription("not copy referrer of successor").
519-
ExpectFailure().
549+
ORAS("manifest", "fetch", dst).
550+
WithDescription("copy referrer of successor").
520551
Exec()
521552
})
522553

‎test/e2e/suite/command/pull.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ var _ = Describe("OCI spec 1.1 registry users:", func() {
122122

123123
It("should copy an artifact with blob", func() {
124124
repo := cpTestRepo("artifact-with-blob")
125-
stateKeys := append(append(foobarStates, foobar.ImageReferrersStateKeys...), foobar.ImageReferrerConfigStateKeys...)
125+
stateKeys := append(append(foobar.ImageLayerStateKeys, foobar.ManifestStateKey, foobar.ImageReferrerConfigStateKeys[0]), foobar.ImageReferrersStateKeys...)
126126
src := RegistryRef(ZOTHost, ArtifactRepo, foobar.SignatureImageReferrer.Digest.String())
127127
dst := RegistryRef(FallbackHost, repo, "")
128128
ORAS("cp", "-r", src, dst, "-v").MatchStatus(stateKeys, true, len(stateKeys)).Exec()

‎test/e2e/testdata/zot/command/images/blobs/sha256/b2a5fcfb112ccde647a5a3dc0215c2c9e7d0ce598924a5ec48aa85beca048286

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"mediaType":"application/vnd.oci.image.index.v1+json","schemaVersion":2,"manifests":[]}

‎test/e2e/testdata/zot/command/images/index.json

+8
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,14 @@
4444
"architecture": "amd64",
4545
"os": "linux"
4646
}
47+
},
48+
{
49+
"mediaType": "application/vnd.oci.image.index.v1+json",
50+
"digest": "sha256:b2a5fcfb112ccde647a5a3dc0215c2c9e7d0ce598924a5ec48aa85beca048286",
51+
"size": 89,
52+
"annotations": {
53+
"org.opencontainers.image.ref.name": "empty_index"
54+
}
4755
}
4856
]
4957
}

0 commit comments

Comments
 (0)