From ac450a6099e16c00494df65a50591bc6ff35cf7d Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 7 Nov 2024 19:11:01 +0000 Subject: [PATCH 1/5] fix: avoid BLOB_UNKNOWN by not enabling ForceBuildMetadata Ref: #385 Ref: coder/kaniko#34 --- envbuilder.go | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index e6f9c8d4..f6cfc844 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -541,7 +541,7 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro NoPush: !opts.PushImage || len(destinations) == 0, CacheRunLayers: true, CacheCopyLayers: true, - ForceBuildMetadata: opts.PushImage, // Force layers with no changes to be cached, required for cache probing. + ForceBuildMetadata: false, // Force layers with no changes to be cached, required for cache probing. CompressedCaching: true, Compression: config.ZStd, // Maps to "default" level, ~100-300 MB/sec according to @@ -1269,7 +1269,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) NoPush: true, CacheRunLayers: true, CacheCopyLayers: true, - ForceBuildMetadata: true, // Force layers with no changes to be cached, required for cache probing. + ForceBuildMetadata: false, // Force layers with no changes to be cached, required for cache probing. CompressedCaching: true, Compression: config.ZStd, // Maps to "default" level, ~100-300 MB/sec according to diff --git a/go.mod b/go.mod index e066b206..9b070ff8 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22.4 // There are a few options we need added to Kaniko! // See: https://github.com/GoogleContainerTools/kaniko/compare/main...coder:kaniko:main -replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20241028054616-350cbb820e05 +replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20241107190527-8aeb946dfc4d // Required to import codersdk due to gvisor dependency. replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20240702054557-aa558fbe5374 diff --git a/go.sum b/go.sum index 5872942d..19317cba 100644 --- a/go.sum +++ b/go.sum @@ -171,8 +171,8 @@ github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoC github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352 h1:L/EjCuZxs5tOcqqCaASj/nu65TRYEFcTt8qRQfHZXX0= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352/go.mod h1:P1KoQSgnKEAG6Mnd3YlGzAophty+yKA9VV48LpfNRvo= -github.com/coder/kaniko v0.0.0-20241028054616-350cbb820e05 h1:KZc6vG/WnSWG8RtUevGrCdZbF7XJaaZ32ocig6sZLQk= -github.com/coder/kaniko v0.0.0-20241028054616-350cbb820e05/go.mod h1:3rM/KOQ4LgF8mE+O1P6pLDa/E57mzxIxNdUOMKi1qpg= +github.com/coder/kaniko v0.0.0-20241107190527-8aeb946dfc4d h1:qqoyQMtIHWYRw8RbDPT+pdOQER9UKm36tsJtiy9MVc8= +github.com/coder/kaniko v0.0.0-20241107190527-8aeb946dfc4d/go.mod h1:3rM/KOQ4LgF8mE+O1P6pLDa/E57mzxIxNdUOMKi1qpg= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 h1:3A0ES21Ke+FxEM8CXx9n47SZOKOpgSE1bbJzlE4qPVs= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0/go.mod h1:5UuS2Ts+nTToAMeOjNlnHFkPahrtDkmpydBen/3wgZc= github.com/coder/quartz v0.1.0 h1:cLL+0g5l7xTf6ordRnUMMiZtRE8Sq5LxpghS63vEXrQ= From 9df56fca08f5e8031bdddf92a101bb7627a9e4af Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 7 Nov 2024 19:12:40 +0000 Subject: [PATCH 2/5] dbg --- envbuilder.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/envbuilder.go b/envbuilder.go index f6cfc844..dda35d9c 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "io/fs" + stdlog "log" "maps" "net" "net/http" @@ -43,6 +44,7 @@ import ( dockerconfig "github.com/docker/cli/cli/config" "github.com/docker/cli/cli/config/configfile" "github.com/fatih/color" + "github.com/google/go-containerregistry/pkg/logs" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/kballard/go-shellquote" @@ -583,6 +585,19 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro endStage("🏗️ Built image!") if opts.PushImage { endStage = startStage("🏗️ Pushing image...") + kOpts.PushRetry = 3 + // kOpts.PushIgnoreImmutableTagErrors = true + logs.Debug = stdlog.New(os.Stderr, "", 0) + logs.Warn = stdlog.New(os.Stderr, "", 0) + logs.Progress = stdlog.New(os.Stderr, "", 0) + layers, _ := image.Layers() + for _, layer := range layers { + mediaType, _ := layer.MediaType() + diffID, _ := layer.DiffID() + digest, _ := layer.Digest() + size, _ := layer.Size() + opts.Logger(log.LevelDebug, "Layer: %s %s %s %d", mediaType, diffID, digest, size) + } if err := executor.DoPush(image, kOpts); err == nil { endStage("🏗️ Pushed image!") } else if !opts.ExitOnPushFailure { From 4b71b2416e07ad6e61e877193236f729f4cc55ce Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Wed, 20 Nov 2024 13:10:05 +0000 Subject: [PATCH 3/5] update fork and add test --- Makefile | 6 ++- envbuilder.go | 29 ++++++-------- go.mod | 2 +- go.sum | 4 +- integration/integration_test.go | 41 ++++++++++++++++++-- integration/testdata/blob-unknown/Dockerfile | 3 ++ 6 files changed, 61 insertions(+), 24 deletions(-) create mode 100644 integration/testdata/blob-unknown/Dockerfile diff --git a/Makefile b/Makefile index ce079015..14ed5182 100644 --- a/Makefile +++ b/Makefile @@ -67,7 +67,7 @@ test-registry-container: .registry-cache # Pulls images referenced in integration tests and pushes them to the local cache. .PHONY: test-images-push -test-images-push: .registry-cache/docker/registry/v2/repositories/envbuilder-test-alpine .registry-cache/docker/registry/v2/repositories/envbuilder-test-ubuntu .registry-cache/docker/registry/v2/repositories/envbuilder-test-codercom-code-server +test-images-push: .registry-cache/docker/registry/v2/repositories/envbuilder-test-alpine .registry-cache/docker/registry/v2/repositories/envbuilder-test-ubuntu .registry-cache/docker/registry/v2/repositories/envbuilder-test-codercom-code-server .registry-cache/docker/registry/v2/repositories/envbuilder-test-blob-unknown .PHONY: test-images-pull test-images-pull: @@ -77,6 +77,7 @@ test-images-pull: docker tag ubuntu:latest localhost:5000/envbuilder-test-ubuntu:latest docker pull codercom/code-server:latest docker tag codercom/code-server:latest localhost:5000/envbuilder-test-codercom-code-server:latest + docker build -t localhost:5000/envbuilder-test-blob-unknown:latest -f integration/testdata/blob-unknown/Dockerfile integration/testdata/blob-unknown .registry-cache: mkdir -p .registry-cache && chmod -R ag+w .registry-cache @@ -89,3 +90,6 @@ test-images-pull: .registry-cache/docker/registry/v2/repositories/envbuilder-test-codercom-code-server: docker push localhost:5000/envbuilder-test-codercom-code-server:latest + +.registry-cache/docker/registry/v2/repositories/envbuilder-test-blob-unknown: + docker push localhost:5000/envbuilder-test-blob-unknown:latest diff --git a/envbuilder.go b/envbuilder.go index dda35d9c..2d7efec3 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -9,7 +9,6 @@ import ( "fmt" "io" "io/fs" - stdlog "log" "maps" "net" "net/http" @@ -44,7 +43,6 @@ import ( dockerconfig "github.com/docker/cli/cli/config" "github.com/docker/cli/cli/config/configfile" "github.com/fatih/color" - "github.com/google/go-containerregistry/pkg/logs" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/kballard/go-shellquote" @@ -543,7 +541,7 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro NoPush: !opts.PushImage || len(destinations) == 0, CacheRunLayers: true, CacheCopyLayers: true, - ForceBuildMetadata: false, // Force layers with no changes to be cached, required for cache probing. + ForceBuildMetadata: opts.PushImage, // Force layers with no changes to be cached, required for cache probing. CompressedCaching: true, Compression: config.ZStd, // Maps to "default" level, ~100-300 MB/sec according to @@ -585,19 +583,16 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro endStage("🏗️ Built image!") if opts.PushImage { endStage = startStage("🏗️ Pushing image...") - kOpts.PushRetry = 3 - // kOpts.PushIgnoreImmutableTagErrors = true - logs.Debug = stdlog.New(os.Stderr, "", 0) - logs.Warn = stdlog.New(os.Stderr, "", 0) - logs.Progress = stdlog.New(os.Stderr, "", 0) - layers, _ := image.Layers() - for _, layer := range layers { - mediaType, _ := layer.MediaType() - diffID, _ := layer.DiffID() - digest, _ := layer.Digest() - size, _ := layer.Size() - opts.Logger(log.LevelDebug, "Layer: %s %s %s %d", mediaType, diffID, digest, size) - } + + // To debug registry issues, enable logging: + // + // import ( + // stdlog "log" + // reglogs "github.com/google/go-containerregistry/pkg/logs" + // ) + // reglogs.Debug = stdlog.New(os.Stderr, "", 0) + // reglogs.Warn = stdlog.New(os.Stderr, "", 0) + // reglogs.Progress = stdlog.New(os.Stderr, "", 0) if err := executor.DoPush(image, kOpts); err == nil { endStage("🏗️ Pushed image!") } else if !opts.ExitOnPushFailure { @@ -1284,7 +1279,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) NoPush: true, CacheRunLayers: true, CacheCopyLayers: true, - ForceBuildMetadata: false, // Force layers with no changes to be cached, required for cache probing. + ForceBuildMetadata: true, // Force layers with no changes to be cached, required for cache probing. CompressedCaching: true, Compression: config.ZStd, // Maps to "default" level, ~100-300 MB/sec according to diff --git a/go.mod b/go.mod index 9b070ff8..a9427751 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22.4 // There are a few options we need added to Kaniko! // See: https://github.com/GoogleContainerTools/kaniko/compare/main...coder:kaniko:main -replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20241107190527-8aeb946dfc4d +replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20241120115946-327ae3f1c802 // Required to import codersdk due to gvisor dependency. replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20240702054557-aa558fbe5374 diff --git a/go.sum b/go.sum index 19317cba..c0ab432f 100644 --- a/go.sum +++ b/go.sum @@ -171,8 +171,8 @@ github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoC github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352 h1:L/EjCuZxs5tOcqqCaASj/nu65TRYEFcTt8qRQfHZXX0= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352/go.mod h1:P1KoQSgnKEAG6Mnd3YlGzAophty+yKA9VV48LpfNRvo= -github.com/coder/kaniko v0.0.0-20241107190527-8aeb946dfc4d h1:qqoyQMtIHWYRw8RbDPT+pdOQER9UKm36tsJtiy9MVc8= -github.com/coder/kaniko v0.0.0-20241107190527-8aeb946dfc4d/go.mod h1:3rM/KOQ4LgF8mE+O1P6pLDa/E57mzxIxNdUOMKi1qpg= +github.com/coder/kaniko v0.0.0-20241120115946-327ae3f1c802 h1:6vcTuyI9pLlmEkSOw0cT+T74DuqYWMcBJFGj+32C7qk= +github.com/coder/kaniko v0.0.0-20241120115946-327ae3f1c802/go.mod h1:3rM/KOQ4LgF8mE+O1P6pLDa/E57mzxIxNdUOMKi1qpg= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 h1:3A0ES21Ke+FxEM8CXx9n47SZOKOpgSE1bbJzlE4qPVs= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0/go.mod h1:5UuS2Ts+nTToAMeOjNlnHFkPahrtDkmpydBen/3wgZc= github.com/coder/quartz v0.1.0 h1:cLL+0g5l7xTf6ordRnUMMiZtRE8Sq5LxpghS63vEXrQ= diff --git a/integration/integration_test.go b/integration/integration_test.go index aaa25d16..b7e75438 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -57,9 +57,10 @@ import ( ) const ( - testContainerLabel = "envbox-integration-test" - testImageAlpine = "localhost:5000/envbuilder-test-alpine:latest" - testImageUbuntu = "localhost:5000/envbuilder-test-ubuntu:latest" + testContainerLabel = "envbox-integration-test" + testImageAlpine = "localhost:5000/envbuilder-test-alpine:latest" + testImageUbuntu = "localhost:5000/envbuilder-test-ubuntu:latest" + testImageBlobUnknown = "localhost:5000/envbuilder-test-blob-unknown:latest" // nolint:gosec // Throw-away key for testing. DO NOT REUSE. testSSHKey = `-----BEGIN OPENSSH PRIVATE KEY----- @@ -2354,6 +2355,38 @@ USER devalot } require.Fail(t, "expected pid 1 to be running as devalot") }) + + t.Run("PushDuplicateLayersNoBlobUnknown", func(t *testing.T) { + t.Parallel() + + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ + ".devcontainer/Dockerfile": fmt.Sprintf(`FROM %s +USER root +RUN echo "hi i r empty" +RUN echo "who u" +`, testImageBlobUnknown), + ".devcontainer/devcontainer.json": `{ + "name": "Test", + "build": { + "dockerfile": "Dockerfile" + }, + }`, + }, + }) + + // NOTE(mafredri): The in-memory registry doesn't catch this error so we + // have to use registry:2. + ref, err := name.ParseReference(fmt.Sprintf("localhost:5000/test-blob-unknown-%s", uuid.NewString())) + require.NoError(t, err) + opts := []string{ + envbuilderEnv("GIT_URL", srv.URL), + envbuilderEnv("CACHE_REPO", ref.String()), + envbuilderEnv("VERBOSE", "1"), + } + + _ = pushImage(t, ref, nil, opts...) + }) } func TestChownHomedir(t *testing.T) { @@ -2532,6 +2565,8 @@ func getCachedImage(ctx context.Context, t *testing.T, cli *client.Client, env . } func startContainerFromRef(ctx context.Context, t *testing.T, cli *client.Client, ref name.Reference) container.CreateResponse { + t.Helper() + // Ensure that we can pull the image. rc, err := cli.ImagePull(ctx, ref.String(), image.PullOptions{}) require.NoError(t, err) diff --git a/integration/testdata/blob-unknown/Dockerfile b/integration/testdata/blob-unknown/Dockerfile new file mode 100644 index 00000000..614d04a7 --- /dev/null +++ b/integration/testdata/blob-unknown/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:latest + +WORKDIR /home From 9347b30c556de8bf8b956b2e8846edb2429b22c8 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Wed, 20 Nov 2024 13:14:05 +0000 Subject: [PATCH 4/5] Commentary --- integration/testdata/blob-unknown/Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/integration/testdata/blob-unknown/Dockerfile b/integration/testdata/blob-unknown/Dockerfile index 614d04a7..fffcc574 100644 --- a/integration/testdata/blob-unknown/Dockerfile +++ b/integration/testdata/blob-unknown/Dockerfile @@ -1,3 +1,7 @@ FROM alpine:latest +# This will produce an empty layer via Docker. It will allow us to test for a +# conflicting empty layer produced by Kaniko. This is to check against the +# BLOB_UNKNOWN error when trying to upload the built image to a registry and +# Kaniko having overwritten this blob with its own. WORKDIR /home From d72eaff987d73cf7e001f790e6e423eb1cbeb506 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Wed, 20 Nov 2024 13:22:36 +0000 Subject: [PATCH 5/5] fork you very much --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a9427751..00987aa7 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22.4 // There are a few options we need added to Kaniko! // See: https://github.com/GoogleContainerTools/kaniko/compare/main...coder:kaniko:main -replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20241120115946-327ae3f1c802 +replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20241120132148-131d6094d781 // Required to import codersdk due to gvisor dependency. replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20240702054557-aa558fbe5374 diff --git a/go.sum b/go.sum index c0ab432f..1bb43776 100644 --- a/go.sum +++ b/go.sum @@ -171,8 +171,8 @@ github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoC github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352 h1:L/EjCuZxs5tOcqqCaASj/nu65TRYEFcTt8qRQfHZXX0= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352/go.mod h1:P1KoQSgnKEAG6Mnd3YlGzAophty+yKA9VV48LpfNRvo= -github.com/coder/kaniko v0.0.0-20241120115946-327ae3f1c802 h1:6vcTuyI9pLlmEkSOw0cT+T74DuqYWMcBJFGj+32C7qk= -github.com/coder/kaniko v0.0.0-20241120115946-327ae3f1c802/go.mod h1:3rM/KOQ4LgF8mE+O1P6pLDa/E57mzxIxNdUOMKi1qpg= +github.com/coder/kaniko v0.0.0-20241120132148-131d6094d781 h1:/4SMdrjLQL1BseLSnMd9nYQSI+E63CXcyFGC7ZHHj8I= +github.com/coder/kaniko v0.0.0-20241120132148-131d6094d781/go.mod h1:3rM/KOQ4LgF8mE+O1P6pLDa/E57mzxIxNdUOMKi1qpg= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 h1:3A0ES21Ke+FxEM8CXx9n47SZOKOpgSE1bbJzlE4qPVs= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0/go.mod h1:5UuS2Ts+nTToAMeOjNlnHFkPahrtDkmpydBen/3wgZc= github.com/coder/quartz v0.1.0 h1:cLL+0g5l7xTf6ordRnUMMiZtRE8Sq5LxpghS63vEXrQ=