From 2050b20ea78495a40b63cc1c05d7b6c081c47776 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Mon, 4 Oct 2021 14:56:42 +0200 Subject: [PATCH] build: add shm-size support Signed-off-by: CrazyMax --- bake/bake.go | 14 +++++++++++ bake/bake_test.go | 8 +++++++ bake/compose.go | 6 +++++ bake/compose_test.go | 43 ++++++++++++++++------------------ build/build.go | 7 ++++++ commands/bake.go | 3 +++ commands/build.go | 7 ++++-- docs/reference/buildx_bake.md | 14 +++++++---- docs/reference/buildx_build.md | 7 ++++++ 9 files changed, 80 insertions(+), 29 deletions(-) diff --git a/bake/bake.go b/bake/bake.go index 199259de500d..994f6e5ae537 100644 --- a/bake/bake.go +++ b/bake/bake.go @@ -15,6 +15,7 @@ import ( "github.com/docker/buildx/build" "github.com/docker/buildx/util/buildflags" "github.com/docker/buildx/util/platformutil" + "github.com/docker/cli/opts" "github.com/docker/docker/pkg/urlutil" hcl "github.com/hashicorp/hcl/v2" "github.com/moby/buildkit/client/llb" @@ -413,6 +414,7 @@ type Target struct { Outputs []string `json:"output,omitempty" hcl:"output,optional"` Pull *bool `json:"pull,omitempty" hcl:"pull,optional"` NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional"` + ShmSize *string `json:"shm-size,omitempty" hcl:"shm-size,optional"` // IMPORTANT: if you add more fields here, do not forget to update newOverrides and README. } @@ -479,6 +481,9 @@ func (t *Target) Merge(t2 *Target) { if t2.NoCache != nil { t.NoCache = t2.NoCache } + if t2.ShmSize != nil { // no merge + t.ShmSize = t2.ShmSize + } t.Inherits = append(t.Inherits, t2.Inherits...) } @@ -530,6 +535,8 @@ func (t *Target) AddOverrides(overrides map[string]Override) error { return errors.Errorf("invalid value %s for boolean key no-cache", value) } t.NoCache = &noCache + case "shm-size": + t.ShmSize = &value case "pull": pull, err := strconv.ParseBool(value) if err != nil { @@ -619,6 +626,12 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) { if t.Pull != nil { pull = *t.Pull } + shmSize := new(opts.MemBytes) + if t.ShmSize != nil { + if err := shmSize.Set(*t.ShmSize); err != nil { + return nil, errors.Errorf("invalid value %s for membytes key shm-size", *t.ShmSize) + } + } bi := build.Inputs{ ContextPath: contextPath, @@ -641,6 +654,7 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) { Labels: t.Labels, NoCache: noCache, Pull: pull, + ShmSize: *shmSize, } platforms, err := platformutil.Parse(t.Platforms) diff --git a/bake/bake_test.go b/bake/bake_test.go index caf43cf47088..a68bb850611e 100644 --- a/bake/bake_test.go +++ b/bake/bake_test.go @@ -20,6 +20,7 @@ target "webDEP" { VAR_BOTH = "webDEP" } no-cache = true + shm-size = "128m" } target "webapp" { @@ -42,6 +43,7 @@ target "webapp" { require.Equal(t, ".", *m["webapp"].Context) require.Equal(t, "webDEP", m["webapp"].Args["VAR_INHERITED"]) require.Equal(t, true, *m["webapp"].NoCache) + require.Equal(t, "128m", *m["webapp"].ShmSize) require.Nil(t, m["webapp"].Pull) }) @@ -111,6 +113,12 @@ target "webapp" { require.Equal(t, false, *m["webapp"].NoCache) }) + t.Run("ShmSizeOverride", func(t *testing.T) { + m, _, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.shm-size=256m"}, nil) + require.NoError(t, err) + require.Equal(t, "256m", *m["webapp"].ShmSize) + }) + t.Run("PullOverride", func(t *testing.T) { m, _, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.pull=false"}, nil) require.NoError(t, err) diff --git a/bake/compose.go b/bake/compose.go index 48c71dbe24e8..d3528cd2f2b0 100644 --- a/bake/compose.go +++ b/bake/compose.go @@ -80,6 +80,8 @@ func ParseCompose(dt []byte) (*Config, error) { return val, ok })), CacheFrom: s.Build.CacheFrom, + // TODO(@crazy-max): uncomment when available in docker/compose-go + //ShmSize: s.Build.ShmSize } if err = t.composeExtTarget(s.Build.Extensions); err != nil { return nil, err @@ -153,6 +155,10 @@ func (t *Target) composeExtTarget(exts map[string]interface{}) error { t.Secrets = append(t.Secrets, res.(string)) } } + case "shm-size": + if res, ok := val.(string); ok { + t.ShmSize = &res + } case "ssh": if res, k := val.(string); k { t.SSH = append(t.SSH, res) diff --git a/bake/compose_test.go b/bake/compose_test.go index daa5f661ea4e..7fdd578ce4d1 100644 --- a/bake/compose_test.go +++ b/bake/compose_test.go @@ -27,7 +27,7 @@ services: require.NoError(t, err) require.Equal(t, 1, len(c.Groups)) - require.Equal(t, c.Groups[0].Name, "default") + require.Equal(t, "default", c.Groups[0].Name) sort.Strings(c.Groups[0].Targets) require.Equal(t, []string{"db", "webapp"}, c.Groups[0].Targets) @@ -102,9 +102,9 @@ services: sort.Slice(c.Targets, func(i, j int) bool { return c.Targets[i].Name < c.Targets[j].Name }) - require.Equal(t, c.Targets[0].Name, "db") + require.Equal(t, "db", c.Targets[0].Name) require.Equal(t, "db", *c.Targets[0].Target) - require.Equal(t, c.Targets[1].Name, "webapp") + require.Equal(t, "webapp", c.Targets[1].Name) require.Equal(t, "webapp", *c.Targets[1].Target) } @@ -132,9 +132,9 @@ services: c, err := ParseCompose(dt) require.NoError(t, err) - require.Equal(t, c.Targets[0].Args["FOO"], "bar") - require.Equal(t, c.Targets[0].Args["BAR"], "zzz_foo") - require.Equal(t, c.Targets[0].Args["BRB"], "FOO") + require.Equal(t, "bar", c.Targets[0].Args["FOO"]) + require.Equal(t, "zzz_foo", c.Targets[0].Args["BAR"]) + require.Equal(t, "FOO", c.Targets[0].Args["BRB"]) } func TestBogusCompose(t *testing.T) { @@ -255,6 +255,7 @@ services: platforms: linux/arm64 output: type=docker no-cache: true + shm-size: 128m `) c, err := ParseCompose(dt) @@ -263,21 +264,17 @@ services: sort.Slice(c.Targets, func(i, j int) bool { return c.Targets[i].Name < c.Targets[j].Name }) - require.Equal(t, c.Targets[0].Args, map[string]string{"CT_ECR": "foo", "CT_TAG": "bar"}) - require.Equal(t, c.Targets[0].Tags, []string{"ct-addon:foo", "ct-addon:alp"}) - require.Equal(t, c.Targets[0].Platforms, []string{"linux/amd64", "linux/arm64"}) - require.Equal(t, c.Targets[0].CacheFrom, []string{"type=local,src=path/to/cache"}) - require.Equal(t, c.Targets[0].CacheTo, []string{"local,dest=path/to/cache"}) - require.Equal(t, c.Targets[0].Pull, newBool(true)) - require.Equal(t, c.Targets[1].Tags, []string{"ct-fake-aws:bar"}) - require.Equal(t, c.Targets[1].Secrets, []string{"id=mysecret,src=/local/secret", "id=mysecret2,src=/local/secret2"}) - require.Equal(t, c.Targets[1].SSH, []string{"default"}) - require.Equal(t, c.Targets[1].Platforms, []string{"linux/arm64"}) - require.Equal(t, c.Targets[1].Outputs, []string{"type=docker"}) - require.Equal(t, c.Targets[1].NoCache, newBool(true)) -} - -func newBool(val bool) *bool { - b := val - return &b + require.Equal(t, map[string]string{"CT_ECR": "foo", "CT_TAG": "bar"}, c.Targets[0].Args) + require.Equal(t, []string{"ct-addon:foo", "ct-addon:alp"}, c.Targets[0].Tags) + require.Equal(t, []string{"linux/amd64", "linux/arm64"}, c.Targets[0].Platforms) + require.Equal(t, []string{"type=local,src=path/to/cache"}, c.Targets[0].CacheFrom) + require.Equal(t, []string{"local,dest=path/to/cache"}, c.Targets[0].CacheTo) + require.Equal(t, true, *c.Targets[0].Pull) + require.Equal(t, []string{"ct-fake-aws:bar"}, c.Targets[1].Tags) + require.Equal(t, []string{"id=mysecret,src=/local/secret", "id=mysecret2,src=/local/secret2"}, c.Targets[1].Secrets) + require.Equal(t, []string{"default"}, c.Targets[1].SSH) + require.Equal(t, []string{"linux/arm64"}, c.Targets[1].Platforms) + require.Equal(t, []string{"type=docker"}, c.Targets[1].Outputs) + require.Equal(t, true, *c.Targets[1].NoCache) + require.Equal(t, "128m", *c.Targets[1].ShmSize) } diff --git a/build/build.go b/build/build.go index 21d6662a954d..ec707c817b64 100644 --- a/build/build.go +++ b/build/build.go @@ -20,6 +20,7 @@ import ( "github.com/docker/buildx/util/imagetools" "github.com/docker/buildx/util/progress" clitypes "github.com/docker/cli/cli/config/types" + "github.com/docker/cli/opts" "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" dockerclient "github.com/docker/docker/client" @@ -56,6 +57,7 @@ type Options struct { ImageIDFile string ExtraHosts []string NetworkMode string + ShmSize opts.MemBytes NoCache bool Target string @@ -554,6 +556,11 @@ func toSolveOpt(ctx context.Context, d driver.Driver, multiDriver bool, opt Opti } so.FrontendAttrs["add-hosts"] = extraHosts + // setup shm size + if opt.ShmSize.Value() > 0 { + so.FrontendAttrs["shm-size"] = strconv.FormatInt(opt.ShmSize.Value()/1024, 10) + } + return &so, releaseF, nil } diff --git a/commands/bake.go b/commands/bake.go index 844dd47676b6..a8c98dacac23 100644 --- a/commands/bake.go +++ b/commands/bake.go @@ -74,6 +74,9 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error if in.pull != nil { overrides = append(overrides, fmt.Sprintf("*.pull=%t", *in.pull)) } + if in.shmSize > 0 { + overrides = append(overrides, fmt.Sprintf("*.shm-size=%s", in.shmSize.String())) + } contextPathHash, _ := os.Getwd() ctx2, cancel := context.WithCancel(context.TODO()) diff --git a/commands/build.go b/commands/build.go index c8716d7d145b..d74a0b872607 100644 --- a/commands/build.go +++ b/commands/build.go @@ -15,6 +15,7 @@ import ( "github.com/docker/buildx/util/tracing" "github.com/docker/cli/cli" "github.com/docker/cli/cli/command" + "github.com/docker/cli/opts" "github.com/docker/docker/pkg/ioutils" "github.com/moby/buildkit/client" "github.com/moby/buildkit/session/auth/authprovider" @@ -74,6 +75,8 @@ type commonOptions struct { progress string pull *bool metadataFile string + shmSize opts.MemBytes + // golangci-lint#826 // nolint:structcheck exportPush bool @@ -125,6 +128,7 @@ func runBuild(dockerCli command.Cli, in buildOptions) (err error) { ImageIDFile: in.imageIDFile, ExtraHosts: in.extraHosts, NetworkMode: in.networkMode, + ShmSize: in.shmSize, } platforms, err := platformutil.Parse(in.platforms) @@ -323,8 +327,6 @@ func buildCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command { flags.MarkHidden("memory") flags.StringVar(&ignore, "memory-swap", "", "Swap limit equal to memory plus swap: `-1` to enable unlimited swap") flags.MarkHidden("memory-swap") - flags.StringVar(&ignore, "shm-size", "", "Size of `/dev/shm`") - flags.MarkHidden("shm-size") flags.Int64VarP(&ignoreInt, "cpu-shares", "c", 0, "CPU shares (relative weight)") flags.MarkHidden("cpu-shares") flags.Int64Var(&ignoreInt, "cpu-period", 0, "Limit the CPU CFS (Completely Fair Scheduler) period") @@ -366,6 +368,7 @@ func commonBuildFlags(options *commonOptions, flags *pflag.FlagSet) { flags.StringVar(&options.progress, "progress", "auto", "Set type of progress output (`auto`, `plain`, `tty`). Use plain to show container output") options.pull = flags.Bool("pull", false, "Always attempt to pull a newer version of the image") flags.StringVar(&options.metadataFile, "metadata-file", "", "Write build result metadata to the file") + flags.Var(&options.shmSize, "shm-size", "Size of `/dev/shm`") } func listToMap(values []string, defaultEnv bool) map[string]string { diff --git a/docs/reference/buildx_bake.md b/docs/reference/buildx_bake.md index a9e1a53e66e5..9e002da52064 100644 --- a/docs/reference/buildx_bake.md +++ b/docs/reference/buildx_bake.md @@ -25,6 +25,7 @@ Build from a file | [`--pull`](#pull) | Always attempt to pull a newer version of the image | | `--push` | Shorthand for `--set=*.output=type=registry` | | [`--set stringArray`](#set) | Override target value (e.g., `targetpattern.key=value`) | +| [`--shm-size bytes`](#shm-size) | Size of `/dev/shm` | @@ -278,6 +279,9 @@ Same as `build --pull`. Override target configurations from command line. The pattern matching syntax is defined in https://golang.org/pkg/path/#Match. +### Size of `/dev/shm` (--shm-size) + +Same as `build --shm-size`. **Examples** @@ -291,7 +295,7 @@ $ docker buildx bake --set foo*.no-cache # bypass caching only for Complete list of overridable fields: `args`, `cache-from`, `cache-to`, `context`, `dockerfile`, `labels`, `no-cache`, -`output`, `platform`, `pull`, `secrets`, `ssh`, `tags`, `target` +`output`, `platform`, `pull`, `secrets`, `shm-size`, `ssh`, `tags`, `target` ### File definition @@ -801,6 +805,7 @@ services: platforms: linux/arm64 output: type=docker no-cache: true + shm-size: 128m ``` ```console @@ -851,7 +856,8 @@ $ docker buildx bake --print "output": [ "type=docker" ], - "no-cache": true + "no-cache": true, + "shm-size": "128m" } } } @@ -859,8 +865,8 @@ $ docker buildx bake --print Complete list of valid fields for `x-bake`: -`tags`, `cache-from`, `cache-to`, `secret`, `ssh`, `platforms`, `output`, -`pull`, `no-cache` +`cache-from`, `cache-to`, `no-cache`, `platforms`, `output`, `pull`, `secret`, +`shm-size`, `ssh`, `tags` ### Built-in variables diff --git a/docs/reference/buildx_build.md b/docs/reference/buildx_build.md index db031332952f..b3f34cc89a02 100644 --- a/docs/reference/buildx_build.md +++ b/docs/reference/buildx_build.md @@ -35,6 +35,7 @@ Start a build | [`--push`](#push) | Shorthand for `--output=type=registry` | | `-q`, `--quiet` | Suppress the build output and print image ID on success | | `--secret stringArray` | Secret file to expose to the build (format: `id=mysecret,src=/local/secret`) | +| [`--shm-size bytes`](#shm-size) | Size of `/dev/shm` | | `--ssh stringArray` | SSH agent socket or keys to expose to the build (format: `default\|[=\|[,]]`) | | [`-t`](https://docs.docker.com/engine/reference/commandline/build/#tag-an-image--t), [`--tag stringArray`](https://docs.docker.com/engine/reference/commandline/build/#tag-an-image--t) | Name and optionally a tag (format: `name:tag`) | | [`--target string`](https://docs.docker.com/engine/reference/commandline/build/#specifying-target-build-stage---target) | Set the target build stage to build. | @@ -317,3 +318,9 @@ with `--allow-insecure-entitlement` (see [`create --buildkitd-flags`](buildx_cre $ docker buildx create --use --name insecure-builder --buildkitd-flags '--allow-insecure-entitlement security.insecure' $ docker buildx build --allow security.insecure . ``` + +### Size of `/dev/shm` (--shm-size) + +The format is ``. `number` must be greater than `0`. Unit is +optional and can be `b` (bytes), `k` (kilobytes), `m` (megabytes), or `g` +(gigabytes). If you omit the unit, the system uses bytes.