build: add shm-size support

Signed-off-by: CrazyMax <[email protected]>

bake/bake.go

@@ -15,6 +15,7 @@ import (
 	"github.com/docker/buildx/build"
 	"github.com/docker/buildx/util/buildflags"
 	"github.com/docker/buildx/util/platformutil"
+	"github.com/docker/cli/opts"
 	"github.com/docker/docker/pkg/urlutil"
 	hcl "github.com/hashicorp/hcl/v2"
 	"github.com/moby/buildkit/client/llb"
@@ -413,6 +414,7 @@ type Target struct {
 	Outputs          []string          `json:"output,omitempty" hcl:"output,optional"`
 	Pull             *bool             `json:"pull,omitempty" hcl:"pull,optional"`
 	NoCache          *bool             `json:"no-cache,omitempty" hcl:"no-cache,optional"`
+	ShmSize          *string           `json:"shm-size,omitempty" hcl:"shm-size,optional"`
 
 	// IMPORTANT: if you add more fields here, do not forget to update newOverrides and README.
 }
@@ -479,6 +481,9 @@ func (t *Target) Merge(t2 *Target) {
 	if t2.NoCache != nil {
 		t.NoCache = t2.NoCache
 	}
+	if t2.ShmSize != nil { // no merge
+		t.ShmSize = t2.ShmSize
+	}
 	t.Inherits = append(t.Inherits, t2.Inherits...)
 }
 
@@ -530,6 +535,8 @@ func (t *Target) AddOverrides(overrides map[string]Override) error {
 				return errors.Errorf("invalid value %s for boolean key no-cache", value)
 			}
 			t.NoCache = &noCache
+		case "shm-size":
+			t.ShmSize = &value
 		case "pull":
 			pull, err := strconv.ParseBool(value)
 			if err != nil {
@@ -619,6 +626,12 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
 	if t.Pull != nil {
 		pull = *t.Pull
 	}
+	shmSize := new(opts.MemBytes)
+	if t.ShmSize != nil {
+		if err := shmSize.Set(*t.ShmSize); err != nil {
+			return nil, errors.Errorf("invalid value %s for membytes key shm-size", *t.ShmSize)
+		}
+	}
 
 	bi := build.Inputs{
 		ContextPath:    contextPath,
@@ -641,6 +654,7 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
 		Labels:    t.Labels,
 		NoCache:   noCache,
 		Pull:      pull,
+		ShmSize:   *shmSize,
 	}
 
 	platforms, err := platformutil.Parse(t.Platforms)

bake/bake_test.go

@@ -20,6 +20,7 @@ target "webDEP" {
 		VAR_BOTH = "webDEP"
 	}
 	no-cache = true
+	shm-size = "128m"
 }
 
 target "webapp" {
@@ -42,6 +43,7 @@ target "webapp" {
 		require.Equal(t, ".", *m["webapp"].Context)
 		require.Equal(t, "webDEP", m["webapp"].Args["VAR_INHERITED"])
 		require.Equal(t, true, *m["webapp"].NoCache)
+		require.Equal(t, "128m", *m["webapp"].ShmSize)
 		require.Nil(t, m["webapp"].Pull)
 	})
 
@@ -111,6 +113,12 @@ target "webapp" {
 		require.Equal(t, false, *m["webapp"].NoCache)
 	})
 
+	t.Run("ShmSizeOverride", func(t *testing.T) {
+		m, _, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.shm-size=256m"}, nil)
+		require.NoError(t, err)
+		require.Equal(t, "256m", *m["webapp"].ShmSize)
+	})
+
 	t.Run("PullOverride", func(t *testing.T) {
 		m, _, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.pull=false"}, nil)
 		require.NoError(t, err)

bake/compose.go

@@ -80,6 +80,8 @@ func ParseCompose(dt []byte) (*Config, error) {
 					return val, ok
 				})),
 				CacheFrom: s.Build.CacheFrom,
+				// TODO(@crazy-max): uncomment when available in docker/compose-go
+				//ShmSize: s.Build.ShmSize
 			}
 			if err = t.composeExtTarget(s.Build.Extensions); err != nil {
 				return nil, err
@@ -153,6 +155,10 @@ func (t *Target) composeExtTarget(exts map[string]interface{}) error {
 						t.Secrets = append(t.Secrets, res.(string))
 					}
 				}
+			case "shm-size":
+				if res, ok := val.(string); ok {
+					t.ShmSize = &res
+				}
 			case "ssh":
 				if res, k := val.(string); k {
 					t.SSH = append(t.SSH, res)

bake/compose_test.go

@@ -27,7 +27,7 @@ services:
 	require.NoError(t, err)
 
 	require.Equal(t, 1, len(c.Groups))
-	require.Equal(t, c.Groups[0].Name, "default")
+	require.Equal(t, "default", c.Groups[0].Name)
 	sort.Strings(c.Groups[0].Targets)
 	require.Equal(t, []string{"db", "webapp"}, c.Groups[0].Targets)
 
@@ -102,9 +102,9 @@ services:
 	sort.Slice(c.Targets, func(i, j int) bool {
 		return c.Targets[i].Name < c.Targets[j].Name
 	})
-	require.Equal(t, c.Targets[0].Name, "db")
+	require.Equal(t, "db", c.Targets[0].Name)
 	require.Equal(t, "db", *c.Targets[0].Target)
-	require.Equal(t, c.Targets[1].Name, "webapp")
+	require.Equal(t, "webapp", c.Targets[1].Name)
 	require.Equal(t, "webapp", *c.Targets[1].Target)
 }
 
@@ -132,9 +132,9 @@ services:
 
 	c, err := ParseCompose(dt)
 	require.NoError(t, err)
-	require.Equal(t, c.Targets[0].Args["FOO"], "bar")
-	require.Equal(t, c.Targets[0].Args["BAR"], "zzz_foo")
-	require.Equal(t, c.Targets[0].Args["BRB"], "FOO")
+	require.Equal(t, "bar", c.Targets[0].Args["FOO"])
+	require.Equal(t, "zzz_foo", c.Targets[0].Args["BAR"])
+	require.Equal(t, "FOO", c.Targets[0].Args["BRB"])
 }
 
 func TestBogusCompose(t *testing.T) {
@@ -255,6 +255,7 @@ services:
         platforms: linux/arm64
         output: type=docker
         no-cache: true
+        shm-size: 128m
 `)
 
 	c, err := ParseCompose(dt)
@@ -263,21 +264,17 @@ services:
 	sort.Slice(c.Targets, func(i, j int) bool {
 		return c.Targets[i].Name < c.Targets[j].Name
 	})
-	require.Equal(t, c.Targets[0].Args, map[string]string{"CT_ECR": "foo", "CT_TAG": "bar"})
-	require.Equal(t, c.Targets[0].Tags, []string{"ct-addon:foo", "ct-addon:alp"})
-	require.Equal(t, c.Targets[0].Platforms, []string{"linux/amd64", "linux/arm64"})
-	require.Equal(t, c.Targets[0].CacheFrom, []string{"type=local,src=path/to/cache"})
-	require.Equal(t, c.Targets[0].CacheTo, []string{"local,dest=path/to/cache"})
-	require.Equal(t, c.Targets[0].Pull, newBool(true))
-	require.Equal(t, c.Targets[1].Tags, []string{"ct-fake-aws:bar"})
-	require.Equal(t, c.Targets[1].Secrets, []string{"id=mysecret,src=/local/secret", "id=mysecret2,src=/local/secret2"})
-	require.Equal(t, c.Targets[1].SSH, []string{"default"})
-	require.Equal(t, c.Targets[1].Platforms, []string{"linux/arm64"})
-	require.Equal(t, c.Targets[1].Outputs, []string{"type=docker"})
-	require.Equal(t, c.Targets[1].NoCache, newBool(true))
-}
-
-func newBool(val bool) *bool {
-	b := val
-	return &b
+	require.Equal(t, map[string]string{"CT_ECR": "foo", "CT_TAG": "bar"}, c.Targets[0].Args)
+	require.Equal(t, []string{"ct-addon:foo", "ct-addon:alp"}, c.Targets[0].Tags)
+	require.Equal(t, []string{"linux/amd64", "linux/arm64"}, c.Targets[0].Platforms)
+	require.Equal(t, []string{"type=local,src=path/to/cache"}, c.Targets[0].CacheFrom)
+	require.Equal(t, []string{"local,dest=path/to/cache"}, c.Targets[0].CacheTo)
+	require.Equal(t, true, *c.Targets[0].Pull)
+	require.Equal(t, []string{"ct-fake-aws:bar"}, c.Targets[1].Tags)
+	require.Equal(t, []string{"id=mysecret,src=/local/secret", "id=mysecret2,src=/local/secret2"}, c.Targets[1].Secrets)
+	require.Equal(t, []string{"default"}, c.Targets[1].SSH)
+	require.Equal(t, []string{"linux/arm64"}, c.Targets[1].Platforms)
+	require.Equal(t, []string{"type=docker"}, c.Targets[1].Outputs)
+	require.Equal(t, true, *c.Targets[1].NoCache)
+	require.Equal(t, "128m", *c.Targets[1].ShmSize)
 }

build/build.go

@@ -20,6 +20,7 @@ import (
 	"github.com/docker/buildx/util/imagetools"
 	"github.com/docker/buildx/util/progress"
 	clitypes "github.com/docker/cli/cli/config/types"
+	"github.com/docker/cli/opts"
 	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types"
 	dockerclient "github.com/docker/docker/client"
@@ -56,6 +57,7 @@ type Options struct {
 	ImageIDFile string
 	ExtraHosts  []string
 	NetworkMode string
+	ShmSize     opts.MemBytes
 
 	NoCache   bool
 	Target    string
@@ -554,6 +556,11 @@ func toSolveOpt(ctx context.Context, d driver.Driver, multiDriver bool, opt Opti
 	}
 	so.FrontendAttrs["add-hosts"] = extraHosts
 
+	// setup shm size
+	if opt.ShmSize.Value() > 0 {
+		so.FrontendAttrs["shm-size"] = strconv.FormatInt(opt.ShmSize.Value()/1024, 10)
+	}
+
 	return &so, releaseF, nil
 }
 

commands/bake.go

@@ -74,6 +74,9 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error
 	if in.pull != nil {
 		overrides = append(overrides, fmt.Sprintf("*.pull=%t", *in.pull))
 	}
+	if in.shmSize > 0 {
+		overrides = append(overrides, fmt.Sprintf("*.shm-size=%s", in.shmSize.String()))
+	}
 	contextPathHash, _ := os.Getwd()
 
 	ctx2, cancel := context.WithCancel(context.TODO())

commands/build.go

@@ -15,6 +15,7 @@ import (
 	"github.com/docker/buildx/util/tracing"
 	"github.com/docker/cli/cli"
 	"github.com/docker/cli/cli/command"
+	"github.com/docker/cli/opts"
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/session/auth/authprovider"
@@ -74,6 +75,8 @@ type commonOptions struct {
 	progress     string
 	pull         *bool
 	metadataFile string
+	shmSize      opts.MemBytes
+
 	// golangci-lint#826
 	// nolint:structcheck
 	exportPush bool
@@ -125,6 +128,7 @@ func runBuild(dockerCli command.Cli, in buildOptions) (err error) {
 		ImageIDFile: in.imageIDFile,
 		ExtraHosts:  in.extraHosts,
 		NetworkMode: in.networkMode,
+		ShmSize:     in.shmSize,
 	}
 
 	platforms, err := platformutil.Parse(in.platforms)
@@ -323,8 +327,6 @@ func buildCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
 	flags.MarkHidden("memory")
 	flags.StringVar(&ignore, "memory-swap", "", "Swap limit equal to memory plus swap: `-1` to enable unlimited swap")
 	flags.MarkHidden("memory-swap")
-	flags.StringVar(&ignore, "shm-size", "", "Size of `/dev/shm`")
-	flags.MarkHidden("shm-size")
 	flags.Int64VarP(&ignoreInt, "cpu-shares", "c", 0, "CPU shares (relative weight)")
 	flags.MarkHidden("cpu-shares")
 	flags.Int64Var(&ignoreInt, "cpu-period", 0, "Limit the CPU CFS (Completely Fair Scheduler) period")
@@ -366,6 +368,7 @@ func commonBuildFlags(options *commonOptions, flags *pflag.FlagSet) {
 	flags.StringVar(&options.progress, "progress", "auto", "Set type of progress output (`auto`, `plain`, `tty`). Use plain to show container output")
 	options.pull = flags.Bool("pull", false, "Always attempt to pull a newer version of the image")
 	flags.StringVar(&options.metadataFile, "metadata-file", "", "Write build result metadata to the file")
+	flags.Var(&options.shmSize, "shm-size", "Size of `/dev/shm`")
 }
 
 func listToMap(values []string, defaultEnv bool) map[string]string {

docs/reference/buildx_bake.md

@@ -25,6 +25,7 @@ Build from a file
 | [`--pull`](#pull) | Always attempt to pull a newer version of the image |
 | `--push` | Shorthand for `--set=*.output=type=registry` |
 | [`--set stringArray`](#set) | Override target value (e.g., `targetpattern.key=value`) |
+| [`--shm-size bytes`](#shm-size) | Size of `/dev/shm` |
 
 
 <!---MARKER_GEN_END-->
@@ -278,6 +279,9 @@ Same as `build --pull`.
 Override target configurations from command line. The pattern matching syntax
 is defined in https://golang.org/pkg/path/#Match.
 
+### <a name="shm-size"></a> Size of `/dev/shm` (--shm-size)
+
+Same as `build --shm-size`.
 
 **Examples**
 
@@ -291,7 +295,7 @@ $ docker buildx bake --set foo*.no-cache              # bypass caching only for
 
 Complete list of overridable fields:
 `args`, `cache-from`, `cache-to`, `context`, `dockerfile`, `labels`, `no-cache`,
-`output`, `platform`, `pull`, `secrets`, `ssh`, `tags`, `target`
+`output`, `platform`, `pull`, `secrets`, `shm-size`, `ssh`, `tags`, `target`
 
 ### File definition
 
@@ -801,6 +805,7 @@ services:
         platforms: linux/arm64
         output: type=docker
         no-cache: true
+        shm-size: 128m
 ```
 
 ```console
@@ -851,16 +856,17 @@ $ docker buildx bake --print
       "output": [
         "type=docker"
       ],
-      "no-cache": true
+      "no-cache": true,
+      "shm-size": "128m"
     }
   }
 }
 ```
 
 Complete list of valid fields for `x-bake`:
 
-`tags`, `cache-from`, `cache-to`, `secret`, `ssh`, `platforms`, `output`,
-`pull`, `no-cache`
+`cache-from`, `cache-to`, `no-cache`, `platforms`, `output`, `pull`, `secret`,
+`shm-size`, `ssh`, `tags`
 
 ### Built-in variables
 

docs/reference/buildx_build.md

@@ -35,6 +35,7 @@ Start a build
 | [`--push`](#push) | Shorthand for `--output=type=registry` |
 | `-q`, `--quiet` | Suppress the build output and print image ID on success |
 | `--secret stringArray` | Secret file to expose to the build (format: `id=mysecret,src=/local/secret`) |
+| [`--shm-size bytes`](#shm-size) | Size of `/dev/shm` |
 | `--ssh stringArray` | SSH agent socket or keys to expose to the build (format: `default\|<id>[=<socket>\|<key>[,<key>]]`) |
 | [`-t`](https://docs.docker.com/engine/reference/commandline/build/#tag-an-image--t), [`--tag stringArray`](https://docs.docker.com/engine/reference/commandline/build/#tag-an-image--t) | Name and optionally a tag (format: `name:tag`) |
 | [`--target string`](https://docs.docker.com/engine/reference/commandline/build/#specifying-target-build-stage---target) | Set the target build stage to build. |
@@ -317,3 +318,9 @@ with `--allow-insecure-entitlement` (see [`create --buildkitd-flags`](buildx_cre
 $ docker buildx create --use --name insecure-builder --buildkitd-flags '--allow-insecure-entitlement security.insecure'
 $ docker buildx build --allow security.insecure .
 ```
+
+### <a name="shm-size"></a> Size of `/dev/shm` (--shm-size)
+
+The format is `<number><unit>`. `number` must be greater than `0`. Unit is
+optional and can be `b` (bytes), `k` (kilobytes), `m` (megabytes), or `g`
+(gigabytes). If you omit the unit, the system uses bytes.