NET-12097 Support PodSecurityAdmission at the restricted level (#4478)

* Set API gateway security context to comply with best practices

* update deployment security context

* Set SeccompProfile on injected dataplane sidecar

* Drop all capabilities in the injected sidecar

* Set required securityContext properties on connect-inject-init container

* Add changelog entry

* May it please the linter

* May it please the linter

* Add helpful logs to failure message assertion

* Set default value for API gateway's mapPrivilegedContainerPorts

* Update invalidated unit tests

* May it please the linter

* Allow privilege escalation when expected for backwards compatibility

* add init sec comp to mesh gateway deployment

* Update invalidated unit tests

---------

Co-authored-by: Sarah Alsmiller <[email protected]>

Author: nathancoleman

.changelog/4478.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+security: Support running Consul under Pod Security Admissions (PSA) restricted mode.
+```

acceptance/framework/k8s/deploy.go

@@ -11,13 +11,14 @@ import (
 	"time"
 
 	"github.com/gruntwork-io/terratest/modules/k8s"
-	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
-	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
 	"github.com/hashicorp/consul/sdk/testutil/retry"
 	"github.com/stretchr/testify/require"
 	v1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
 	"k8s.io/apimachinery/pkg/util/yaml"
+
+	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
+	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
 )
 
 // Deploy creates a Kubernetes deployment by applying configuration stored at filepath,
@@ -141,15 +142,15 @@ func CheckStaticServerConnectionMultipleFailureMessages(t *testing.T, options *k
 			require.Contains(r, output, expectedOutput)
 		} else {
 			require.Error(r, err)
-			require.Condition(r, func() bool {
+			require.Conditionf(r, func() bool {
 				exists := false
 				for _, msg := range failureMessages {
 					if strings.Contains(output, msg) {
 						exists = true
 					}
 				}
 				return exists
-			})
+			}, "expected failure messages %q but got %q", failureMessages, output)
 		}
 	})
 }

acceptance/tests/api-gateway/api_gateway_external_servers_test.go

@@ -8,15 +8,16 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/hashicorp/consul-k8s/acceptance/framework/consul"
-	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
-	"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
-	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
 	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/consul/sdk/testutil/retry"
 	"github.com/stretchr/testify/require"
 	"k8s.io/apimachinery/pkg/types"
 	gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
+
+	"github.com/hashicorp/consul-k8s/acceptance/framework/consul"
+	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
+	"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
+	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
 )
 
 // TestAPIGateway_ExternalServers tests that connect works when using external servers.

charts/consul/templates/mesh-gateway-deployment.yaml

@@ -126,6 +126,15 @@ spec:
       {{- end }}
       initContainers:
       - name: mesh-gateway-init
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+          seccompProfile:
+            type: RuntimeDefault
+          capabilities:
+            drop:
+              - ALL
         image: {{ .Values.global.imageK8S }}
         {{ template "consul.imagePullPolicy" . }}
         env:
@@ -188,6 +197,11 @@ spec:
         image: {{ .Values.global.imageConsulDataplane | quote }}
         {{ template "consul.imagePullPolicy" . }}
         securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+          seccompProfile:
+            type: RuntimeDefault
           capabilities:
             {{ if not .Values.meshGateway.hostNetwork}}
             drop:

control-plane/api-gateway/gatekeeper/dataplane.go

@@ -10,6 +10,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/ptr"
+	gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
 
 	"github.com/hashicorp/consul-k8s/control-plane/api-gateway/common"
 	"github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1"
@@ -27,7 +28,7 @@ const (
 	volumeNameForTLSCerts        = "consul-gateway-tls-certificates"
 )
 
-func consulDataplaneContainer(metrics common.MetricsConfig, config common.HelmConfig, gcc v1alpha1.GatewayClassConfig, name, namespace string, mounts []corev1.VolumeMount) (corev1.Container, error) {
+func consulDataplaneContainer(metrics common.MetricsConfig, config common.HelmConfig, gcc v1alpha1.GatewayClassConfig, gateway gwv1beta1.Gateway, mounts []corev1.VolumeMount) (corev1.Container, error) {
 	// Extract the service account token's volume mount.
 	var (
 		err             error
@@ -38,7 +39,7 @@ func consulDataplaneContainer(metrics common.MetricsConfig, config common.HelmCo
 		bearerTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token"
 	}
 
-	args, err := getDataplaneArgs(metrics, namespace, config, bearerTokenFile, name)
+	args, err := getDataplaneArgs(metrics, gateway.Namespace, config, bearerTokenFile, gateway.Name)
 	if err != nil {
 		return corev1.Container{}, err
 	}
@@ -54,7 +55,7 @@ func consulDataplaneContainer(metrics common.MetricsConfig, config common.HelmCo
 	}
 
 	container := corev1.Container{
-		Name:            name,
+		Name:            gateway.Name,
 		Image:           config.ImageDataplane,
 		ImagePullPolicy: corev1.PullPolicy(config.GlobalImagePullPolicy),
 
@@ -110,19 +111,33 @@ func consulDataplaneContainer(metrics common.MetricsConfig, config common.HelmCo
 		container.Resources = *gcc.Spec.DeploymentSpec.Resources
 	}
 
-	// If running in vanilla K8s, run as root to allow binding to privileged ports;
-	// otherwise, allow the user to be assigned by OpenShift.
+	// For backwards-compatibility, we allow privilege escalation if port mapping
+	// is disabled and the Gateway utilizes a privileged port (< 1024).
+	usingPrivilegedPorts := false
+	if gcc.Spec.MapPrivilegedContainerPorts == 0 {
+		for _, listener := range gateway.Spec.Listeners {
+			if listener.Port < 1024 {
+				usingPrivilegedPorts = true
+				break
+			}
+		}
+	}
+
 	container.SecurityContext = &corev1.SecurityContext{
-		ReadOnlyRootFilesystem: ptr.To(true),
+		AllowPrivilegeEscalation: ptr.To(usingPrivilegedPorts),
+		ReadOnlyRootFilesystem:   ptr.To(true),
+		RunAsNonRoot:             ptr.To(true),
+		SeccompProfile: &corev1.SeccompProfile{
+			Type: corev1.SeccompProfileTypeRuntimeDefault,
+		},
 		// Drop any Linux capabilities you'd get as root other than NET_BIND_SERVICE.
+		// NET_BIND_SERVICE is a requirement for consul-dataplane, even though we don't
+		// bind to privileged ports.
 		Capabilities: &corev1.Capabilities{
 			Add:  []corev1.Capability{netBindCapability},
 			Drop: []corev1.Capability{allCapabilities},
 		},
 	}
-	if !config.EnableOpenShift {
-		container.SecurityContext.RunAsUser = ptr.To(int64(0))
-	}
 
 	return container, nil
 }

control-plane/api-gateway/gatekeeper/deployment.go

@@ -109,7 +109,7 @@ func (g *Gatekeeper) deployment(gateway gwv1beta1.Gateway, gcc v1alpha1.GatewayC
 
 	volumes, mounts := volumesAndMounts(gateway)
 
-	container, err := consulDataplaneContainer(metrics, config, gcc, gateway.Name, gateway.Namespace, mounts)
+	container, err := consulDataplaneContainer(metrics, config, gcc, gateway, mounts)
 	if err != nil {
 		return nil, err
 	}

control-plane/api-gateway/gatekeeper/init.go

@@ -213,6 +213,9 @@ func (g *Gatekeeper) initContainer(config common.HelmConfig, name, namespace str
 		},
 		AllowPrivilegeEscalation: ptr.To(false),
 		ReadOnlyRootFilesystem:   ptr.To(true),
+		SeccompProfile: &corev1.SeccompProfile{
+			Type: corev1.SeccompProfileTypeRuntimeDefault,
+		},
 	}
 
 	return container, nil

control-plane/connect-inject/webhook/consul_dataplane_sidecar.go

@@ -10,12 +10,13 @@ import (
 	"strings"
 
 	"github.com/google/shlex"
-	"github.com/hashicorp/consul-k8s/control-plane/connect-inject/common"
-	"github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/ptr"
+
+	"github.com/hashicorp/consul-k8s/control-plane/connect-inject/common"
+	"github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants"
 )
 
 const (
@@ -264,10 +265,14 @@ func (w *MeshWebhook) consulDataplaneSidecar(namespace corev1.Namespace, pod cor
 		RunAsGroup:               ptr.To(group),
 		RunAsNonRoot:             ptr.To(true),
 		AllowPrivilegeEscalation: ptr.To(false),
+		SeccompProfile: &corev1.SeccompProfile{
+			Type: corev1.SeccompProfileTypeRuntimeDefault,
+		},
 		// consul-dataplane requires the NET_BIND_SERVICE capability regardless of binding port #.
 		// See https://developer.hashicorp.com/consul/docs/connect/dataplane#technical-constraints
 		Capabilities: &corev1.Capabilities{
-			Add: []corev1.Capability{"NET_BIND_SERVICE"},
+			Add:  []corev1.Capability{"NET_BIND_SERVICE"},
+			Drop: []corev1.Capability{"ALL"},
 		},
 		ReadOnlyRootFilesystem: ptr.To(true),
 	}

control-plane/connect-inject/webhook/consul_dataplane_sidecar_test.go

@@ -808,7 +808,11 @@ func TestHandlerConsulDataplaneSidecar_withSecurityContext(t *testing.T) {
 				ReadOnlyRootFilesystem:   ptr.To(true),
 				AllowPrivilegeEscalation: ptr.To(false),
 				Capabilities: &corev1.Capabilities{
-					Add: []corev1.Capability{"NET_BIND_SERVICE"},
+					Add:  []corev1.Capability{"NET_BIND_SERVICE"},
+					Drop: []corev1.Capability{"ALL"},
+				},
+				SeccompProfile: &corev1.SeccompProfile{
+					Type: corev1.SeccompProfileTypeRuntimeDefault,
 				},
 			},
 		},
@@ -822,7 +826,11 @@ func TestHandlerConsulDataplaneSidecar_withSecurityContext(t *testing.T) {
 				ReadOnlyRootFilesystem:   ptr.To(true),
 				AllowPrivilegeEscalation: ptr.To(false),
 				Capabilities: &corev1.Capabilities{
-					Add: []corev1.Capability{"NET_BIND_SERVICE"},
+					Add:  []corev1.Capability{"NET_BIND_SERVICE"},
+					Drop: []corev1.Capability{"ALL"},
+				},
+				SeccompProfile: &corev1.SeccompProfile{
+					Type: corev1.SeccompProfileTypeRuntimeDefault,
 				},
 			},
 		},
@@ -836,7 +844,11 @@ func TestHandlerConsulDataplaneSidecar_withSecurityContext(t *testing.T) {
 				ReadOnlyRootFilesystem:   ptr.To(true),
 				AllowPrivilegeEscalation: ptr.To(false),
 				Capabilities: &corev1.Capabilities{
-					Add: []corev1.Capability{"NET_BIND_SERVICE"},
+					Add:  []corev1.Capability{"NET_BIND_SERVICE"},
+					Drop: []corev1.Capability{"ALL"},
+				},
+				SeccompProfile: &corev1.SeccompProfile{
+					Type: corev1.SeccompProfileTypeRuntimeDefault,
 				},
 			},
 		},
@@ -850,7 +862,11 @@ func TestHandlerConsulDataplaneSidecar_withSecurityContext(t *testing.T) {
 				ReadOnlyRootFilesystem:   ptr.To(true),
 				AllowPrivilegeEscalation: ptr.To(false),
 				Capabilities: &corev1.Capabilities{
-					Add: []corev1.Capability{"NET_BIND_SERVICE"},
+					Add:  []corev1.Capability{"NET_BIND_SERVICE"},
+					Drop: []corev1.Capability{"ALL"},
+				},
+				SeccompProfile: &corev1.SeccompProfile{
+					Type: corev1.SeccompProfileTypeRuntimeDefault,
 				},
 			},
 		},

control-plane/connect-inject/webhook/container_init.go

@@ -289,6 +289,19 @@ func (w *MeshWebhook) containerInit(namespace corev1.Namespace, pod corev1.Pod,
 				},
 			}
 		}
+	} else {
+		container.SecurityContext = &corev1.SecurityContext{
+			AllowPrivilegeEscalation: ptr.To(false),
+			Capabilities: &corev1.Capabilities{
+				Add:  []corev1.Capability{},
+				Drop: []corev1.Capability{"ALL"},
+			},
+			ReadOnlyRootFilesystem: ptr.To(true),
+			RunAsNonRoot:           ptr.To(true),
+			SeccompProfile: &corev1.SeccompProfile{
+				Type: corev1.SeccompProfileTypeRuntimeDefault,
+			},
+		}
 	}
 
 	return container, nil

control-plane/connect-inject/webhook/container_init_test.go

@@ -9,14 +9,15 @@ import (
 	"testing"
 	"time"
 
-	"github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants"
-	"github.com/hashicorp/consul-k8s/control-plane/consul"
-	"github.com/hashicorp/consul-k8s/control-plane/namespaces"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/ptr"
+
+	"github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants"
+	"github.com/hashicorp/consul-k8s/control-plane/consul"
+	"github.com/hashicorp/consul-k8s/control-plane/namespaces"
 )
 
 const k8sNamespace = "k8snamespace"
@@ -328,6 +329,20 @@ func TestHandlerContainerInit_transparentProxy(t *testing.T) {
 					ReadOnlyRootFilesystem:   ptr.To(true),
 					AllowPrivilegeEscalation: ptr.To(false),
 				}
+			} else {
+				// When tproxy disabled
+				expectedSecurityContext = &corev1.SecurityContext{
+					AllowPrivilegeEscalation: ptr.To(false),
+					Capabilities: &corev1.Capabilities{
+						Add:  []corev1.Capability{},
+						Drop: []corev1.Capability{"ALL"},
+					},
+					ReadOnlyRootFilesystem: ptr.To(true),
+					RunAsNonRoot:           ptr.To(true),
+					SeccompProfile: &corev1.SeccompProfile{
+						Type: corev1.SeccompProfileTypeRuntimeDefault,
+					},
+				}
 			}
 			ns := corev1.Namespace{
 				ObjectMeta: metav1.ObjectMeta{