Create Elastic Agent enrolment tokens in the operator

config/crds/v1/all-crds.yaml

@@ -746,6 +746,11 @@ spec:
                 - standalone
                 - fleet
                 type: string
+              policyID:
+                description: PolicyID optionally determines into which Agent Policy
+                  this Agent will be enrolled. If left empty the default policy will
+                  be used.
+                type: string
               revisionHistoryLimit:
                 description: RevisionHistoryLimit is the number of revisions to retain
                   to allow rollback in the underlying DaemonSet or Deployment.

config/crds/v1/bases/agent.k8s.elastic.co_agents.yaml

@@ -15799,6 +15799,11 @@ spec:
                 - standalone
                 - fleet
                 type: string
+              policyID:
+                description: PolicyID optionally determines into which Agent Policy
+                  this Agent will be enrolled. If left empty the default policy will
+                  be used.
+                type: string
               revisionHistoryLimit:
                 description: RevisionHistoryLimit is the number of revisions to retain
                   to allow rollback in the underlying DaemonSet or Deployment.

deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml

@@ -752,6 +752,11 @@ spec:
                 - standalone
                 - fleet
                 type: string
+              policyID:
+                description: PolicyID optionally determines into which Agent Policy
+                  this Agent will be enrolled. If left empty the default policy will
+                  be used.
+                type: string
               revisionHistoryLimit:
                 description: RevisionHistoryLimit is the number of revisions to retain
                   to allow rollback in the underlying DaemonSet or Deployment.

docs/orchestrating-elastic-stack-applications/agent-fleet.asciidoc

@@ -312,7 +312,7 @@ spec:
 [id="{p}-elastic-agent-fleet-configuration-setting-referenced-resources"]
 === Set referenced resources
 
-Both Fleet Server and Elastic Agent in Fleet mode can facilitate the Fleet setup. Fleet Server can set up Fleet in Kibana (which otherwise requires manual steps) and enroll itself in the default Fleet Server policy. Elastic Agent can enroll itself in the default Elastic Agent policy. To allow ECK to set this up, provide a reference to ECK-managed Kibana through `kibanaRef` configuration element.
+Both Fleet Server and Elastic Agent in Fleet mode can be automatically setup with Fleet by ECK. The ECK operator can set up Fleet in Kibana (which otherwise requires manual steps) and enroll Fleet Server in the default Fleet Server policy. Elastic Agent can be automatically enrolled in the default Elastic Agent policy. To allow ECK to set this up, provide a reference to a ECK-managed Kibana through the `kibanaRef` configuration element.
 
 [source,yaml,subs="attributes,+macros"]
 ----
@@ -325,7 +325,7 @@ spec:
     name: kibana
 ----
 
-ECK can also facilitate the connection between Elastic Agents and ECK-managed Fleet Server. To allow ECK to set this up, provide a reference to Fleet Server through `fleetServerRef` configuration element.
+ECK can also facilitate the connection between Elastic Agents and a ECK-managed Fleet Server. To allow ECK to set this up, provide a reference to Fleet Server through the `fleetServerRef` configuration element.
 
 [source,yaml,subs="attributes,+macros"]
 ----
@@ -339,7 +339,7 @@ spec:
 ----
 
 
-Set `elasticsearchRefs` element in your Fleet Server to point to the Elasticsearch cluster that will manage Fleet. Leave `elasticsearchRefs` empty or unset for any Elastic Agent running in Fleet mode as the Elasticsearch cluster to target will come from Kibana `xpack.fleet.agents.elasticsearch.hosts` configuration element.
+Set the `elasticsearchRefs` element in your Fleet Server to point to the Elasticsearch cluster that will manage Fleet. Leave `elasticsearchRefs` empty or unset it for any Elastic Agent running in Fleet mode as the Elasticsearch cluster to target will come from Kibana's `xpack.fleet.agents.elasticsearch.hosts` configuration element.
 
 NOTE: Currently, Elastic Agent in Fleet mode supports only a single output, so only a single Elasticsearch cluster can be referenced.
 
@@ -359,9 +359,7 @@ By default, every reference targets all instances in your Elasticsearch, Kibana
 [id="{p}-elastic-agent-fleet-configuration-custom-configuration"]
 === Customize Elastic Agent configuration
 
-In contrast to what happens with Elastic Agent as standalone, the configuration is managed through Fleet, and it cannot be defined through `config` or `configRef` elements.
-
-You can only configure the setup part of the Fleet Server and Elastic Agent. You can override each of the environment variables that agents consume, as documented in link:https://www.elastic.co/guide/en/fleet/current/agent-environment-variables.html[Elastic Agent environment variables]. This allows different setups where components are deployed both in local Kubernetes cluster and externally.
+In contrast to Elastic Agents in standalone mode, the configuration is managed through Fleet, and it cannot be defined through `config` or `configRef` elements.
 
 [id="{p}-elastic-agent-fleet-configuration-upgrade-specification"]
 === Upgrade the Elastic Agent specification
@@ -470,27 +468,23 @@ To deploy Elastic Agent in clusters with the Pod Security Policy admission contr
 
 By default, ECK creates a Service for Fleet Server that Elastic Agents can connect through. You can customize it using the `http` configuration element. Check more information on how to link:k8s-services.html[make changes] to the Service and link:k8s-tls-certificates.html[customize] the TLS configuration.
 
-[id="{p}-elastic-agent-fleet-configuration-override-default-fleet-configuration-settings"]
-=== Override default Fleet configuration settings
-
-ECK uses environment variables to control how Elastic Agent and Fleet Server should be configured. Sometimes, it might be required to override some of these settings. For example, if Kibana TLS certificate is signed by a well-known root and can't include `kibana-kb-http.namespace.svc` as a SAN, `KIBANA_FLEET_HOST` can be overriden to point to the URL that the certificate specifies. To do that, specify environment variable, as shown in the following example.
+[id="{p}-elastic-agent-control-fleet-policy-selection"]
+=== Control Fleet policy selection
 
+ECK uses the default policy to enroll Elastic Agents in Fleet and the default Fleet Server policy to enroll Fleet Server. A different policy can be chosen by using the `policyID` attribute in the Elastic Agent resource:
 [source,yaml]
 ----
-...
+
+apiVersion: agent.k8s.elastic.co/v1alpha1
+kind: Agent
+metadata:
+  name: fleet-server-sample
 spec:
-  deployment:
-    podTemplate:
-      spec:
-        containers:
-        - name: agent
-          env:
-          - name: KIBANA_FLEET_HOST
-            value: "https://kibana.example.com:443"
+  policyID: my-custom-policy
 ...
 ----
 
-Check the Elastic Agent link:https://www.elastic.co/guide/en/fleet/current/agent-environment-variables.html[docs] to get a list of all the environment variables that can be used.
+Please note that the environment variables related to policy selection mentioned in the Elastic Agent link:https://www.elastic.co/guide/en/fleet/current/agent-environment-variables.html[docs] like `FLEET_SERVER_POLICY_ID` will not take effect if enrollment is managed by the ECK operator.
 
 
 [id="{p}-elastic-agent-fleet-configuration-examples"]

docs/reference/api-docs.asciidoc

@@ -93,6 +93,7 @@ AgentSpec defines the desired state of the Agent
 | *`http`* __xref:{anchor_prefix}-github.ghproxy.top-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-httpconfig[$$HTTPConfig$$]__ | HTTP holds the HTTP layer configuration for the Agent in Fleet mode with Fleet Server enabled.
 | *`mode`* __xref:{anchor_prefix}-github.ghproxy.top-elastic-cloud-on-k8s-v2-pkg-apis-agent-v1alpha1-agentmode[$$AgentMode$$]__ | Mode specifies the source of configuration for the Agent. The configuration can be specified locally through `config` or `configRef` (`standalone` mode), or come from Fleet during runtime (`fleet` mode). Defaults to `standalone` mode.
 | *`fleetServerEnabled`* __boolean__ | FleetServerEnabled determines whether this Agent will launch Fleet Server. Don't set unless `mode` is set to `fleet`.
+| *`policyID`* __string__ | PolicyID optionally determines into which Agent Policy this Agent will be enrolled. If left empty the default policy will be used.
 | *`kibanaRef`* __xref:{anchor_prefix}-github.ghproxy.top-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-objectselector[$$ObjectSelector$$]__ | KibanaRef is a reference to Kibana where Fleet should be set up and this Agent should be enrolled. Don't set unless `mode` is set to `fleet`.
 | *`fleetServerRef`* __xref:{anchor_prefix}-github.ghproxy.top-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-objectselector[$$ObjectSelector$$]__ | FleetServerRef is a reference to Fleet Server that this Agent should connect to to obtain it's configuration. Don't set unless `mode` is set to `fleet`.
 |===

pkg/apis/agent/v1alpha1/agent_types.go

@@ -87,6 +87,11 @@ type AgentSpec struct {
 	// +kubebuilder:validation:Optional
 	FleetServerEnabled bool `json:"fleetServerEnabled,omitempty"`
 
+	// PolicyID optionally determines into which Agent Policy this Agent will be enrolled. If left empty the default
+	// policy will be used.
+	// +kubebuilder:validation:Optional
+	PolicyID string `json:"policyID,omitempty"`
+
 	// KibanaRef is a reference to Kibana where Fleet should be set up and this Agent should be enrolled. Don't set
 	// unless `mode` is set to `fleet`.
 	// +kubebuilder:validation:Optional

pkg/controller/agent/config.go

@@ -5,18 +5,22 @@
 package agent
 
 import (
+	"context"
+	"crypto/x509"
 	"errors"
 	"fmt"
 	"hash"
 	"path"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 
 	agentv1alpha1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/agent/v1alpha1"
 	commonv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/common/v1"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/association"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/common"
+	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/certificates"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/labels"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/reconciler"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/settings"
@@ -25,8 +29,9 @@ import (
 )
 
 type connectionSettings struct {
-	host, ca    string
-	credentials association.Credentials
+	host, caFileName, version string
+	credentials               association.Credentials
+	caCerts                   []*x509.Certificate
 }
 
 func reconcileConfig(params Params, configHash hash.Hash) *reconciler.Results {
@@ -141,29 +146,32 @@ func getUserConfig(params Params) (*settings.CanonicalConfig, error) {
 	return common.ParseConfigRef(params, &params.Agent, params.Agent.Spec.ConfigRef, ConfigFileName)
 }
 
-func extractConnectionSettings(
+// extractPodConnectionSettings extracts connections settings to be used inside an Elastic Agent Pod. That is without
+// certificates which are mounted directly into the Pod, instead the connection settings contain a path which points to
+// the future location of the certificates in the Pod.
+func extractPodConnectionSettings(
 	agent agentv1alpha1.Agent,
 	client k8s.Client,
 	associationType commonv1.AssociationType,
-) (connectionSettings, error) {
+) (connectionSettings, *commonv1.AssociationConf, error) {
 	assoc, err := association.SingleAssociationOfType(agent.GetAssociations(), associationType)
 	if err != nil {
-		return connectionSettings{}, err
+		return connectionSettings{}, nil, err
 	}
 
 	if assoc == nil {
 		errTemplate := "association of type %s not found in %d associations"
-		return connectionSettings{}, fmt.Errorf(errTemplate, associationType, len(agent.GetAssociations()))
+		return connectionSettings{}, nil, fmt.Errorf(errTemplate, associationType, len(agent.GetAssociations()))
 	}
 
 	credentials, err := association.ElasticsearchAuthSettings(client, assoc)
 	if err != nil {
-		return connectionSettings{}, err
+		return connectionSettings{}, nil, err
 	}
 
 	assocConf, err := assoc.AssociationConf()
 	if err != nil {
-		return connectionSettings{}, err
+		return connectionSettings{}, nil, err
 	}
 
 	ca := ""
@@ -173,7 +181,39 @@ func extractConnectionSettings(
 
 	return connectionSettings{
 		host:        assocConf.GetURL(),
-		ca:          ca,
+		caFileName:  ca,
 		credentials: credentials,
-	}, err
+		version:     assocConf.Version,
+	}, assocConf, err
+}
+
+// extractClientConnectionSettings same as extractPodConnectionSettings but for use inside the operator or any other
+// client that needs direct access to the relevant CA certificates of the associated object (if TLS is configured)
+func extractClientConnectionSettings(
+	ctx context.Context,
+	agent agentv1alpha1.Agent,
+	client k8s.Client,
+	associationType commonv1.AssociationType,
+) (connectionSettings, error) {
+	settings, assocConf, err := extractPodConnectionSettings(agent, client, associationType)
+	if err != nil {
+		return connectionSettings{}, err
+	}
+	if !assocConf.GetCACertProvided() {
+		return settings, nil
+	}
+	var caSecret corev1.Secret
+	if err := client.Get(ctx, types.NamespacedName{Name: assocConf.GetCASecretName(), Namespace: agent.Namespace}, &caSecret); err != nil {
+		return connectionSettings{}, err
+	}
+	bytes, ok := caSecret.Data[CAFileName]
+	if !ok {
+		return connectionSettings{}, fmt.Errorf("no %s in %s", CAFileName, k8s.ExtractNamespacedName(&caSecret))
+	}
+	certs, err := certificates.ParsePEMCerts(bytes)
+	if err != nil {
+		return connectionSettings{}, err
+	}
+	settings.caCerts = certs
+	return settings, nil
 }