From f45c2149b0eb00d988fb9d34339acbff9c666145 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Tue, 28 Jun 2022 09:13:18 +0200 Subject: [PATCH 01/26] wip --- pkg/controller/agent/config.go | 58 ++++++- pkg/controller/agent/config_test.go | 6 +- pkg/controller/agent/fleet.go | 232 ++++++++++++++++++++++++++++ pkg/controller/agent/pod.go | 70 +++++---- 4 files changed, 325 insertions(+), 41 deletions(-) create mode 100644 pkg/controller/agent/fleet.go diff --git a/pkg/controller/agent/config.go b/pkg/controller/agent/config.go index ad10900d1e..ab36b0187f 100644 --- a/pkg/controller/agent/config.go +++ b/pkg/controller/agent/config.go @@ -5,6 +5,8 @@ package agent import ( + "context" + "crypto/x509" "errors" "fmt" "hash" @@ -12,11 +14,13 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" agentv1alpha1 "github.com/elastic/cloud-on-k8s/pkg/apis/agent/v1alpha1" commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" "github.com/elastic/cloud-on-k8s/pkg/controller/association" "github.com/elastic/cloud-on-k8s/pkg/controller/common" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/certificates" "github.com/elastic/cloud-on-k8s/pkg/controller/common/reconciler" "github.com/elastic/cloud-on-k8s/pkg/controller/common/settings" "github.com/elastic/cloud-on-k8s/pkg/controller/common/tracing" @@ -24,8 +28,9 @@ import ( ) type connectionSettings struct { - host, ca string - credentials association.Credentials + host, caFileName, version string + credentials association.Credentials + caCerts []*x509.Certificate } func reconcileConfig(params Params, configHash hash.Hash) *reconciler.Results { @@ -140,7 +145,7 @@ func getUserConfig(params Params) (*settings.CanonicalConfig, error) { return common.ParseConfigRef(params, ¶ms.Agent, params.Agent.Spec.ConfigRef, ConfigFileName) } -func extractConnectionSettings( +func extractPodConnectionSettings( agent agentv1alpha1.Agent, client k8s.Client, associationType commonv1.AssociationType, @@ -172,7 +177,52 @@ func extractConnectionSettings( return connectionSettings{ host: assocConf.GetURL(), - ca: ca, + caFileName: ca, credentials: credentials, }, err } + +func extractClientConnectionSettings( + agent agentv1alpha1.Agent, + client k8s.Client, + associationType commonv1.AssociationType, +) (connectionSettings, error) { + assoc, err := association.SingleAssociationOfType(agent.GetAssociations(), associationType) + if err != nil { + return connectionSettings{}, err + } + + if assoc == nil { + errTemplate := "association of type %s not found in %d associations" + return connectionSettings{}, fmt.Errorf(errTemplate, associationType, len(agent.GetAssociations())) + } + + credentials, err := association.ElasticsearchAuthSettings(client, assoc) + if err != nil { + return connectionSettings{}, err + } + + assocConf, err := assoc.AssociationConf() + if err != nil { + return connectionSettings{}, err + } + settings := connectionSettings{ + host: assocConf.GetURL(), + credentials: credentials, + version: assocConf.Version, + } + if !assocConf.GetCACertProvided() { + return settings, nil + } + var caSecret corev1.Secret + if err := client.Get(context.Background(), types.NamespacedName{Name: assocConf.GetCASecretName(), Namespace: agent.Namespace}, &caSecret); err != nil { + return connectionSettings{}, err + } + bytes := caSecret.Data[CAFileName] + certs, err := certificates.ParsePEMCerts(bytes) + if err != nil { + return connectionSettings{}, err + } + settings.caCerts = certs + return settings, nil +} diff --git a/pkg/controller/agent/config_test.go b/pkg/controller/agent/config_test.go index 46a03824ea..2eca170406 100644 --- a/pkg/controller/agent/config_test.go +++ b/pkg/controller/agent/config_test.go @@ -110,8 +110,8 @@ func TestExtractConnectionSettings(t *testing.T) { }), assocType: commonv1.KibanaAssociationType, wantConnectionSettings: connectionSettings{ - host: "url", - ca: "/mnt/elastic-internal/kibana-association/ns/kibana/certs/ca.crt", + host: "url", + caFileName: "/mnt/elastic-internal/kibana-association/ns/kibana/certs/ca.crt", credentials: association.Credentials{ Username: "user", Password: "password", @@ -121,7 +121,7 @@ func TestExtractConnectionSettings(t *testing.T) { }, } { t.Run(tt.name, func(t *testing.T) { - gotConnectionSettings, gotErr := extractConnectionSettings(tt.agent, tt.client, tt.assocType) + gotConnectionSettings, gotErr := extractPodConnectionSettings(tt.agent, tt.client, tt.assocType) require.Equal(t, tt.wantConnectionSettings, gotConnectionSettings) require.Equal(t, tt.wantErr, gotErr != nil) diff --git a/pkg/controller/agent/fleet.go b/pkg/controller/agent/fleet.go new file mode 100644 index 0000000000..979a040ecc --- /dev/null +++ b/pkg/controller/agent/fleet.go @@ -0,0 +1,232 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package agent + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/go-logr/logr" + "github.com/pkg/errors" + + agentv1alpha1 "github.com/elastic/cloud-on-k8s/pkg/apis/agent/v1alpha1" + "github.com/elastic/cloud-on-k8s/pkg/controller/common" + "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" + "github.com/elastic/cloud-on-k8s/pkg/utils/net" + "github.com/elastic/cloud-on-k8s/pkg/utils/stringsutil" +) + +const FleetTokenAnnotation = "fleet.eck.k8s.elastic.co/token" + +// TODO common kibana/http client? +type APIError struct { + StatusCode int + msg string +} + +func (e *APIError) Error() string { + return e.msg +} + +type EnrollmentAPIKeyResult struct { + Item EnrollmentAPIKey `json:"item"` +} + +type EnrollmentAPIKey struct { + ID string `json:"id,omitempty"` + Active bool `json:"active,omitempty"` + APIKey string `json:"api_key,omitempty"` + PolicyID string `json:"policy_id,omitempty"` +} + +type AgentPolicyList struct { + Items []AgentPolicy `json:"items"` +} + +type AgentPolicy struct { + ID string `json:"id"` + IsDefault bool `json:"is_default"` + IsDefaultFleetServer bool `json:"is_default_fleet_server"` +} + +type fleetAPI struct { + client *http.Client + endpoint string + username string + password string + kibanaVersion string + log logr.Logger +} + +func newFleetAPI(dialer net.Dialer, settings connectionSettings, logger logr.Logger) fleetAPI { + return fleetAPI{ + client: common.HTTPClient(dialer, settings.caCerts, 60*time.Second), + kibanaVersion: settings.version, + endpoint: settings.host, + username: settings.credentials.Username, + password: settings.credentials.Password, + log: logger, + } +} + +func (f fleetAPI) request( + ctx context.Context, + method string, + pathWithQuery string, + requestObj, + responseObj interface{}) error { + + var body io.Reader = http.NoBody + if requestObj != nil { + outData, err := json.Marshal(requestObj) + if err != nil { + return err + } + body = bytes.NewBuffer(outData) + } + + request, err := http.NewRequestWithContext(ctx, method, stringsutil.Concat(f.endpoint, "/api/fleet/", pathWithQuery), body) + if err != nil { + return err + } + + // Sets headers allowing ES to distinguish between deprecated APIs used internally and by the user + if request.Header == nil { + request.Header = make(http.Header) + } + request.Header.Set(common.InternalProductRequestHeaderKey, common.InternalProductRequestHeaderValue) + request.Header.Set("kbn-xsrf", "true") + request.SetBasicAuth(f.username, f.password) + + f.log.V(1).Info( + "Fleet API HTTP request", + "method", request.Method, + "url", request.URL.Redacted(), + ) + + resp, err := f.client.Do(request) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return &APIError{ + StatusCode: resp.StatusCode, + msg: fmt.Sprintf("failed to request %s, status is %d)", request.URL.Redacted(), resp.StatusCode), + } + } + if responseObj != nil { + if err := json.NewDecoder(resp.Body).Decode(responseObj); err != nil { + return err + } + } + + return nil + +} + +func (f fleetAPI) CreateEnrollmentAPIKey(ctx context.Context, policyID string) (EnrollmentAPIKey, error) { + path := "enrollment_api_keys" + if strings.HasPrefix(f.kibanaVersion, "7") { + path = strings.Replace(path, "_", "-", -1) + } + var response EnrollmentAPIKeyResult + err := f.request(ctx, http.MethodPost, path, EnrollmentAPIKey{PolicyID: policyID}, &response) + return response.Item, err +} + +func (f fleetAPI) GetEnrollmentAPIKey(ctx context.Context, keyID string) (EnrollmentAPIKey, error) { + path := "enrollment_api_keys" + if strings.HasPrefix(f.kibanaVersion, "7") { + path = strings.Replace(path, "_", "-", -1) + } + var response EnrollmentAPIKeyResult + err := f.request(ctx, http.MethodGet, fmt.Sprintf("%s/%s", path, keyID), nil, &response) + return response.Item, err +} + +func (f fleetAPI) ListAgentPolicies(ctx context.Context) (AgentPolicyList, error) { + var list AgentPolicyList + err := f.request(ctx, http.MethodGet, "agent_policies", nil, &list) + return list, err +} + +func (f fleetAPI) DefaultFleetServerPolicyID(ctx context.Context) (string, error) { + policies, err := f.ListAgentPolicies(ctx) + if err != nil { + return "", err + } + for _, p := range policies.Items { + if p.IsDefaultFleetServer { + return p.ID, nil + } + } + return "", errors.New("no default fleet server policy found") +} + +func (f fleetAPI) DefaultAgentPolicyID(ctx context.Context) (string, error) { + policies, err := f.ListAgentPolicies(ctx) + if err != nil { + return "", err + } + for _, p := range policies.Items { + if p.IsDefault { + return p.ID, nil + } + } + return "", errors.New("no default agent policy found") +} + +func reconcileEnrollmentToken( + ctx context.Context, + agent agentv1alpha1.Agent, + client k8s.Client, + api fleetAPI, +) (string, error) { + tokenName, exists := agent.Annotations[FleetTokenAnnotation] + policyID, err := reconcilePolicyID(ctx, agent, api) + if err != nil { + return "", err + } + if exists { + key, err := api.GetEnrollmentAPIKey(ctx, tokenName) + if err != nil { + return "", err + } + if key.Active { + return key.APIKey, nil + } + } + key, err := api.CreateEnrollmentAPIKey(ctx, policyID) + if err != nil { + return "", err + } + // TODO this creates conflicts solve on top level + agent.Annotations[FleetTokenAnnotation] = key.ID + err = client.Update(ctx, &agent) + if err != nil { + return "", err + } + // TODO failed update creates dangling API key + return key.APIKey, nil +} + +func reconcilePolicyID(ctx context.Context, agent agentv1alpha1.Agent, api fleetAPI) (string, error) { + /*if agent.Spec.PolicyID != "" { + return agent.Spec.PolicyID + }*/ + if agent.Spec.FleetServerEnabled { + return api.DefaultFleetServerPolicyID(ctx) + } + return api.DefaultAgentPolicyID(ctx) + +} diff --git a/pkg/controller/agent/pod.go b/pkg/controller/agent/pod.go index e89dc68510..ba45fc9044 100644 --- a/pkg/controller/agent/pod.go +++ b/pkg/controller/agent/pod.go @@ -5,12 +5,13 @@ package agent import ( + "context" "fmt" "hash" "path" "sort" - "strconv" + "github.com/go-logr/logr" pkgerrors "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -31,6 +32,7 @@ import ( "github.com/elastic/cloud-on-k8s/pkg/controller/common/volume" "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" "github.com/elastic/cloud-on-k8s/pkg/utils/maps" + netutil "github.com/elastic/cloud-on-k8s/pkg/utils/net" ) const ( @@ -63,9 +65,10 @@ const ( KibanaFleetCA = "KIBANA_FLEET_CA" // Below are the names of environment variables used to configure Elastic Agent to Fleet connection in Fleet mode. - FleetEnroll = "FLEET_ENROLL" - FleetCA = "FLEET_CA" - FleetURL = "FLEET_URL" + FleetEnroll = "FLEET_ENROLL" + FleetEnrollmentToken = "FLEET_ENROLLMENT_TOKEN" + FleetCA = "FLEET_CA" + FleetURL = "FLEET_URL" // Below are the names of environment variables used to configure Fleet Server and its connection to Elasticsearch // in Fleet mode. @@ -226,7 +229,7 @@ func amendBuilderForFleetMode(params Params, fleetCerts *certificates.Certificat } func applyEnvVars(params Params, builder *defaults.PodTemplateBuilder) (*defaults.PodTemplateBuilder, error) { - fleetModeEnvVars, err := getFleetModeEnvVars(params.Agent, params.Client) + fleetModeEnvVars, err := getFleetModeEnvVars(params.Context, params.Agent, params.Client, params.OperatorParams.Dialer, params.Logger()) if err != nil { return nil, err } @@ -370,8 +373,8 @@ func writeEsAssocToConfigHash(params Params, esAssociation commonv1.Association, } func getVolumesFromAssociations(associations []commonv1.Association) ([]volume.VolumeLike, error) { - var vols []volume.VolumeLike //nolint:prealloc - for i, assoc := range associations { + var vols []volume.VolumeLike //nolint:prealloc + for i, assoc := range associations { // TODO filter Kibana out assocConf, err := assoc.AssociationConf() if err != nil { return nil, err @@ -427,15 +430,15 @@ func certificatesDir(association commonv1.Association) string { ) } -func getFleetModeEnvVars(agent agentv1alpha1.Agent, client k8s.Client) (map[string]string, error) { +func getFleetModeEnvVars(ctx context.Context, agent agentv1alpha1.Agent, client k8s.Client, dialer netutil.Dialer, logger logr.Logger) (map[string]string, error) { result := map[string]string{} - for _, f := range []func(agentv1alpha1.Agent, k8s.Client) (map[string]string, error){ + for _, f := range []func(context.Context, agentv1alpha1.Agent, k8s.Client, netutil.Dialer, logr.Logger) (map[string]string, error){ getFleetSetupKibanaEnvVars, getFleetSetupFleetEnvVars, getFleetSetupFleetServerEnvVars, } { - envVars, err := f(agent, client) + envVars, err := f(ctx, agent, client, dialer, logger) if err != nil { return nil, err } @@ -445,32 +448,31 @@ func getFleetModeEnvVars(agent agentv1alpha1.Agent, client k8s.Client) (map[stri return result, nil } -func getFleetSetupKibanaEnvVars(agent agentv1alpha1.Agent, client k8s.Client) (map[string]string, error) { - if agent.Spec.KibanaRef.IsDefined() { - kbConnectionSettings, err := extractConnectionSettings(agent, client, commonv1.KibanaAssociationType) - if err != nil { - return nil, err - } - - envVars := map[string]string{ - KibanaFleetHost: kbConnectionSettings.host, - KibanaFleetUsername: kbConnectionSettings.credentials.Username, - KibanaFleetPassword: kbConnectionSettings.credentials.Password, - KibanaFleetSetup: strconv.FormatBool(agent.Spec.KibanaRef.IsDefined()), - } +func getFleetSetupKibanaEnvVars(ctx context.Context, agent agentv1alpha1.Agent, client k8s.Client, dialer netutil.Dialer, logger logr.Logger) (map[string]string, error) { + if !agent.Spec.KibanaRef.IsDefined() { + return map[string]string{}, nil + } - // don't set ca key if ca is not available - if kbConnectionSettings.ca != "" { - envVars[KibanaFleetCA] = kbConnectionSettings.ca - } + kbConnectionSettings, err := extractClientConnectionSettings(agent, client, commonv1.KibanaAssociationType) + if err != nil { + return nil, err + } - return envVars, nil + token, err := reconcileEnrollmentToken( + ctx, agent, client, + newFleetAPI(dialer, kbConnectionSettings, logger.WithValues("namespace", agent.Namespace, "agent_name", agent.Name)), + ) + if err != nil { + return nil, err + } + envVars := map[string]string{ + FleetEnrollmentToken: token, } - return map[string]string{}, nil + return envVars, nil } -func getFleetSetupFleetEnvVars(agent agentv1alpha1.Agent, client k8s.Client) (map[string]string, error) { +func getFleetSetupFleetEnvVars(_ context.Context, agent agentv1alpha1.Agent, client k8s.Client, _ netutil.Dialer, _ logr.Logger) (map[string]string, error) { fleetCfg := map[string]string{} if agent.Spec.KibanaRef.IsDefined() { @@ -512,7 +514,7 @@ func getFleetSetupFleetEnvVars(agent agentv1alpha1.Agent, client k8s.Client) (ma return fleetCfg, nil } -func getFleetSetupFleetServerEnvVars(agent agentv1alpha1.Agent, client k8s.Client) (map[string]string, error) { +func getFleetSetupFleetServerEnvVars(_ context.Context, agent agentv1alpha1.Agent, client k8s.Client, _ netutil.Dialer, _ logr.Logger) (map[string]string, error) { if !agent.Spec.FleetServerEnabled { return map[string]string{}, nil } @@ -525,7 +527,7 @@ func getFleetSetupFleetServerEnvVars(agent agentv1alpha1.Agent, client k8s.Clien esExpected := len(agent.Spec.ElasticsearchRefs) > 0 && agent.Spec.ElasticsearchRefs[0].IsDefined() if esExpected { - esConnectionSettings, err := extractConnectionSettings(agent, client, commonv1.ElasticsearchAssociationType) + esConnectionSettings, err := extractPodConnectionSettings(agent, client, commonv1.ElasticsearchAssociationType) if err != nil { return nil, err } @@ -540,8 +542,8 @@ func getFleetSetupFleetServerEnvVars(agent agentv1alpha1.Agent, client k8s.Clien } // don't set ca key if ca is not available - if esConnectionSettings.ca != "" { - fleetServerCfg[FleetServerElasticsearchCA] = esConnectionSettings.ca + if esConnectionSettings.caFileName != "" { + fleetServerCfg[FleetServerElasticsearchCA] = esConnectionSettings.caFileName } } From 8a3f379d891c99577dde016a23d6f63bb91bb725 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Tue, 28 Jun 2022 16:43:50 +0200 Subject: [PATCH 02/26] pagination --- pkg/controller/agent/fleet.go | 67 +++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/pkg/controller/agent/fleet.go b/pkg/controller/agent/fleet.go index 979a040ecc..9c22694158 100644 --- a/pkg/controller/agent/fleet.go +++ b/pkg/controller/agent/fleet.go @@ -36,6 +36,19 @@ func (e *APIError) Error() string { return e.msg } +// IsNotFound checks whether the error was an HTTP 404 error. +func IsNotFound(err error) bool { + return isHTTPError(err, http.StatusNotFound) +} + +func isHTTPError(err error, statusCode int) bool { + apiErr := new(APIError) + if errors.As(err, &apiErr) { + return apiErr.StatusCode == statusCode + } + return false +} + type EnrollmentAPIKeyResult struct { Item EnrollmentAPIKey `json:"item"` } @@ -55,6 +68,7 @@ type AgentPolicy struct { ID string `json:"id"` IsDefault bool `json:"is_default"` IsDefaultFleetServer bool `json:"is_default_fleet_server"` + Status string `json:"status"` } type fleetAPI struct { @@ -154,36 +168,47 @@ func (f fleetAPI) GetEnrollmentAPIKey(ctx context.Context, keyID string) (Enroll return response.Item, err } -func (f fleetAPI) ListAgentPolicies(ctx context.Context) (AgentPolicyList, error) { - var list AgentPolicyList - err := f.request(ctx, http.MethodGet, "agent_policies", nil, &list) - return list, err +func (f fleetAPI) findAgentPolicy(ctx context.Context, filter func(policy AgentPolicy) bool) (AgentPolicy, error) { + page := 1 + for { + var list AgentPolicyList + err := f.request(ctx, http.MethodGet, fmt.Sprintf("agent_policies?perPage=20&page=%d", page), nil, &list) + if err != nil { + return AgentPolicy{}, err + } + if len(list.Items) == 0 { + // goto NOTFOUND + break + } + for _, p := range list.Items { + if filter(p) { + return p, nil + } + } + page++ + } + // NOTFOUND: + return AgentPolicy{}, errors.New("no matching agent policy found") } func (f fleetAPI) DefaultFleetServerPolicyID(ctx context.Context) (string, error) { - policies, err := f.ListAgentPolicies(ctx) + policy, err := f.findAgentPolicy(ctx, func(policy AgentPolicy) bool { + return policy.IsDefaultFleetServer && policy.Status == "active" + }) if err != nil { return "", err } - for _, p := range policies.Items { - if p.IsDefaultFleetServer { - return p.ID, nil - } - } - return "", errors.New("no default fleet server policy found") + return policy.ID, nil } func (f fleetAPI) DefaultAgentPolicyID(ctx context.Context) (string, error) { - policies, err := f.ListAgentPolicies(ctx) + policy, err := f.findAgentPolicy(ctx, func(policy AgentPolicy) bool { + return policy.IsDefault && policy.Status == "active" + }) if err != nil { return "", err } - for _, p := range policies.Items { - if p.IsDefault { - return p.ID, nil - } - } - return "", errors.New("no default agent policy found") + return policy.ID, nil } func reconcileEnrollmentToken( @@ -199,13 +224,19 @@ func reconcileEnrollmentToken( } if exists { key, err := api.GetEnrollmentAPIKey(ctx, tokenName) + if err != nil && IsNotFound(err) { + goto CREATE + } if err != nil { return "", err } + // TODO check token is for the correct policy if key.Active { return key.APIKey, nil } } + +CREATE: key, err := api.CreateEnrollmentAPIKey(ctx, policyID) if err != nil { return "", err From 287507b0db336621d4de0295cc920e18f92abcb9 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Wed, 29 Jun 2022 14:04:24 +0200 Subject: [PATCH 03/26] move fleet token reconciliation to top level --- pkg/controller/agent/driver.go | 8 ++++- pkg/controller/agent/fleet.go | 61 +++++++++++++++++++++++++--------- pkg/controller/agent/pod.go | 42 +++++++++-------------- 3 files changed, 69 insertions(+), 42 deletions(-) diff --git a/pkg/controller/agent/driver.go b/pkg/controller/agent/driver.go index 723f2b0e5c..b18db82178 100644 --- a/pkg/controller/agent/driver.go +++ b/pkg/controller/agent/driver.go @@ -128,6 +128,12 @@ func internalReconcile(params Params) (*reconciler.Results, agentv1alpha1.AgentS } _, _ = configHash.Write(fleetCerts.Data[certificates.CertFileName]) } + + fleetToken, err := maybeReconcileFleetEnrollment(params) + if err != nil { + return results.WithError(err), params.Status + } + if res := reconcileConfig(params, configHash); res.HasError() { return results.WithResults(res), params.Status } @@ -137,7 +143,7 @@ func internalReconcile(params Params) (*reconciler.Results, agentv1alpha1.AgentS return results.WithError(err), params.Status } - podTemplate, err := buildPodTemplate(params, fleetCerts, configHash) + podTemplate, err := buildPodTemplate(params, fleetCerts, fleetToken, configHash) if err != nil { return results.WithError(err), params.Status } diff --git a/pkg/controller/agent/fleet.go b/pkg/controller/agent/fleet.go index 9c22694158..65dc3c1a65 100644 --- a/pkg/controller/agent/fleet.go +++ b/pkg/controller/agent/fleet.go @@ -16,8 +16,10 @@ import ( "github.com/go-logr/logr" "github.com/pkg/errors" + k8serrors "k8s.io/apimachinery/pkg/util/errors" agentv1alpha1 "github.com/elastic/cloud-on-k8s/pkg/apis/agent/v1alpha1" + commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" "github.com/elastic/cloud-on-k8s/pkg/controller/common" "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" "github.com/elastic/cloud-on-k8s/pkg/utils/net" @@ -148,26 +150,31 @@ func (f fleetAPI) request( } -func (f fleetAPI) CreateEnrollmentAPIKey(ctx context.Context, policyID string) (EnrollmentAPIKey, error) { +func (f fleetAPI) enrollmentAPIKeyPath() string { path := "enrollment_api_keys" if strings.HasPrefix(f.kibanaVersion, "7") { - path = strings.Replace(path, "_", "-", -1) + path = "enrollment-api-keys" } + return path +} + +func (f fleetAPI) CreateEnrollmentAPIKey(ctx context.Context, policyID string) (EnrollmentAPIKey, error) { + var response EnrollmentAPIKeyResult - err := f.request(ctx, http.MethodPost, path, EnrollmentAPIKey{PolicyID: policyID}, &response) + err := f.request(ctx, http.MethodPost, f.enrollmentAPIKeyPath(), EnrollmentAPIKey{PolicyID: policyID}, &response) return response.Item, err } func (f fleetAPI) GetEnrollmentAPIKey(ctx context.Context, keyID string) (EnrollmentAPIKey, error) { - path := "enrollment_api_keys" - if strings.HasPrefix(f.kibanaVersion, "7") { - path = strings.Replace(path, "_", "-", -1) - } var response EnrollmentAPIKeyResult - err := f.request(ctx, http.MethodGet, fmt.Sprintf("%s/%s", path, keyID), nil, &response) + err := f.request(ctx, http.MethodGet, fmt.Sprintf("%s/%s", f.enrollmentAPIKeyPath(), keyID), nil, &response) return response.Item, err } +func (f fleetAPI) DeleteEnrollmentAPIKey(ctx context.Context, keyID string) error { + return f.request(ctx, http.MethodDelete, fmt.Sprintf("%s/%s", f.enrollmentAPIKeyPath(), keyID), nil, nil) +} + func (f fleetAPI) findAgentPolicy(ctx context.Context, filter func(policy AgentPolicy) bool) (AgentPolicy, error) { page := 1 for { @@ -177,7 +184,6 @@ func (f fleetAPI) findAgentPolicy(ctx context.Context, filter func(policy AgentP return AgentPolicy{}, err } if len(list.Items) == 0 { - // goto NOTFOUND break } for _, p := range list.Items { @@ -187,7 +193,6 @@ func (f fleetAPI) findAgentPolicy(ctx context.Context, filter func(policy AgentP } page++ } - // NOTFOUND: return AgentPolicy{}, errors.New("no matching agent policy found") } @@ -211,27 +216,52 @@ func (f fleetAPI) DefaultAgentPolicyID(ctx context.Context) (string, error) { return policy.ID, nil } +// todo name +func maybeReconcileFleetEnrollment(params Params) (string, error) { + if !params.Agent.Spec.KibanaRef.IsDefined() { + return "", nil + } + + kbConnectionSettings, err := extractClientConnectionSettings(params.Agent, params.Client, commonv1.KibanaAssociationType) + if err != nil { + return "", err + } + + token, err := reconcileEnrollmentToken( + params.Context, params.Agent, params.Client, + newFleetAPI( + params.OperatorParams.Dialer, + kbConnectionSettings, + params.Logger().WithValues("namespace", params.Agent.Namespace, "agent_name", params.Agent.Name)), + ) + return token, err +} + func reconcileEnrollmentToken( ctx context.Context, agent agentv1alpha1.Agent, client k8s.Client, api fleetAPI, ) (string, error) { + // do we have an existing token that we have rolled out previously? tokenName, exists := agent.Annotations[FleetTokenAnnotation] + // what policy should we enroll this agent in? policyID, err := reconcilePolicyID(ctx, agent, api) if err != nil { return "", err } if exists { + // get the enrollment token identified by the annotation key, err := api.GetEnrollmentAPIKey(ctx, tokenName) + // the annotation might contain corrupted or no longer valid information if err != nil && IsNotFound(err) { goto CREATE } if err != nil { return "", err } - // TODO check token is for the correct policy - if key.Active { + // if the token is valid and for the right policy we are done here + if key.Active && key.PolicyID == policyID { return key.APIKey, nil } } @@ -241,13 +271,14 @@ CREATE: if err != nil { return "", err } - // TODO this creates conflicts solve on top level + // this potentially creates conflicts we could introduce reconciler state similar to the ES controller and handle it + // on the top level but we would then potentially create redundant enrollment tokens in the Fleet API agent.Annotations[FleetTokenAnnotation] = key.ID err = client.Update(ctx, &agent) if err != nil { - return "", err + // we have failed to store the token id in an annotation let's try to remove the token again + return "", k8serrors.NewAggregate([]error{err, api.DeleteEnrollmentAPIKey(ctx, key.ID)}) } - // TODO failed update creates dangling API key return key.APIKey, nil } diff --git a/pkg/controller/agent/pod.go b/pkg/controller/agent/pod.go index ba45fc9044..12013ca727 100644 --- a/pkg/controller/agent/pod.go +++ b/pkg/controller/agent/pod.go @@ -5,13 +5,12 @@ package agent import ( - "context" + "errors" "fmt" "hash" "path" "sort" - "github.com/go-logr/logr" pkgerrors "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -32,7 +31,6 @@ import ( "github.com/elastic/cloud-on-k8s/pkg/controller/common/volume" "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" "github.com/elastic/cloud-on-k8s/pkg/utils/maps" - netutil "github.com/elastic/cloud-on-k8s/pkg/utils/net" ) const ( @@ -121,7 +119,7 @@ var ( } ) -func buildPodTemplate(params Params, fleetCerts *certificates.CertificatesSecret, configHash hash.Hash32) (corev1.PodTemplateSpec, error) { +func buildPodTemplate(params Params, fleetCerts *certificates.CertificatesSecret, fleetToken string, configHash hash.Hash32) (corev1.PodTemplateSpec, error) { defer tracing.Span(¶ms.Context)() spec := ¶ms.Agent.Spec builder := defaults.NewPodTemplateBuilder(params.GetPodTemplate(), ContainerName) @@ -138,7 +136,7 @@ func buildPodTemplate(params Params, fleetCerts *certificates.CertificatesSecret // fleet mode requires some special treatment if spec.FleetModeEnabled() { var err error - if builder, err = amendBuilderForFleetMode(params, fleetCerts, builder, configHash); err != nil { + if builder, err = amendBuilderForFleetMode(params, fleetCerts, fleetToken, builder, configHash); err != nil { return corev1.PodTemplateSpec{}, err } } else if spec.StandaloneModeEnabled() { @@ -186,7 +184,7 @@ func buildPodTemplate(params Params, fleetCerts *certificates.CertificatesSecret return builder.PodTemplate, nil } -func amendBuilderForFleetMode(params Params, fleetCerts *certificates.CertificatesSecret, builder *defaults.PodTemplateBuilder, configHash hash.Hash) (*defaults.PodTemplateBuilder, error) { +func amendBuilderForFleetMode(params Params, fleetCerts *certificates.CertificatesSecret, fleetToken string, builder *defaults.PodTemplateBuilder, configHash hash.Hash) (*defaults.PodTemplateBuilder, error) { esAssociation, err := getRelatedEsAssoc(params) if err != nil { return nil, err @@ -203,7 +201,7 @@ func amendBuilderForFleetMode(params Params, fleetCerts *certificates.Certificat } // ES, Kibana and FleetServer connection info are inject using environment variables - builder, err = applyEnvVars(params, builder) + builder, err = applyEnvVars(params, fleetToken, builder) if err != nil { return nil, err } @@ -228,8 +226,8 @@ func amendBuilderForFleetMode(params Params, fleetCerts *certificates.Certificat return builder, nil } -func applyEnvVars(params Params, builder *defaults.PodTemplateBuilder) (*defaults.PodTemplateBuilder, error) { - fleetModeEnvVars, err := getFleetModeEnvVars(params.Context, params.Agent, params.Client, params.OperatorParams.Dialer, params.Logger()) +func applyEnvVars(params Params, fleetToken string, builder *defaults.PodTemplateBuilder) (*defaults.PodTemplateBuilder, error) { + fleetModeEnvVars, err := getFleetModeEnvVars(params.Agent, params.Client, fleetToken) if err != nil { return nil, err } @@ -430,15 +428,15 @@ func certificatesDir(association commonv1.Association) string { ) } -func getFleetModeEnvVars(ctx context.Context, agent agentv1alpha1.Agent, client k8s.Client, dialer netutil.Dialer, logger logr.Logger) (map[string]string, error) { +func getFleetModeEnvVars(agent agentv1alpha1.Agent, client k8s.Client, fleetToken string) (map[string]string, error) { result := map[string]string{} - for _, f := range []func(context.Context, agentv1alpha1.Agent, k8s.Client, netutil.Dialer, logr.Logger) (map[string]string, error){ + for _, f := range []func(agentv1alpha1.Agent, k8s.Client, string) (map[string]string, error){ getFleetSetupKibanaEnvVars, getFleetSetupFleetEnvVars, getFleetSetupFleetServerEnvVars, } { - envVars, err := f(ctx, agent, client, dialer, logger) + envVars, err := f(agent, client, fleetToken) if err != nil { return nil, err } @@ -448,31 +446,23 @@ func getFleetModeEnvVars(ctx context.Context, agent agentv1alpha1.Agent, client return result, nil } -func getFleetSetupKibanaEnvVars(ctx context.Context, agent agentv1alpha1.Agent, client k8s.Client, dialer netutil.Dialer, logger logr.Logger) (map[string]string, error) { +func getFleetSetupKibanaEnvVars(agent agentv1alpha1.Agent, _ k8s.Client, fleetToken string) (map[string]string, error) { if !agent.Spec.KibanaRef.IsDefined() { return map[string]string{}, nil } - kbConnectionSettings, err := extractClientConnectionSettings(agent, client, commonv1.KibanaAssociationType) - if err != nil { - return nil, err + if fleetToken == "" { + return nil, errors.New("fleet enrollment token must not be empty, potential programmer error") } - token, err := reconcileEnrollmentToken( - ctx, agent, client, - newFleetAPI(dialer, kbConnectionSettings, logger.WithValues("namespace", agent.Namespace, "agent_name", agent.Name)), - ) - if err != nil { - return nil, err - } envVars := map[string]string{ - FleetEnrollmentToken: token, + FleetEnrollmentToken: fleetToken, } return envVars, nil } -func getFleetSetupFleetEnvVars(_ context.Context, agent agentv1alpha1.Agent, client k8s.Client, _ netutil.Dialer, _ logr.Logger) (map[string]string, error) { +func getFleetSetupFleetEnvVars(agent agentv1alpha1.Agent, client k8s.Client, _ string) (map[string]string, error) { fleetCfg := map[string]string{} if agent.Spec.KibanaRef.IsDefined() { @@ -514,7 +504,7 @@ func getFleetSetupFleetEnvVars(_ context.Context, agent agentv1alpha1.Agent, cli return fleetCfg, nil } -func getFleetSetupFleetServerEnvVars(_ context.Context, agent agentv1alpha1.Agent, client k8s.Client, _ netutil.Dialer, _ logr.Logger) (map[string]string, error) { +func getFleetSetupFleetServerEnvVars(agent agentv1alpha1.Agent, client k8s.Client, _ string) (map[string]string, error) { if !agent.Spec.FleetServerEnabled { return map[string]string{}, nil } From af7eb453db24b9a83028c9b8d26f018c4a6007b5 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Wed, 29 Jun 2022 15:17:14 +0200 Subject: [PATCH 04/26] fix unit tests --- pkg/controller/agent/pod.go | 10 +- pkg/controller/agent/pod_test.go | 182 +++++++------------------------ 2 files changed, 41 insertions(+), 151 deletions(-) diff --git a/pkg/controller/agent/pod.go b/pkg/controller/agent/pod.go index ab14953aed..87d7439229 100644 --- a/pkg/controller/agent/pod.go +++ b/pkg/controller/agent/pod.go @@ -55,13 +55,6 @@ const ( // VersionLabelName is a label used to track the version of a Agent Pod. VersionLabelName = "agent.k8s.elastic.co/version" - // Below are the names of environment variables used to configure Elastic Agent to Kibana connection in Fleet mode. - KibanaFleetHost = "KIBANA_FLEET_HOST" - KibanaFleetUsername = "KIBANA_FLEET_USERNAME" - KibanaFleetPassword = "KIBANA_FLEET_PASSWORD" //nolint:gosec - KibanaFleetSetup = "KIBANA_FLEET_SETUP" - KibanaFleetCA = "KIBANA_FLEET_CA" - // Below are the names of environment variables used to configure Elastic Agent to Fleet connection in Fleet mode. FleetEnroll = "FLEET_ENROLL" FleetEnrollmentToken = "FLEET_ENROLLMENT_TOKEN" @@ -112,8 +105,7 @@ var ( } secretEnvVarNames = map[string]struct{}{ - KibanaFleetUsername: {}, - KibanaFleetPassword: {}, + FleetEnrollmentToken: {}, FleetServerElasticsearchUsername: {}, FleetServerElasticsearchPassword: {}, } diff --git a/pkg/controller/agent/pod_test.go b/pkg/controller/agent/pod_test.go index e74d8639f8..ba9b8d9bcb 100644 --- a/pkg/controller/agent/pod_test.go +++ b/pkg/controller/agent/pod_test.go @@ -11,14 +11,13 @@ import ( "path" "testing" + "github.com/go-test/deep" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "github.com/go-test/deep" - agentv1alpha1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/agent/v1alpha1" commonv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/common/v1" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/certificates" @@ -178,7 +177,7 @@ func Test_amendBuilderForFleetMode(t *testing.T) { builder := generateBuilder() hash := sha256.New224() - gotBuilder, gotErr := amendBuilderForFleetMode(tt.params, fleetCerts, builder, hash) + gotBuilder, gotErr := amendBuilderForFleetMode(tt.params, fleetCerts, "", builder, hash) require.Nil(t, gotErr) require.NotNil(t, gotBuilder) @@ -230,13 +229,14 @@ func Test_applyEnvVars(t *testing.T) { URL: "kb-url", }) - podTemplateBuilderWithFleetCASet := generateBuilder() - podTemplateBuilderWithFleetCASet = podTemplateBuilderWithFleetCASet.WithEnv(corev1.EnvVar{Name: "KIBANA_FLEET_CA", Value: ""}) + podTemplateBuilderWithFleetTokenSet := generateBuilder() + podTemplateBuilderWithFleetTokenSet = podTemplateBuilderWithFleetTokenSet.WithEnv(corev1.EnvVar{Name: "FLEET_ENROLLMENT_TOKEN", Value: "custom"}) f := false for _, tt := range []struct { name string params Params + fleetToken string podTemplateBuilder *defaults.PodTemplateBuilder wantContainer corev1.Container wantSecretData map[string][]byte @@ -244,86 +244,46 @@ func Test_applyEnvVars(t *testing.T) { { name: "elastic agent, without fleet server, with fleet server ref, with kibana ref", params: Params{ - Agent: agent, - Client: k8s.NewFakeClient( - &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Name: "kb-secret-name", Namespace: "default"}, - Data: map[string][]byte{"kb-user": []byte("kb-password")}, - }, - &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Name: "kb-ca-secret-name", Namespace: "default"}, - Data: map[string][]byte{"kb-user": []byte("kb-password")}, - }, - ), + Agent: agent, + Client: k8s.NewFakeClient(), }, + fleetToken: "test-token", podTemplateBuilder: generateBuilder(), wantContainer: corev1.Container{ Name: "agent", Env: []corev1.EnvVar{ {Name: "FLEET_CA", Value: "/mnt/elastic-internal/fleetserver-association/default/fs/certs/ca.crt"}, {Name: "FLEET_ENROLL", Value: "true"}, - {Name: "FLEET_URL", Value: "fs-url"}, - {Name: "KIBANA_FLEET_CA", Value: "/mnt/elastic-internal/kibana-association/default/kb/certs/ca.crt"}, - {Name: "KIBANA_FLEET_HOST", Value: "kb-url"}, - {Name: "KIBANA_FLEET_PASSWORD", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ + {Name: "FLEET_ENROLLMENT_TOKEN", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{Name: "agent-agent-envvars"}, - Key: "KIBANA_FLEET_PASSWORD", - Optional: &f, - }}}, - {Name: "KIBANA_FLEET_SETUP", Value: "true"}, - {Name: "KIBANA_FLEET_USERNAME", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: "agent-agent-envvars"}, - Key: "KIBANA_FLEET_USERNAME", + Key: "FLEET_ENROLLMENT_TOKEN", Optional: &f, }}}, + {Name: "FLEET_URL", Value: "fs-url"}, }, }, wantSecretData: map[string][]byte{ - "KIBANA_FLEET_USERNAME": []byte("kb-user"), - "KIBANA_FLEET_PASSWORD": []byte("kb-password"), + "FLEET_ENROLLMENT_TOKEN": []byte("test-token"), }, }, { - name: "elastic agent, without fleet server, with fleet server ref, with kibana ref, ca override", + name: "elastic agent, without fleet server, with fleet server ref, with kibana ref, token override", params: Params{ - Agent: agent, - Client: k8s.NewFakeClient( - &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Name: "kb-secret-name", Namespace: "default"}, - Data: map[string][]byte{"kb-user": []byte("kb-password")}, - }, - &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Name: "kb-ca-secret-name", Namespace: "default"}, - Data: map[string][]byte{"kb-user": []byte("kb-password")}, - }, - ), + Agent: agent, + Client: k8s.NewFakeClient(), }, - podTemplateBuilder: podTemplateBuilderWithFleetCASet, + fleetToken: "test-token", + podTemplateBuilder: podTemplateBuilderWithFleetTokenSet, wantContainer: corev1.Container{ Name: "agent", Env: []corev1.EnvVar{ - {Name: "KIBANA_FLEET_CA", Value: ""}, + {Name: "FLEET_ENROLLMENT_TOKEN", Value: "custom"}, {Name: "FLEET_CA", Value: "/mnt/elastic-internal/fleetserver-association/default/fs/certs/ca.crt"}, {Name: "FLEET_ENROLL", Value: "true"}, {Name: "FLEET_URL", Value: "fs-url"}, - {Name: "KIBANA_FLEET_HOST", Value: "kb-url"}, - {Name: "KIBANA_FLEET_PASSWORD", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: "agent-agent-envvars"}, - Key: "KIBANA_FLEET_PASSWORD", - Optional: &f, - }}}, - {Name: "KIBANA_FLEET_SETUP", Value: "true"}, - {Name: "KIBANA_FLEET_USERNAME", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: "agent-agent-envvars"}, - Key: "KIBANA_FLEET_USERNAME", - Optional: &f, - }}}, }, }, - wantSecretData: map[string][]byte{ - "KIBANA_FLEET_USERNAME": []byte("kb-user"), - "KIBANA_FLEET_PASSWORD": []byte("kb-password"), - }, + wantSecretData: nil, }, { name: "elastic agent, with fleet server, with kibana ref", @@ -341,12 +301,6 @@ func Test_applyEnvVars(t *testing.T) { }, }, }, - &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Name: "kb-secret-name", Namespace: "default"}, - Data: map[string][]byte{ - "kb-user": []byte("kb-password"), - }, - }, &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Name: "es-secret-name", Namespace: "default"}, Data: map[string][]byte{ @@ -355,12 +309,18 @@ func Test_applyEnvVars(t *testing.T) { }, ), }, + fleetToken: "test-token", podTemplateBuilder: generateBuilder(), wantContainer: corev1.Container{ Name: "agent", Env: []corev1.EnvVar{ {Name: "FLEET_CA", Value: "/usr/share/fleet-server/config/http-certs/ca.crt"}, {Name: "FLEET_ENROLL", Value: "true"}, + {Name: "FLEET_ENROLLMENT_TOKEN", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "agent-agent-envvars"}, + Key: "FLEET_ENROLLMENT_TOKEN", + Optional: &f, + }}}, {Name: "FLEET_SERVER_CERT", Value: "/usr/share/fleet-server/config/http-certs/tls.crt"}, {Name: "FLEET_SERVER_CERT_KEY", Value: "/usr/share/fleet-server/config/http-certs/tls.key"}, {Name: "FLEET_SERVER_ELASTICSEARCH_HOST", Value: "es-url"}, @@ -376,30 +336,17 @@ func Test_applyEnvVars(t *testing.T) { }}}, {Name: "FLEET_SERVER_ENABLE", Value: "true"}, {Name: "FLEET_URL", Value: "https://agent-agent-http.default.svc:8220"}, - {Name: "KIBANA_FLEET_HOST", Value: "kb-url"}, - {Name: "KIBANA_FLEET_PASSWORD", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: "agent-agent-envvars"}, - Key: "KIBANA_FLEET_PASSWORD", - Optional: &f, - }}}, - {Name: "KIBANA_FLEET_SETUP", Value: "true"}, - {Name: "KIBANA_FLEET_USERNAME", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: "agent-agent-envvars"}, - Key: "KIBANA_FLEET_USERNAME", - Optional: &f, - }}}, }, }, wantSecretData: map[string][]byte{ - "KIBANA_FLEET_USERNAME": []byte("kb-user"), - "KIBANA_FLEET_PASSWORD": []byte("kb-password"), + "FLEET_ENROLLMENT_TOKEN": []byte("test-token"), "FLEET_SERVER_ELASTICSEARCH_USERNAME": []byte("es-user"), "FLEET_SERVER_ELASTICSEARCH_PASSWORD": []byte("es-password"), }, }, } { t.Run(tt.name, func(t *testing.T) { - gotBuilder, err := applyEnvVars(tt.params, tt.podTemplateBuilder) + gotBuilder, err := applyEnvVars(tt.params, tt.fleetToken, tt.podTemplateBuilder) require.NoError(t, err) @@ -812,15 +759,6 @@ func Test_writeEsAssocToConfigHash(t *testing.T) { } func Test_getFleetSetupKibanaEnvVars(t *testing.T) { - client := k8s.NewFakeClient(&corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "ns", - Name: "secret-name", - }, - Data: map[string][]byte{ - "user": []byte("password"), - }, - }) agent := agentv1alpha1.Agent{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns", @@ -833,79 +771,39 @@ func Test_getFleetSetupKibanaEnvVars(t *testing.T) { }, }, } - agent2 := agent - - assocWithoutCa := &agentv1alpha1.AgentKibanaAssociation{ - Agent: &agent, - } - - assocWithCa := &agentv1alpha1.AgentKibanaAssociation{ - Agent: &agent2, - } - - assocWithoutCa.SetAssociationConf(&commonv1.AssociationConf{ - AuthSecretName: "secret-name", - AuthSecretKey: "user", - URL: "url", - }) - - assocWithCa.SetAssociationConf(&commonv1.AssociationConf{ - AuthSecretName: "secret-name", - AuthSecretKey: "user", - URL: "url", - CACertProvided: true, - CASecretName: "ca-secret-name", - }) for _, tt := range []struct { name string agent agentv1alpha1.Agent + fleetToken string wantErr bool wantEnvVars map[string]string - client k8s.Client }{ { name: "no kibana ref", agent: agentv1alpha1.Agent{}, wantEnvVars: map[string]string{}, wantErr: false, - client: client, }, { - name: "kibana ref present, kibana without ca populated", - agent: *assocWithoutCa.Agent, - wantEnvVars: map[string]string{ - "KIBANA_FLEET_HOST": "url", - "KIBANA_FLEET_USERNAME": "user", - "KIBANA_FLEET_PASSWORD": "password", - "KIBANA_FLEET_SETUP": "true", - }, - wantErr: false, - client: client, + name: "kibana ref present, but no token", + agent: agent, + fleetToken: "", + wantEnvVars: nil, + wantErr: true, }, { - name: "kibana ref present, kibana with ca populated", - agent: *assocWithCa.Agent, + name: "kibana ref present, token populated", + agent: agent, + fleetToken: "test-token", wantEnvVars: map[string]string{ - "KIBANA_FLEET_HOST": "url", - "KIBANA_FLEET_USERNAME": "user", - "KIBANA_FLEET_PASSWORD": "password", - "KIBANA_FLEET_SETUP": "true", - "KIBANA_FLEET_CA": "/mnt/elastic-internal/kibana-association/ns/kibana/certs/ca.crt", + "FLEET_ENROLLMENT_TOKEN": "test-token", }, wantErr: false, - client: client, - }, - { - name: "no user secret", - agent: *assocWithoutCa.Agent, - wantEnvVars: nil, - wantErr: true, - client: k8s.NewFakeClient(), }, } { t.Run(tt.name, func(t *testing.T) { - gotEnvVars, gotErr := getFleetSetupKibanaEnvVars(tt.agent, tt.client) + gotEnvVars, gotErr := getFleetSetupKibanaEnvVars(tt.agent, k8s.NewFakeClient(), tt.fleetToken) require.Equal(t, tt.wantEnvVars, gotEnvVars) require.Equal(t, tt.wantErr, gotErr != nil) @@ -1078,7 +976,7 @@ func Test_getFleetSetupFleetEnvVars(t *testing.T) { }, } { t.Run(tt.name, func(t *testing.T) { - gotEnvVars, gotErr := getFleetSetupFleetEnvVars(tt.agent, tt.client) + gotEnvVars, gotErr := getFleetSetupFleetEnvVars(tt.agent, tt.client, "") require.Equal(t, tt.wantEnvVars, gotEnvVars) require.Equal(t, tt.wantErr, gotErr != nil) @@ -1198,7 +1096,7 @@ func Test_getFleetSetupFleetServerEnvVars(t *testing.T) { }, } { t.Run(tt.name, func(t *testing.T) { - gotEnvVars, gotErr := getFleetSetupFleetServerEnvVars(tt.agent, tt.client) + gotEnvVars, gotErr := getFleetSetupFleetServerEnvVars(tt.agent, tt.client, "") require.Equal(t, tt.wantEnvVars, gotEnvVars) require.Equal(t, tt.wantErr, gotErr != nil) From e5fb3e214c030f321d6f3aa7e5c7775be6ca87bf Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Wed, 29 Jun 2022 15:47:13 +0200 Subject: [PATCH 05/26] refactor common error handling --- pkg/controller/agent/fleet.go | 33 ++------------------- pkg/controller/common/http/http_client.go | 35 +++++++++++++++++++++++ test/e2e/test/kibana/http_client.go | 17 ++--------- 3 files changed, 41 insertions(+), 44 deletions(-) diff --git a/pkg/controller/agent/fleet.go b/pkg/controller/agent/fleet.go index 1c5f84c98a..32766dba36 100644 --- a/pkg/controller/agent/fleet.go +++ b/pkg/controller/agent/fleet.go @@ -28,29 +28,6 @@ import ( const FleetTokenAnnotation = "fleet.eck.k8s.elastic.co/token" -// TODO common kibana/http client? -type APIError struct { - StatusCode int - msg string -} - -func (e *APIError) Error() string { - return e.msg -} - -// IsNotFound checks whether the error was an HTTP 404 error. -func IsNotFound(err error) bool { - return isHTTPError(err, http.StatusNotFound) -} - -func isHTTPError(err error, statusCode int) bool { - apiErr := new(APIError) - if errors.As(err, &apiErr) { - return apiErr.StatusCode == statusCode - } - return false -} - type EnrollmentAPIKeyResult struct { Item EnrollmentAPIKey `json:"item"` } @@ -134,11 +111,8 @@ func (f fleetAPI) request( } defer resp.Body.Close() - if resp.StatusCode < 200 || resp.StatusCode > 299 { - return &APIError{ - StatusCode: resp.StatusCode, - msg: fmt.Sprintf("failed to request %s, status is %d)", request.URL.Redacted(), resp.StatusCode), - } + if err := commonhttp.MaybeAPIError(resp); err != nil { + return err } if responseObj != nil { if err := json.NewDecoder(resp.Body).Decode(responseObj); err != nil { @@ -216,7 +190,6 @@ func (f fleetAPI) DefaultAgentPolicyID(ctx context.Context) (string, error) { return policy.ID, nil } -// todo name func maybeReconcileFleetEnrollment(params Params) (string, error) { if !params.Agent.Spec.KibanaRef.IsDefined() { return "", nil @@ -254,7 +227,7 @@ func reconcileEnrollmentToken( // get the enrollment token identified by the annotation key, err := api.GetEnrollmentAPIKey(ctx, tokenName) // the annotation might contain corrupted or no longer valid information - if err != nil && IsNotFound(err) { + if err != nil && commonhttp.IsNotFound(err) { goto CREATE } if err != nil { diff --git a/pkg/controller/common/http/http_client.go b/pkg/controller/common/http/http_client.go index 4275d8499e..86fec2d911 100644 --- a/pkg/controller/common/http/http_client.go +++ b/pkg/controller/common/http/http_client.go @@ -8,6 +8,7 @@ import ( "crypto/tls" "crypto/x509" "errors" + "fmt" "net/http" "time" @@ -67,3 +68,37 @@ func Client(dialer net.Dialer, caCerts []*x509.Certificate, timeout time.Duratio Timeout: timeout, } } + +// APIError to represent non-200 HTTP responses as Go errors. +type APIError struct { + StatusCode int + msg string +} + +// MaybeAPIError creates an APIError from a http.Response if the status code is not 2xx. +func MaybeAPIError(resp *http.Response) *APIError { + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return &APIError{ + StatusCode: resp.StatusCode, + msg: fmt.Sprintf("failed to request %s, status is %d)", resp.Request.URL.Redacted(), resp.StatusCode), + } + } + return nil +} + +func (e *APIError) Error() string { + return e.msg +} + +// IsNotFound checks whether the error was an HTTP 404 error. +func IsNotFound(err error) bool { + return isHTTPError(err, http.StatusNotFound) +} + +func isHTTPError(err error, statusCode int) bool { + apiErr := new(APIError) + if errors.As(err, &apiErr) { + return apiErr.StatusCode == statusCode + } + return false +} diff --git a/test/e2e/test/kibana/http_client.go b/test/e2e/test/kibana/http_client.go index ceb35db706..90453d1349 100644 --- a/test/e2e/test/kibana/http_client.go +++ b/test/e2e/test/kibana/http_client.go @@ -15,19 +15,11 @@ import ( "github.com/pkg/errors" kbv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/kibana/v1" + commonhttp "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/http" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/kibana/network" "github.com/elastic/cloud-on-k8s/v2/test/e2e/test" ) -type APIError struct { - StatusCode int - msg string -} - -func (e *APIError) Error() string { - return e.msg -} - func NewKibanaClient(kb kbv1.Kibana, k *test.K8sClient) (*http.Client, error) { var caCerts []*x509.Certificate if kb.Spec.HTTP.TLS.Enabled() { @@ -78,11 +70,8 @@ func DoRequest(k *test.K8sClient, kb kbv1.Kibana, password string, method string return nil, errors.Wrap(err, "while doing request") } defer resp.Body.Close() - if resp.StatusCode < 200 || resp.StatusCode > 299 { - return nil, &APIError{ - StatusCode: resp.StatusCode, - msg: fmt.Sprintf("fail to request %s, status is %d)", pathAndQuery, resp.StatusCode), - } + if err := commonhttp.MaybeAPIError(resp); err != nil { + return nil, err } return ioutil.ReadAll(resp.Body) From 62eb37133729a9d589c479dfa019fb329c571e85 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Wed, 29 Jun 2022 15:56:41 +0200 Subject: [PATCH 06/26] Add policy field to Agent CRD --- config/crds/v1/all-crds.yaml | 5 +++++ config/crds/v1/bases/agent.k8s.elastic.co_agents.yaml | 5 +++++ .../charts/eck-operator-crds/templates/all-crds.yaml | 5 +++++ docs/reference/api-docs.asciidoc | 1 + pkg/apis/agent/v1alpha1/agent_types.go | 5 +++++ pkg/controller/agent/fleet.go | 6 +++--- 6 files changed, 24 insertions(+), 3 deletions(-) diff --git a/config/crds/v1/all-crds.yaml b/config/crds/v1/all-crds.yaml index 706e63ae5a..f8be038312 100644 --- a/config/crds/v1/all-crds.yaml +++ b/config/crds/v1/all-crds.yaml @@ -752,6 +752,11 @@ spec: - standalone - fleet type: string + policyID: + description: PolicyID optionally determines into which Agent Policy + this Agent will be enrolled. If left empty the default policy will + be used. + type: string secureSettings: description: SecureSettings is a list of references to Kubernetes Secrets containing sensitive configuration options for the Agent. diff --git a/config/crds/v1/bases/agent.k8s.elastic.co_agents.yaml b/config/crds/v1/bases/agent.k8s.elastic.co_agents.yaml index 763f56d898..7647776fd1 100644 --- a/config/crds/v1/bases/agent.k8s.elastic.co_agents.yaml +++ b/config/crds/v1/bases/agent.k8s.elastic.co_agents.yaml @@ -15683,6 +15683,11 @@ spec: - standalone - fleet type: string + policyID: + description: PolicyID optionally determines into which Agent Policy + this Agent will be enrolled. If left empty the default policy will + be used. + type: string secureSettings: description: SecureSettings is a list of references to Kubernetes Secrets containing sensitive configuration options for the Agent. diff --git a/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml b/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml index 09c813a63d..0096abb71b 100644 --- a/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml +++ b/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml @@ -758,6 +758,11 @@ spec: - standalone - fleet type: string + policyID: + description: PolicyID optionally determines into which Agent Policy + this Agent will be enrolled. If left empty the default policy will + be used. + type: string secureSettings: description: SecureSettings is a list of references to Kubernetes Secrets containing sensitive configuration options for the Agent. diff --git a/docs/reference/api-docs.asciidoc b/docs/reference/api-docs.asciidoc index 30aebdfef9..6b126ddbeb 100644 --- a/docs/reference/api-docs.asciidoc +++ b/docs/reference/api-docs.asciidoc @@ -92,6 +92,7 @@ AgentSpec defines the desired state of the Agent | *`http`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-httpconfig[$$HTTPConfig$$]__ | HTTP holds the HTTP layer configuration for the Agent in Fleet mode with Fleet Server enabled. | *`mode`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-agent-v1alpha1-agentmode[$$AgentMode$$]__ | Mode specifies the source of configuration for the Agent. The configuration can be specified locally through `config` or `configRef` (`standalone` mode), or come from Fleet during runtime (`fleet` mode). Defaults to `standalone` mode. | *`fleetServerEnabled`* __boolean__ | FleetServerEnabled determines whether this Agent will launch Fleet Server. Don't set unless `mode` is set to `fleet`. +| *`policyID`* __string__ | PolicyID optionally determines into which Agent Policy this Agent will be enrolled. If left empty the default policy will be used. | *`kibanaRef`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-objectselector[$$ObjectSelector$$]__ | KibanaRef is a reference to Kibana where Fleet should be set up and this Agent should be enrolled. Don't set unless `mode` is set to `fleet`. | *`fleetServerRef`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-objectselector[$$ObjectSelector$$]__ | FleetServerRef is a reference to Fleet Server that this Agent should connect to to obtain it's configuration. Don't set unless `mode` is set to `fleet`. |=== diff --git a/pkg/apis/agent/v1alpha1/agent_types.go b/pkg/apis/agent/v1alpha1/agent_types.go index d530dfb552..6bcd1bfb9b 100644 --- a/pkg/apis/agent/v1alpha1/agent_types.go +++ b/pkg/apis/agent/v1alpha1/agent_types.go @@ -84,6 +84,11 @@ type AgentSpec struct { // +kubebuilder:validation:Optional FleetServerEnabled bool `json:"fleetServerEnabled,omitempty"` + // PolicyID optionally determines into which Agent Policy this Agent will be enrolled. If left empty the default + // policy will be used. + // +kubebuilder:validation:Optional + PolicyID string `json:"policyID,omitempty"` + // KibanaRef is a reference to Kibana where Fleet should be set up and this Agent should be enrolled. Don't set // unless `mode` is set to `fleet`. // +kubebuilder:validation:Optional diff --git a/pkg/controller/agent/fleet.go b/pkg/controller/agent/fleet.go index 32766dba36..712b4da0d5 100644 --- a/pkg/controller/agent/fleet.go +++ b/pkg/controller/agent/fleet.go @@ -256,9 +256,9 @@ CREATE: } func reconcilePolicyID(ctx context.Context, agent agentv1alpha1.Agent, api fleetAPI) (string, error) { - /*if agent.Spec.PolicyID != "" { - return agent.Spec.PolicyID - }*/ + if agent.Spec.PolicyID != "" { + return agent.Spec.PolicyID, nil + } if agent.Spec.FleetServerEnabled { return api.DefaultFleetServerPolicyID(ctx) } From 27a480adb0228418df67e1f7976ebdd583ae2ae0 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Wed, 29 Jun 2022 16:38:26 +0200 Subject: [PATCH 07/26] lint --- pkg/controller/agent/fleet.go | 44 +++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/pkg/controller/agent/fleet.go b/pkg/controller/agent/fleet.go index 712b4da0d5..526403a69b 100644 --- a/pkg/controller/agent/fleet.go +++ b/pkg/controller/agent/fleet.go @@ -26,12 +26,14 @@ import ( "github.com/elastic/cloud-on-k8s/v2/pkg/utils/stringsutil" ) -const FleetTokenAnnotation = "fleet.eck.k8s.elastic.co/token" +const FleetTokenAnnotation = "fleet.eck.k8s.elastic.co/token" //nolint:gosec +// EnrollmentAPIKeyResult wrapper for a single result in the Fleet API. type EnrollmentAPIKeyResult struct { Item EnrollmentAPIKey `json:"item"` } +// EnrollmentAPIKey is the representation of an enrollment token in the Fleet API. type EnrollmentAPIKey struct { ID string `json:"id,omitempty"` Active bool `json:"active,omitempty"` @@ -39,11 +41,13 @@ type EnrollmentAPIKey struct { PolicyID string `json:"policy_id,omitempty"` } -type AgentPolicyList struct { - Items []AgentPolicy `json:"items"` +// PolicyList is a wrapper for a list of agent policies as returned by the Fleet API. +type PolicyList struct { //nolint:revive + Items []Policy `json:"items"` } -type AgentPolicy struct { +// Policy is the representation of an agent policy in the Fleet API. +type Policy struct { ID string `json:"id"` IsDefault bool `json:"is_default"` IsDefaultFleetServer bool `json:"is_default_fleet_server"` @@ -132,30 +136,30 @@ func (f fleetAPI) enrollmentAPIKeyPath() string { return path } -func (f fleetAPI) CreateEnrollmentAPIKey(ctx context.Context, policyID string) (EnrollmentAPIKey, error) { +func (f fleetAPI) createEnrollmentAPIKey(ctx context.Context, policyID string) (EnrollmentAPIKey, error) { var response EnrollmentAPIKeyResult err := f.request(ctx, http.MethodPost, f.enrollmentAPIKeyPath(), EnrollmentAPIKey{PolicyID: policyID}, &response) return response.Item, err } -func (f fleetAPI) GetEnrollmentAPIKey(ctx context.Context, keyID string) (EnrollmentAPIKey, error) { +func (f fleetAPI) getEnrollmentAPIKey(ctx context.Context, keyID string) (EnrollmentAPIKey, error) { var response EnrollmentAPIKeyResult err := f.request(ctx, http.MethodGet, fmt.Sprintf("%s/%s", f.enrollmentAPIKeyPath(), keyID), nil, &response) return response.Item, err } -func (f fleetAPI) DeleteEnrollmentAPIKey(ctx context.Context, keyID string) error { +func (f fleetAPI) deleteEnrollmentAPIKey(ctx context.Context, keyID string) error { return f.request(ctx, http.MethodDelete, fmt.Sprintf("%s/%s", f.enrollmentAPIKeyPath(), keyID), nil, nil) } -func (f fleetAPI) findAgentPolicy(ctx context.Context, filter func(policy AgentPolicy) bool) (AgentPolicy, error) { +func (f fleetAPI) findAgentPolicy(ctx context.Context, filter func(policy Policy) bool) (Policy, error) { page := 1 for { - var list AgentPolicyList + var list PolicyList err := f.request(ctx, http.MethodGet, fmt.Sprintf("agent_policies?perPage=20&page=%d", page), nil, &list) if err != nil { - return AgentPolicy{}, err + return Policy{}, err } if len(list.Items) == 0 { break @@ -167,11 +171,11 @@ func (f fleetAPI) findAgentPolicy(ctx context.Context, filter func(policy AgentP } page++ } - return AgentPolicy{}, errors.New("no matching agent policy found") + return Policy{}, errors.New("no matching agent policy found") } -func (f fleetAPI) DefaultFleetServerPolicyID(ctx context.Context) (string, error) { - policy, err := f.findAgentPolicy(ctx, func(policy AgentPolicy) bool { +func (f fleetAPI) defaultFleetServerPolicyID(ctx context.Context) (string, error) { + policy, err := f.findAgentPolicy(ctx, func(policy Policy) bool { return policy.IsDefaultFleetServer && policy.Status == "active" }) if err != nil { @@ -180,8 +184,8 @@ func (f fleetAPI) DefaultFleetServerPolicyID(ctx context.Context) (string, error return policy.ID, nil } -func (f fleetAPI) DefaultAgentPolicyID(ctx context.Context) (string, error) { - policy, err := f.findAgentPolicy(ctx, func(policy AgentPolicy) bool { +func (f fleetAPI) defaultAgentPolicyID(ctx context.Context) (string, error) { + policy, err := f.findAgentPolicy(ctx, func(policy Policy) bool { return policy.IsDefault && policy.Status == "active" }) if err != nil { @@ -225,7 +229,7 @@ func reconcileEnrollmentToken( } if exists { // get the enrollment token identified by the annotation - key, err := api.GetEnrollmentAPIKey(ctx, tokenName) + key, err := api.getEnrollmentAPIKey(ctx, tokenName) // the annotation might contain corrupted or no longer valid information if err != nil && commonhttp.IsNotFound(err) { goto CREATE @@ -240,7 +244,7 @@ func reconcileEnrollmentToken( } CREATE: - key, err := api.CreateEnrollmentAPIKey(ctx, policyID) + key, err := api.createEnrollmentAPIKey(ctx, policyID) if err != nil { return "", err } @@ -250,7 +254,7 @@ CREATE: err = client.Update(ctx, &agent) if err != nil { // we have failed to store the token id in an annotation let's try to remove the token again - return "", k8serrors.NewAggregate([]error{err, api.DeleteEnrollmentAPIKey(ctx, key.ID)}) + return "", k8serrors.NewAggregate([]error{err, api.deleteEnrollmentAPIKey(ctx, key.ID)}) } return key.APIKey, nil } @@ -260,8 +264,8 @@ func reconcilePolicyID(ctx context.Context, agent agentv1alpha1.Agent, api fleet return agent.Spec.PolicyID, nil } if agent.Spec.FleetServerEnabled { - return api.DefaultFleetServerPolicyID(ctx) + return api.defaultFleetServerPolicyID(ctx) } - return api.DefaultAgentPolicyID(ctx) + return api.defaultAgentPolicyID(ctx) } From cbbf8eec09c5aa1753cc93976bca56c82d9fe400 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Wed, 29 Jun 2022 19:51:40 +0200 Subject: [PATCH 08/26] more lint --- pkg/controller/agent/fleet.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pkg/controller/agent/fleet.go b/pkg/controller/agent/fleet.go index 526403a69b..3e1a7d8420 100644 --- a/pkg/controller/agent/fleet.go +++ b/pkg/controller/agent/fleet.go @@ -42,7 +42,7 @@ type EnrollmentAPIKey struct { } // PolicyList is a wrapper for a list of agent policies as returned by the Fleet API. -type PolicyList struct { //nolint:revive +type PolicyList struct { Items []Policy `json:"items"` } @@ -80,7 +80,6 @@ func (f fleetAPI) request( pathWithQuery string, requestObj, responseObj interface{}) error { - var body io.Reader = http.NoBody if requestObj != nil { outData, err := json.Marshal(requestObj) @@ -125,7 +124,6 @@ func (f fleetAPI) request( } return nil - } func (f fleetAPI) enrollmentAPIKeyPath() string { @@ -137,7 +135,6 @@ func (f fleetAPI) enrollmentAPIKeyPath() string { } func (f fleetAPI) createEnrollmentAPIKey(ctx context.Context, policyID string) (EnrollmentAPIKey, error) { - var response EnrollmentAPIKeyResult err := f.request(ctx, http.MethodPost, f.enrollmentAPIKeyPath(), EnrollmentAPIKey{PolicyID: policyID}, &response) return response.Item, err @@ -267,5 +264,4 @@ func reconcilePolicyID(ctx context.Context, agent agentv1alpha1.Agent, api fleet return api.defaultFleetServerPolicyID(ctx) } return api.defaultAgentPolicyID(ctx) - } From 87dbe848c4e79efebbc01ef6f5d9114040067b50 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Wed, 29 Jun 2022 20:37:47 +0200 Subject: [PATCH 09/26] check for Kibana reachability --- pkg/controller/agent/driver.go | 6 ++-- pkg/controller/agent/fleet.go | 38 ++++++++++++++++++--- pkg/controller/common/reconciler/results.go | 9 ++++- 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/pkg/controller/agent/driver.go b/pkg/controller/agent/driver.go index bdd1d8d13c..f5369c8dc8 100644 --- a/pkg/controller/agent/driver.go +++ b/pkg/controller/agent/driver.go @@ -129,9 +129,9 @@ func internalReconcile(params Params) (*reconciler.Results, agentv1alpha1.AgentS _, _ = configHash.Write(fleetCerts.Data[certificates.CertFileName]) } - fleetToken, err := maybeReconcileFleetEnrollment(params) - if err != nil { - return results.WithError(err), params.Status + fleetToken := maybeReconcileFleetEnrollment(params, results) + if results.HasRequeue() || results.HasError() { + return results, params.Status } if res := reconcileConfig(params, configHash); res.HasError() { diff --git a/pkg/controller/agent/fleet.go b/pkg/controller/agent/fleet.go index 3e1a7d8420..0af138dad9 100644 --- a/pkg/controller/agent/fleet.go +++ b/pkg/controller/agent/fleet.go @@ -16,11 +16,15 @@ import ( "github.com/go-logr/logr" "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/types" k8serrors "k8s.io/apimachinery/pkg/util/errors" + "sigs.k8s.io/controller-runtime/pkg/reconcile" agentv1alpha1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/agent/v1alpha1" commonv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/common/v1" + v1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/kibana/v1" commonhttp "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/http" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/reconciler" "github.com/elastic/cloud-on-k8s/v2/pkg/utils/k8s" "github.com/elastic/cloud-on-k8s/v2/pkg/utils/net" "github.com/elastic/cloud-on-k8s/v2/pkg/utils/stringsutil" @@ -42,7 +46,7 @@ type EnrollmentAPIKey struct { } // PolicyList is a wrapper for a list of agent policies as returned by the Fleet API. -type PolicyList struct { +type PolicyList struct { Items []Policy `json:"items"` } @@ -191,14 +195,25 @@ func (f fleetAPI) defaultAgentPolicyID(ctx context.Context) (string, error) { return policy.ID, nil } -func maybeReconcileFleetEnrollment(params Params) (string, error) { +func maybeReconcileFleetEnrollment(params Params, result *reconciler.Results) string { if !params.Agent.Spec.KibanaRef.IsDefined() { - return "", nil + return "" + } + + reachable, err := isKibanaReachable(params.Context, params.Client, params.Agent.Spec.KibanaRef.NamespacedName()) + if err != nil { + result.WithError(err) + return "" + } + if !reachable { + result.WithResult(reconcile.Result{Requeue: true}) + return "" } kbConnectionSettings, err := extractClientConnectionSettings(params.Agent, params.Client, commonv1.KibanaAssociationType) if err != nil { - return "", err + result.WithError(err) + return "" } token, err := reconcileEnrollmentToken( @@ -208,7 +223,20 @@ func maybeReconcileFleetEnrollment(params Params) (string, error) { kbConnectionSettings, params.Logger().WithValues("namespace", params.Agent.Namespace, "agent_name", params.Agent.Name)), ) - return token, err + result.WithError(err) + return token +} + +func isKibanaReachable(ctx context.Context, client k8s.Client, kibanaNSN types.NamespacedName) (bool, error) { + var kb v1.Kibana + err := client.Get(ctx, kibanaNSN, &kb) + if err != nil { + return false, err + } + if kb.Status.Health != commonv1.GreenHealth { + return false, nil // requeue + } + return true, nil } func reconcileEnrollmentToken( diff --git a/pkg/controller/common/reconciler/results.go b/pkg/controller/common/reconciler/results.go index f9a6e03253..385c6ede19 100644 --- a/pkg/controller/common/reconciler/results.go +++ b/pkg/controller/common/reconciler/results.go @@ -93,6 +93,13 @@ func (r *Results) HasError() bool { return len(r.errors) > 0 } +func (r *Results) HasRequeue() bool { + if r == nil { + return false + } + return r.currResult.Result.Requeue || r.currResult.Result.RequeueAfter > 0 +} + // WithResults appends the results and error from the other Results. func (r *Results) WithResults(other *Results) *Results { if other != nil { @@ -157,5 +164,5 @@ func (r *Results) IsReconciled() (bool, string) { if !r.currResult.incomplete { return true, "" } - return !(r.currResult.Result.Requeue || r.currResult.Result.RequeueAfter > 0), r.currResult.reason + return !r.HasRequeue(), r.currResult.reason } From a05bc381c87336ad46ccab91c54e1a54445d2374 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Wed, 29 Jun 2022 21:12:22 +0200 Subject: [PATCH 10/26] call fleet setup in Kibana --- pkg/controller/agent/fleet.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/controller/agent/fleet.go b/pkg/controller/agent/fleet.go index 0af138dad9..dde506db43 100644 --- a/pkg/controller/agent/fleet.go +++ b/pkg/controller/agent/fleet.go @@ -195,12 +195,16 @@ func (f fleetAPI) defaultAgentPolicyID(ctx context.Context) (string, error) { return policy.ID, nil } +func (f fleetAPI) setupFleet(ctx context.Context) error { + return f.request(ctx, http.MethodPost, "setup", nil, nil) +} + func maybeReconcileFleetEnrollment(params Params, result *reconciler.Results) string { if !params.Agent.Spec.KibanaRef.IsDefined() { return "" } - reachable, err := isKibanaReachable(params.Context, params.Client, params.Agent.Spec.KibanaRef.NamespacedName()) + reachable, err := isKibanaReachable(params.Context, params.Client, params.Agent.Spec.KibanaRef.WithDefaultNamespace(params.Agent.Namespace).NamespacedName()) if err != nil { result.WithError(err) return "" @@ -247,6 +251,12 @@ func reconcileEnrollmentToken( ) (string, error) { // do we have an existing token that we have rolled out previously? tokenName, exists := agent.Annotations[FleetTokenAnnotation] + if !exists { + // setup fleet to create default policies (and tokens) + if err := api.setupFleet(ctx); err != nil { + return "", err + } + } // what policy should we enroll this agent in? policyID, err := reconcilePolicyID(ctx, agent, api) if err != nil { From d655374ed5f356da264c258eaad0b31353f24b1f Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Thu, 30 Jun 2022 14:36:52 +0200 Subject: [PATCH 11/26] Use restricted user for Fleet setup when possible --- .../association/controller/agent_kibana.go | 21 ++++++++++++++++++- pkg/controller/elasticsearch/user/roles.go | 11 ++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/pkg/controller/association/controller/agent_kibana.go b/pkg/controller/association/controller/agent_kibana.go index cff3b3c07a..8419120518 100644 --- a/pkg/controller/association/controller/agent_kibana.go +++ b/pkg/controller/association/controller/agent_kibana.go @@ -5,6 +5,7 @@ package controller import ( + pkgerrors "github.com/pkg/errors" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -14,6 +15,8 @@ import ( kbv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/kibana/v1" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/association" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/operator" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/version" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/user" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/kibana" "github.com/elastic/cloud-on-k8s/v2/pkg/utils/rbac" ) @@ -43,7 +46,23 @@ func AddAgentKibana(mgr manager.Manager, accessReviewer rbac.AccessReviewer, par ElasticsearchRef: getElasticsearchFromKibana, UserSecretSuffix: "agent-kb-user", ESUserRole: func(associated commonv1.Associated) (string, error) { - return "superuser", nil + agent, ok := associated.(*agentv1alpha1.Agent) + if !ok { + return "", pkgerrors.Errorf( + "Agent expected, got %s/%s", + associated.GetObjectKind().GroupVersionKind().Group, + associated.GetObjectKind().GroupVersionKind().Kind, + ) + } + v, err := version.Parse(agent.Spec.Version) + if err != nil { + return "", err + } + // Fleet API can only be used as a non-superuser as of 8.1.0 https://github.com/elastic/kibana/issues/108252 + if v.LT(version.MinFor(8, 1, 0)) { + return "superuser", nil + } + return user.FleetAdminUserRole, nil }, }, }) diff --git a/pkg/controller/elasticsearch/user/roles.go b/pkg/controller/elasticsearch/user/roles.go index a893dd6861..4715bc0e59 100644 --- a/pkg/controller/elasticsearch/user/roles.go +++ b/pkg/controller/elasticsearch/user/roles.go @@ -40,6 +40,8 @@ const ( // data to the monitoring Elasticsearch cluster when Stack Monitoring is enabled StackMonitoringUserRole = "eck_stack_mon_user_role" + FleetAdminUserRole = "eck_fleet_admin_user_role" + // V70 indicates version 7.0 V70 = "v70" @@ -136,6 +138,15 @@ var ( }, }, }, + FleetAdminUserRole: esclient.Role{ + Applications: []esclient.ApplicationRole{ + { + Application: "kibana-.kibana", + Resources: []string{"*"}, + Privileges: []string{"feature_fleet.all", "feature_fleetv2.all"}, + }, + }, + }, } ) From 8a59b412bd1b40c6acec04148564ca1fb45899d0 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Thu, 30 Jun 2022 14:51:50 +0200 Subject: [PATCH 12/26] add event on requeue --- pkg/controller/agent/driver.go | 7 +++++++ pkg/controller/agent/fleet.go | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/controller/agent/driver.go b/pkg/controller/agent/driver.go index f5369c8dc8..ed6213afd6 100644 --- a/pkg/controller/agent/driver.go +++ b/pkg/controller/agent/driver.go @@ -21,6 +21,7 @@ import ( commonassociation "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/association" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/certificates" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/defaults" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/events" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/operator" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/reconciler" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/tracing" @@ -131,6 +132,12 @@ func internalReconcile(params Params) (*reconciler.Results, agentv1alpha1.AgentS fleetToken := maybeReconcileFleetEnrollment(params, results) if results.HasRequeue() || results.HasError() { + if results.HasRequeue() { + // we requeue if Kibana is unavailable: surface this condition to the user + message := "Delaying deployment of Elastic Agent in Fleet Mode as Kibana is not not available yet" + params.Logger().Info(message) + params.EventRecorder.Event(¶ms.Agent, corev1.EventTypeWarning, events.EventReasonDelayed, message) + } return results, params.Status } diff --git a/pkg/controller/agent/fleet.go b/pkg/controller/agent/fleet.go index dde506db43..908b9661c1 100644 --- a/pkg/controller/agent/fleet.go +++ b/pkg/controller/agent/fleet.go @@ -225,7 +225,7 @@ func maybeReconcileFleetEnrollment(params Params, result *reconciler.Results) st newFleetAPI( params.OperatorParams.Dialer, kbConnectionSettings, - params.Logger().WithValues("namespace", params.Agent.Namespace, "agent_name", params.Agent.Name)), + params.Logger()), ) result.WithError(err) return token From 0fa5f025265911a25bbded62848da145f518641d Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Sun, 3 Jul 2022 13:57:45 +0200 Subject: [PATCH 13/26] Unit tests --- pkg/controller/agent/fleet.go | 3 + pkg/controller/agent/fleet_test.go | 291 ++++++++++++++++++ pkg/controller/common/http/http_client.go | 6 +- .../elasticsearch/user/reconcile_test.go | 2 +- 4 files changed, 300 insertions(+), 2 deletions(-) create mode 100644 pkg/controller/agent/fleet_test.go diff --git a/pkg/controller/agent/fleet.go b/pkg/controller/agent/fleet.go index 908b9661c1..16e2b66f72 100644 --- a/pkg/controller/agent/fleet.go +++ b/pkg/controller/agent/fleet.go @@ -285,6 +285,9 @@ CREATE: } // this potentially creates conflicts we could introduce reconciler state similar to the ES controller and handle it // on the top level but we would then potentially create redundant enrollment tokens in the Fleet API + if agent.Annotations == nil { + agent.Annotations = map[string]string{} + } agent.Annotations[FleetTokenAnnotation] = key.ID err = client.Update(ctx, &agent) if err != nil { diff --git a/pkg/controller/agent/fleet_test.go b/pkg/controller/agent/fleet_test.go new file mode 100644 index 0000000000..e5068247bb --- /dev/null +++ b/pkg/controller/agent/fleet_test.go @@ -0,0 +1,291 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. + +package agent + +import ( + "context" + "io/ioutil" + "net/http" + "strings" + "testing" + + "github.com/pkg/errors" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/elastic/cloud-on-k8s/v2/pkg/apis/agent/v1alpha1" + "github.com/elastic/cloud-on-k8s/v2/pkg/utils/k8s" + ulog "github.com/elastic/cloud-on-k8s/v2/pkg/utils/log" +) + +var ( + enrollmentKeySample = `{"item":{"id":"some-token-id","active":true,"api_key_id":"2y2otIEB7e2EvmgdFYCC","api_key":"some-token","name":"fe2c49b4-a94a-41ba-b6ed-a64c536874e9","policy_id":"a-policy-id","created_at":"2022-06-30T12:48:44.378Z"}}` + fleetServerKeySample = `{"item":{"id":"some-token-id","active":true,"api_key_id":"2y2otIEB7e2EvmgdFYCC","api_key":"fleet-token","name":"fe2c49b4-a94a-41ba-b6ed-a64c536874e9","policy_id":"fleet-policy-id","created_at":"2022-06-30T12:48:44.378Z"}}` + inactiveEnrollmentKeySample = `{"item":{"id":"some-token-id","active":false,"api_key_id":"2y2otIEB7e2EvmgdFYCC","api_key":"some-token","name":"fe2c49b4-a94a-41ba-b6ed-a64c536874e9","policy_id":"a-policy-id","created_at":"2022-06-30T12:48:44.378Z"}}` + agentPoliciesSample = `{"items":[{"id":"fleet-policy-id","namespace":"default","monitoring_enabled":["logs","metrics"],"name":"Default Fleet Server policy","description":"Default Fleet Server agent policy created by Kibana","is_default":false,"is_default_fleet_server":true,"is_preconfigured":true,"status":"active","is_managed":false,"revision":1,"updated_at":"2022-06-30T12:48:35.349Z","updated_by":"system","package_policies":["dcc6e5b3-ea49-4b96-ae39-a1a3b74d849b"],"agents":1},{"id":"f217f7e0-f872-11ec-8bc1-17034ca5bd9f","namespace":"default","monitoring_enabled":["logs","metrics"],"name":"Default policy","description":"Default agent policy created by Kibana","is_default":true,"is_preconfigured":true,"status":"active","is_managed":false,"revision":1,"updated_at":"2022-06-30T12:48:33.323Z","updated_by":"system","package_policies":["8a7a3e75-47fb-4205-8c1c-db69a2c70458"],"agents":3}],"total":2,"page":1,"perPage":20}` +) + +func Test_reconcileEnrollmentToken(t *testing.T) { + + type args struct { + agent v1alpha1.Agent + client *k8s.Client + api *mockFleetAPI + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "Agent annotated and fixed policy", + args: args{ + agent: v1alpha1.Agent{ + ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ + FleetTokenAnnotation: "some-token-id", + }}, + Spec: v1alpha1.AgentSpec{ + PolicyID: "a-policy-id", + }, + }, + api: mockFleetResponses(map[request]response{ + {"GET", "/api/fleet/entrollment_api_keys/some-token-id"}: {code: 200, body: enrollmentKeySample}, + }), + }, + want: "some-token", + wantErr: false, + }, + { + name: "Agent annotated but default policy", + args: args{ + agent: v1alpha1.Agent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "agent", + Namespace: "ns", + Annotations: map[string]string{ + FleetTokenAnnotation: "some-token-id", + }}, + }, + api: mockFleetResponses(map[request]response{ + // get all policies + {"GET", "/api/fleet/agent_policies"}: {code: 200, body: agentPoliciesSample}, + // check annotated api key + {"GET", "/api/fleet/enrollment_api_keys/some-token-id"}: {code: 200, body: enrollmentKeySample}, + // new token because existing key not valid for policy + {"POST", "/api/fleet/enrollment_api_keys"}: {code: 200, body: enrollmentKeySample}, + }), + }, + want: "some-token", + wantErr: false, + }, + { + name: "Agent annotated but token does not exist", + args: args{ + agent: v1alpha1.Agent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "agent", + Namespace: "ns", + Annotations: map[string]string{ + FleetTokenAnnotation: "invalid-token-id", + }}, + Spec: v1alpha1.AgentSpec{}, + }, + api: mockFleetResponses(map[request]response{ + {"GET", "/api/fleet/agent_policies"}: {code: 200, body: agentPoliciesSample}, + {"GET", "/api/fleet/enrollment_api_keys/invalid-token-id"}: {code: 404}, + {"POST", "/api/fleet/enrollment_api_keys"}: {code: 200, body: enrollmentKeySample}, + }), + }, + want: "some-token", + wantErr: false, + }, + { + name: "Agent annotated but token is invalid", + args: args{ + agent: v1alpha1.Agent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "agent", + Namespace: "ns", + Annotations: map[string]string{ + FleetTokenAnnotation: "invalid-token-id", + }}, + Spec: v1alpha1.AgentSpec{}, + }, + api: mockFleetResponses(map[request]response{ + {"GET", "/api/fleet/agent_policies"}: {code: 200, body: agentPoliciesSample}, + {"GET", "/api/fleet/enrollment_api_keys/invalid-token-id"}: {code: 200, body: inactiveEnrollmentKeySample}, + {"POST", "/api/fleet/enrollment_api_keys"}: {code: 200, body: enrollmentKeySample}, + }), + }, + want: "some-token", + wantErr: false, + }, + { + name: "Agent not annotated yet", + args: args{ + agent: v1alpha1.Agent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "agent", + Namespace: "ns", + }, + }, + api: mockFleetResponses(map[request]response{ + {"POST", "/api/fleet/setup"}: {code: 200}, + {"GET", "/api/fleet/agent_policies"}: {code: 200, body: agentPoliciesSample}, + {"POST", "/api/fleet/enrollment_api_keys"}: {code: 200, body: enrollmentKeySample}, + }), + }, + want: "some-token", + wantErr: false, + }, + { + name: "Error in Fleet API", + args: args{ + agent: v1alpha1.Agent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "agent", + Namespace: "ns", + Annotations: map[string]string{ + FleetTokenAnnotation: "some-token-id", + }, + }, + }, + api: mockFleetResponses(map[request]response{ + {"GET", "/api/fleet/agent_policies"}: {code: 500}, + }), + }, + want: "", + wantErr: true, + }, + { + name: "Fleet Server policy and key", + args: args{ + agent: v1alpha1.Agent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "agent", + Namespace: "ns", + Annotations: map[string]string{ + FleetTokenAnnotation: "some-token-id", + }}, + Spec: v1alpha1.AgentSpec{ + FleetServerEnabled: true, + }}, + api: mockFleetResponses(map[request]response{ + // get all policies + {"GET", "/api/fleet/agent_policies"}: {code: 200, body: agentPoliciesSample}, + // check annotated api key, should be valid + {"GET", "/api/fleet/enrollment_api_keys/some-token-id"}: {code: 200, body: fleetServerKeySample}, + }), + }, + want: "fleet-token", + wantErr: false, + }, + { + name: "Error in Kubernetes API", + args: args{ + agent: v1alpha1.Agent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "agent", + Namespace: "ns", + }, + Spec: v1alpha1.AgentSpec{ + FleetServerEnabled: true, + }}, + client: func() *k8s.Client { + client := k8s.NewFailingClient(errors.New("boom")) + return &client + }(), + api: mockFleetResponses(map[request]response{ + {"POST", "/api/fleet/setup"}: {code: 200}, + // get all policies + {"GET", "/api/fleet/agent_policies"}: {code: 200, body: agentPoliciesSample}, + // check annotated api key, should be valid + {"POST", "/api/fleet/enrollment_api_keys"}: {code: 200, body: fleetServerKeySample}, + // delete token because annotation update failed and we want to avoid dangling keys + {"DELETE", "/api/fleet/enrollment_api_keys/some-token-id"}: {code: 200}, + }), + }, + want: "", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var client = k8s.NewFakeClient(&tt.args.agent) + if tt.args.client != nil { + client = *tt.args.client + } + got, err := reconcileEnrollmentToken(context.Background(), tt.args.agent, client, tt.args.api.fleetAPI) + require.Empty(t, tt.args.api.missingRequests()) + if (err != nil) != tt.wantErr { + t.Errorf("reconcileEnrollmentToken() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("reconcileEnrollmentToken() got = %v, want %v", got, tt.want) + } + }) + } +} + +type RoundTripFunc func(req *http.Request) *http.Response + +func (f RoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req), nil +} + +type request struct { + method string + path string +} + +type response struct { + code int + body string +} + +type mockFleetAPI struct { + fleetAPI + requests map[request]response + callLog map[request]int +} + +func (m *mockFleetAPI) missingRequests() []request { + var missing []request + for req, _ := range m.requests { + if _, ok := m.callLog[req]; !ok { + missing = append(missing, req) + } + } + return missing +} + +func mockFleetResponses(rs map[request]response) *mockFleetAPI { + callLog := map[request]int{} + fn := func(req *http.Request) *http.Response { + r := request{method: req.Method, path: req.URL.Path} + response, exists := rs[r] + if exists { + callLog[r] = callLog[r] + 1 + } + return &http.Response{ + StatusCode: response.code, + Body: ioutil.NopCloser(strings.NewReader(response.body)), + Request: req, + } + } + return &mockFleetAPI{ + fleetAPI: fleetAPI{ + client: &http.Client{ + Transport: RoundTripFunc(fn), + }, + log: ulog.Log, + }, + callLog: callLog, + requests: rs, + } +} diff --git a/pkg/controller/common/http/http_client.go b/pkg/controller/common/http/http_client.go index 86fec2d911..cba5b7db60 100644 --- a/pkg/controller/common/http/http_client.go +++ b/pkg/controller/common/http/http_client.go @@ -78,9 +78,13 @@ type APIError struct { // MaybeAPIError creates an APIError from a http.Response if the status code is not 2xx. func MaybeAPIError(resp *http.Response) *APIError { if resp.StatusCode < 200 || resp.StatusCode > 299 { + url := "unknown url" + if resp.Request != nil { + url = resp.Request.URL.Redacted() + } return &APIError{ StatusCode: resp.StatusCode, - msg: fmt.Sprintf("failed to request %s, status is %d)", resp.Request.URL.Redacted(), resp.StatusCode), + msg: fmt.Sprintf("failed to request %s, status is %d)", url, resp.StatusCode), } } return nil diff --git a/pkg/controller/elasticsearch/user/reconcile_test.go b/pkg/controller/elasticsearch/user/reconcile_test.go index 9f2cd7e8a7..4b27cacce5 100644 --- a/pkg/controller/elasticsearch/user/reconcile_test.go +++ b/pkg/controller/elasticsearch/user/reconcile_test.go @@ -85,6 +85,6 @@ func Test_aggregateRoles(t *testing.T) { c := k8s.NewFakeClient(sampleUserProvidedRolesSecret...) roles, err := aggregateRoles(c, sampleEsWithAuth, initDynamicWatches(), record.NewFakeRecorder(10)) require.NoError(t, err) - require.Len(t, roles, 51) + require.Len(t, roles, 52) require.Contains(t, roles, ProbeUserRole, "role1", "role2") } From 819aece6bc117565a0f932b436b9369768355787 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Sun, 3 Jul 2022 14:18:58 +0200 Subject: [PATCH 14/26] Do not mount Kibana certs --- pkg/controller/agent/pod.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/controller/agent/pod.go b/pkg/controller/agent/pod.go index 87d7439229..8e18ea9ce5 100644 --- a/pkg/controller/agent/pod.go +++ b/pkg/controller/agent/pod.go @@ -363,8 +363,13 @@ func writeEsAssocToConfigHash(params Params, esAssociation commonv1.Association, } func getVolumesFromAssociations(associations []commonv1.Association) ([]volume.VolumeLike, error) { - var vols []volume.VolumeLike //nolint:prealloc - for i, assoc := range associations { // TODO filter Kibana out + var vols []volume.VolumeLike //nolint:prealloc + for i, assoc := range associations { + // the Kibana association is only used by the operator to interact with the Kibana Fleet API but + // not by the individual Elastic Agent Pods. There is therefore no need to mount the Kibana certificate secret. + if assoc.AssociationType() == commonv1.KibanaAssociationType { + continue + } assocConf, err := assoc.AssociationConf() if err != nil { return nil, err From 5ef6c1eb67eb702acbf33fea6cb0f5507803e668 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Sun, 3 Jul 2022 14:30:18 +0200 Subject: [PATCH 15/26] lint --- pkg/controller/agent/fleet_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controller/agent/fleet_test.go b/pkg/controller/agent/fleet_test.go index e5068247bb..899c9f22c4 100644 --- a/pkg/controller/agent/fleet_test.go +++ b/pkg/controller/agent/fleet_test.go @@ -256,7 +256,7 @@ type mockFleetAPI struct { func (m *mockFleetAPI) missingRequests() []request { var missing []request - for req, _ := range m.requests { + for req := range m.requests { if _, ok := m.callLog[req]; !ok { missing = append(missing, req) } @@ -270,7 +270,7 @@ func mockFleetResponses(rs map[request]response) *mockFleetAPI { r := request{method: req.Method, path: req.URL.Path} response, exists := rs[r] if exists { - callLog[r] = callLog[r] + 1 + callLog[r]++ } return &http.Response{ StatusCode: response.code, From 871c78cc8e3e75d389f05de761cab6374a3d1c8b Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Sun, 3 Jul 2022 14:33:26 +0200 Subject: [PATCH 16/26] fix unit tests --- pkg/controller/agent/fleet_test.go | 2 +- pkg/controller/agent/pod_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controller/agent/fleet_test.go b/pkg/controller/agent/fleet_test.go index 899c9f22c4..7be900e0e4 100644 --- a/pkg/controller/agent/fleet_test.go +++ b/pkg/controller/agent/fleet_test.go @@ -53,7 +53,7 @@ func Test_reconcileEnrollmentToken(t *testing.T) { }, }, api: mockFleetResponses(map[request]response{ - {"GET", "/api/fleet/entrollment_api_keys/some-token-id"}: {code: 200, body: enrollmentKeySample}, + {"GET", "/api/fleet/enrollment_api_keys/some-token-id"}: {code: 200, body: enrollmentKeySample}, }), }, want: "some-token", diff --git a/pkg/controller/agent/pod_test.go b/pkg/controller/agent/pod_test.go index ba9b8d9bcb..dacfda8cf4 100644 --- a/pkg/controller/agent/pod_test.go +++ b/pkg/controller/agent/pod_test.go @@ -392,7 +392,7 @@ func Test_getVolumesFromAssociations(t *testing.T) { CASecretName: "fleet-agent-http-certs-public", }) }, - wantAssociationsLength: 2, + wantAssociationsLength: 1, }, { name: "fleet mode enabled, kb no tls ref, fleet ref", From dcf6e87513af540bdaeae1d8ce7b57f342d27592 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Sun, 3 Jul 2022 14:38:08 +0200 Subject: [PATCH 17/26] improve unit test --- pkg/controller/agent/pod_test.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/controller/agent/pod_test.go b/pkg/controller/agent/pod_test.go index dacfda8cf4..42be434acb 100644 --- a/pkg/controller/agent/pod_test.go +++ b/pkg/controller/agent/pod_test.go @@ -378,8 +378,11 @@ func Test_getVolumesFromAssociations(t *testing.T) { params: Params{ Agent: agentv1alpha1.Agent{ Spec: agentv1alpha1.AgentSpec{ - Mode: agentv1alpha1.AgentFleetMode, - KibanaRef: commonv1.ObjectSelector{Name: "kibana"}, + Mode: agentv1alpha1.AgentFleetMode, + KibanaRef: commonv1.ObjectSelector{Name: "kibana"}, + ElasticsearchRefs: []agentv1alpha1.Output{ + {ObjectSelector: commonv1.ObjectSelector{Name: "elasticsearch"}, OutputName: "default"}, + }, FleetServerRef: commonv1.ObjectSelector{Name: "fleet"}, }, }, @@ -391,8 +394,11 @@ func Test_getVolumesFromAssociations(t *testing.T) { assocs[1].SetAssociationConf(&commonv1.AssociationConf{ CASecretName: "fleet-agent-http-certs-public", }) + assocs[2].SetAssociationConf(&commonv1.AssociationConf{ + CASecretName: "elasticsearch-es-ca", + }) }, - wantAssociationsLength: 1, + wantAssociationsLength: 2, }, { name: "fleet mode enabled, kb no tls ref, fleet ref", From 3f82ccaa4d07be0261268ea8fa3eb002c1abf263 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Sun, 3 Jul 2022 19:09:37 +0200 Subject: [PATCH 18/26] fix license header weirdness --- pkg/controller/agent/fleet_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/controller/agent/fleet_test.go b/pkg/controller/agent/fleet_test.go index 7be900e0e4..37a263ce6e 100644 --- a/pkg/controller/agent/fleet_test.go +++ b/pkg/controller/agent/fleet_test.go @@ -1,7 +1,6 @@ // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License -// 2.0; you may not use this file except in compliance with the Elastic License -// 2.0. +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. package agent From 7de3bc38ec4085f59ed19117ba6e25deab2ba6af Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Sun, 3 Jul 2022 20:38:25 +0200 Subject: [PATCH 19/26] reuse getPodConnectionSettings --- pkg/controller/agent/config.go | 40 ++---- pkg/controller/agent/config_test.go | 177 ++++++++++++++++++++++----- pkg/controller/agent/pod.go | 2 +- pkg/controller/agent/testdata/ca.crt | 19 +++ 4 files changed, 176 insertions(+), 62 deletions(-) create mode 100644 pkg/controller/agent/testdata/ca.crt diff --git a/pkg/controller/agent/config.go b/pkg/controller/agent/config.go index f3d0435090..e03c03cd48 100644 --- a/pkg/controller/agent/config.go +++ b/pkg/controller/agent/config.go @@ -146,29 +146,32 @@ func getUserConfig(params Params) (*settings.CanonicalConfig, error) { return common.ParseConfigRef(params, ¶ms.Agent, params.Agent.Spec.ConfigRef, ConfigFileName) } +// extractPodConnectionSettings extracts connections settings to be used inside an Elastic Agent Pod. That is without +// certificates which are mounted directly into the Pod, instead the connection settings contain a path which points to +// the future location of the certificates in the Pod. func extractPodConnectionSettings( agent agentv1alpha1.Agent, client k8s.Client, associationType commonv1.AssociationType, -) (connectionSettings, error) { +) (connectionSettings, *commonv1.AssociationConf, error) { assoc, err := association.SingleAssociationOfType(agent.GetAssociations(), associationType) if err != nil { - return connectionSettings{}, err + return connectionSettings{}, nil, err } if assoc == nil { errTemplate := "association of type %s not found in %d associations" - return connectionSettings{}, fmt.Errorf(errTemplate, associationType, len(agent.GetAssociations())) + return connectionSettings{}, nil, fmt.Errorf(errTemplate, associationType, len(agent.GetAssociations())) } credentials, err := association.ElasticsearchAuthSettings(client, assoc) if err != nil { - return connectionSettings{}, err + return connectionSettings{}, nil, err } assocConf, err := assoc.AssociationConf() if err != nil { - return connectionSettings{}, err + return connectionSettings{}, nil, err } ca := "" @@ -180,38 +183,21 @@ func extractPodConnectionSettings( host: assocConf.GetURL(), caFileName: ca, credentials: credentials, - }, err + version: assocConf.Version, + }, assocConf, err } +// extractClientConnectionSettings same as extractPodConnectionSettings but for use inside the operator or any other +// client that needs direct access to the relevant CA certificates of the associated object (if TLS is configured) func extractClientConnectionSettings( agent agentv1alpha1.Agent, client k8s.Client, associationType commonv1.AssociationType, ) (connectionSettings, error) { - assoc, err := association.SingleAssociationOfType(agent.GetAssociations(), associationType) - if err != nil { - return connectionSettings{}, err - } - - if assoc == nil { - errTemplate := "association of type %s not found in %d associations" - return connectionSettings{}, fmt.Errorf(errTemplate, associationType, len(agent.GetAssociations())) - } - - credentials, err := association.ElasticsearchAuthSettings(client, assoc) - if err != nil { - return connectionSettings{}, err - } - - assocConf, err := assoc.AssociationConf() + settings, assocConf, err := extractPodConnectionSettings(agent, client, associationType) if err != nil { return connectionSettings{}, err } - settings := connectionSettings{ - host: assocConf.GetURL(), - credentials: credentials, - version: assocConf.Version, - } if !assocConf.GetCACertProvided() { return settings, nil } diff --git a/pkg/controller/agent/config_test.go b/pkg/controller/agent/config_test.go index 7722c424aa..93fb77c135 100644 --- a/pkg/controller/agent/config_test.go +++ b/pkg/controller/agent/config_test.go @@ -4,6 +4,9 @@ package agent import ( + "io/ioutil" + "path/filepath" + "reflect" "testing" "github.com/stretchr/testify/require" @@ -13,45 +16,54 @@ import ( agentv1alpha1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/agent/v1alpha1" commonv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/common/v1" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/association" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/certificates" "github.com/elastic/cloud-on-k8s/v2/pkg/utils/k8s" ) -func TestExtractConnectionSettings(t *testing.T) { - agentWithoutCa := agentv1alpha1.Agent{ - ObjectMeta: metav1.ObjectMeta{ - Name: "agent", - Namespace: "ns", - }, - Spec: agentv1alpha1.AgentSpec{ - KibanaRef: commonv1.ObjectSelector{ - Name: "kibana", +var ( + agentFixture = func() *agentv1alpha1.Agent { + return &agentv1alpha1.Agent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "agent", Namespace: "ns", }, - }, + Spec: agentv1alpha1.AgentSpec{ + KibanaRef: commonv1.ObjectSelector{ + Name: "kibana", + Namespace: "ns", + }, + }, + } } - - agentWithCa := agentWithoutCa - - assocWithoutCa := agentv1alpha1.AgentKibanaAssociation{ - Agent: &agentWithoutCa, + assocWithCAFixture = func() agentv1alpha1.AgentKibanaAssociation { + a := agentv1alpha1.AgentKibanaAssociation{ + Agent: agentFixture(), + } + a.SetAssociationConf(&commonv1.AssociationConf{ + AuthSecretName: "secret-name", + AuthSecretKey: "user", + URL: "url", + CACertProvided: true, + CASecretName: "ca-secret-name", + Version: "8.3.0", + }) + return a } - assocWithoutCa.SetAssociationConf(&commonv1.AssociationConf{ - AuthSecretName: "secret-name", - AuthSecretKey: "user", - URL: "url", - }) - - assocWithCa := agentv1alpha1.AgentKibanaAssociation{ - Agent: &agentWithCa, + assocWithoutCAFixture = func() agentv1alpha1.AgentKibanaAssociation { + a := agentv1alpha1.AgentKibanaAssociation{ + Agent: agentFixture(), + } + a.SetAssociationConf(&commonv1.AssociationConf{ + AuthSecretName: "secret-name", + AuthSecretKey: "user", + URL: "url", + Version: "8.3.0", + }) + return a } - assocWithCa.SetAssociationConf(&commonv1.AssociationConf{ - AuthSecretName: "secret-name", - AuthSecretKey: "user", - URL: "url", - CACertProvided: true, - CASecretName: "ca-secret-name", - }) +) +func TestExtractConnectionSettings(t *testing.T) { for _, tt := range []struct { name string agent agentv1alpha1.Agent @@ -69,14 +81,14 @@ func TestExtractConnectionSettings(t *testing.T) { }, { name: "no auth secret", - agent: *assocWithoutCa.Agent, + agent: *assocWithoutCAFixture().Agent, client: k8s.NewFakeClient(), assocType: commonv1.KibanaAssociationType, wantErr: true, }, { name: "happy path without ca", - agent: *assocWithoutCa.Agent, + agent: *assocWithoutCAFixture().Agent, client: k8s.NewFakeClient(&corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "secret-name", @@ -93,12 +105,13 @@ func TestExtractConnectionSettings(t *testing.T) { Username: "user", Password: "password", }, + version: "8.3.0", }, wantErr: false, }, { name: "happy path with ca", - agent: *assocWithCa.Agent, + agent: *assocWithCAFixture().Agent, client: k8s.NewFakeClient(&corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "secret-name", @@ -116,15 +129,111 @@ func TestExtractConnectionSettings(t *testing.T) { Username: "user", Password: "password", }, + version: "8.3.0", }, wantErr: false, }, } { t.Run(tt.name, func(t *testing.T) { - gotConnectionSettings, gotErr := extractPodConnectionSettings(tt.agent, tt.client, tt.assocType) + gotConnectionSettings, _, gotErr := extractPodConnectionSettings(tt.agent, tt.client, tt.assocType) require.Equal(t, tt.wantConnectionSettings, gotConnectionSettings) require.Equal(t, tt.wantErr, gotErr != nil) }) } } + +func Test_extractClientConnectionSettings(t *testing.T) { + // assoc secret all other cases are tested in TestExtractConnectionSettings + assocSecretFixture := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret-name", + Namespace: "ns", + }, + Data: map[string][]byte{ + "user": []byte("password"), + }, + } + + // setup cert fixtures + bytes, err := ioutil.ReadFile(filepath.Join("testdata", "ca.crt")) + require.NoError(t, err) + certs, err := certificates.ParsePEMCerts(bytes) + require.NoError(t, err) + caSecretFixture := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ca-secret-name", + Namespace: "ns", + }, + Data: map[string][]byte{ + "ca.crt": bytes, + }, + } + + type args struct { + agent agentv1alpha1.Agent + client k8s.Client + } + tests := []struct { + name string + args args + want connectionSettings + wantErr bool + }{ + { + name: "agent/kibana with CA", + args: args{ + agent: *assocWithCAFixture().Agent, + client: k8s.NewFakeClient(assocSecretFixture, caSecretFixture), + }, + want: connectionSettings{ + host: "url", + caFileName: "/mnt/elastic-internal/kibana-association/ns/kibana/certs/ca.crt", + version: "8.3.0", + credentials: association.Credentials{ + Username: "user", + Password: "password", + }, + caCerts: certs, + }, + wantErr: false, + }, + { + name: "agent/kibana without CA", + args: args{ + agent: *assocWithoutCAFixture().Agent, + client: k8s.NewFakeClient(assocSecretFixture), + }, + want: connectionSettings{ + host: "url", + credentials: association.Credentials{ + Username: "user", + Password: "password", + }, + version: "8.3.0", + }, + wantErr: false, + }, + { + name: "missing certificates secret", + args: args{ + agent: *assocWithCAFixture().Agent, + client: k8s.NewFakeClient(assocSecretFixture), + }, + want: connectionSettings{}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := extractClientConnectionSettings(tt.args.agent, tt.args.client, "kibana") + if (err != nil) != tt.wantErr { + t.Errorf("extractClientConnectionSettings() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("extractClientConnectionSettings() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/controller/agent/pod.go b/pkg/controller/agent/pod.go index 8e18ea9ce5..37b55c3ab6 100644 --- a/pkg/controller/agent/pod.go +++ b/pkg/controller/agent/pod.go @@ -514,7 +514,7 @@ func getFleetSetupFleetServerEnvVars(agent agentv1alpha1.Agent, client k8s.Clien esExpected := len(agent.Spec.ElasticsearchRefs) > 0 && agent.Spec.ElasticsearchRefs[0].IsDefined() if esExpected { - esConnectionSettings, err := extractPodConnectionSettings(agent, client, commonv1.ElasticsearchAssociationType) + esConnectionSettings, _, err := extractPodConnectionSettings(agent, client, commonv1.ElasticsearchAssociationType) if err != nil { return nil, err } diff --git a/pkg/controller/agent/testdata/ca.crt b/pkg/controller/agent/testdata/ca.crt new file mode 100644 index 0000000000..da2bce043f --- /dev/null +++ b/pkg/controller/agent/testdata/ca.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIC/zCCAeegAwIBAgIJAIVZ8xw3LMNkMA0GCSqGSIb3DQEBCwUAMBYxFDASBgNV +BAMMC21vcmVsbG8ub3ZoMB4XDTE5MDgwOTA5MzQwMFoXDTI5MDgwNjA5MzQwMFow +FjEUMBIGA1UEAwwLbW9yZWxsby5vdmgwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw +ggEKAoIBAQCoM2HYyuTTlu41SlgVO0Hdx7eUQevGSKO6pjPjN49/KKY1z/3DoKzr +seWaGOjiWUAqx/GHX8AsR9ToVoKGBbSNeDxT33pt3I9aCnnOPTt3yDIOlr4ZWnKq +NnNHwfydsMBfBAYgdU/L506KuNHJQ18Zey5+A0roTWyHUT48mQBsjetXg77RfDMB +MYVOWETfl70GKAaAlVGZfJHCkfBzYnPcEjqtcuU/7d27WZrSMhXifzHAEmm0KPER +EWdo4UHTK23wLY6dvkp2O5i0bKHv+PuLpqYrm7R7SWGhhwD651n5S5W20FHDow+d +js0yW2gqYsZZN6S1uAsJ8rdYAEPhK9J9AgMBAAGjUDBOMB0GA1UdDgQWBBQ6Lsen +0HbE+7M6iV9r8n5rZrbl4jAfBgNVHSMEGDAWgBQ6Lsen0HbE+7M6iV9r8n5rZrbl +4jAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAgrLJnK4s/OVnh8CRk +GmikP+ZxhDs4k1nlr7+rTYkU0huoHK8p802w4zd74szYsHpo8kON/zSmFD7JpU4L +o2kseENqMsgrCPhF3+TDwf/Li43pbK162iAq8ZEpYnSXbQsRyP+Tz0lzoEoli6o7 +6KVn4VNookLMyhGIAOmhfbNm0jG+B2zz+bvoTAe9CiDfvq1k0fnuKFzRtRsj09NJ +FNMhSc02N4EDrGpL5CYmEXjPZS3lUsoYPwbYlmUt3Bzuf5hI0mDHCt3BYKH1vFI4 +W8/h9wwGn/yytsH21dkj41KEQK6N65gT9i0fBBiubuS2H1SVMMJ/J7PUqol278Ar +zGpS +-----END CERTIFICATE----- From e8c850e1d35a9244af0fe20d2c38b1e636df6ce7 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Sun, 3 Jul 2022 20:54:11 +0200 Subject: [PATCH 20/26] lint --- pkg/controller/agent/fleet_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/controller/agent/fleet_test.go b/pkg/controller/agent/fleet_test.go index 37a263ce6e..81131c590b 100644 --- a/pkg/controller/agent/fleet_test.go +++ b/pkg/controller/agent/fleet_test.go @@ -28,7 +28,6 @@ var ( ) func Test_reconcileEnrollmentToken(t *testing.T) { - type args struct { agent v1alpha1.Agent client *k8s.Client From 742953e6ffdca208065d5ad73ea94c8fa68f7d67 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Tue, 12 Jul 2022 21:13:51 +0200 Subject: [PATCH 21/26] review feedback --- pkg/controller/agent/config.go | 8 ++++++-- pkg/controller/agent/config_test.go | 3 ++- pkg/controller/agent/driver.go | 2 +- pkg/controller/agent/fleet.go | 5 ++--- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/pkg/controller/agent/config.go b/pkg/controller/agent/config.go index e03c03cd48..a8c38ee160 100644 --- a/pkg/controller/agent/config.go +++ b/pkg/controller/agent/config.go @@ -190,6 +190,7 @@ func extractPodConnectionSettings( // extractClientConnectionSettings same as extractPodConnectionSettings but for use inside the operator or any other // client that needs direct access to the relevant CA certificates of the associated object (if TLS is configured) func extractClientConnectionSettings( + ctx context.Context, agent agentv1alpha1.Agent, client k8s.Client, associationType commonv1.AssociationType, @@ -202,10 +203,13 @@ func extractClientConnectionSettings( return settings, nil } var caSecret corev1.Secret - if err := client.Get(context.Background(), types.NamespacedName{Name: assocConf.GetCASecretName(), Namespace: agent.Namespace}, &caSecret); err != nil { + if err := client.Get(ctx, types.NamespacedName{Name: assocConf.GetCASecretName(), Namespace: agent.Namespace}, &caSecret); err != nil { return connectionSettings{}, err } - bytes := caSecret.Data[CAFileName] + bytes, ok := caSecret.Data[CAFileName] + if !ok { + return connectionSettings{}, fmt.Errorf("no %s in %s", CAFileName, k8s.ExtractNamespacedName(&caSecret)) + } certs, err := certificates.ParsePEMCerts(bytes) if err != nil { return connectionSettings{}, err diff --git a/pkg/controller/agent/config_test.go b/pkg/controller/agent/config_test.go index 93fb77c135..d30a486d03 100644 --- a/pkg/controller/agent/config_test.go +++ b/pkg/controller/agent/config_test.go @@ -4,6 +4,7 @@ package agent import ( + "context" "io/ioutil" "path/filepath" "reflect" @@ -226,7 +227,7 @@ func Test_extractClientConnectionSettings(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := extractClientConnectionSettings(tt.args.agent, tt.args.client, "kibana") + got, err := extractClientConnectionSettings(context.Background(), tt.args.agent, tt.args.client, "kibana") if (err != nil) != tt.wantErr { t.Errorf("extractClientConnectionSettings() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/pkg/controller/agent/driver.go b/pkg/controller/agent/driver.go index ed6213afd6..4fc4c3deb3 100644 --- a/pkg/controller/agent/driver.go +++ b/pkg/controller/agent/driver.go @@ -134,7 +134,7 @@ func internalReconcile(params Params) (*reconciler.Results, agentv1alpha1.AgentS if results.HasRequeue() || results.HasError() { if results.HasRequeue() { // we requeue if Kibana is unavailable: surface this condition to the user - message := "Delaying deployment of Elastic Agent in Fleet Mode as Kibana is not not available yet" + message := "Delaying deployment of Elastic Agent in Fleet Mode as Kibana is not available yet" params.Logger().Info(message) params.EventRecorder.Event(¶ms.Agent, corev1.EventTypeWarning, events.EventReasonDelayed, message) } diff --git a/pkg/controller/agent/fleet.go b/pkg/controller/agent/fleet.go index 16e2b66f72..57dbba8171 100644 --- a/pkg/controller/agent/fleet.go +++ b/pkg/controller/agent/fleet.go @@ -82,8 +82,7 @@ func (f fleetAPI) request( ctx context.Context, method string, pathWithQuery string, - requestObj, - responseObj interface{}) error { + requestObj, responseObj interface{}) error { var body io.Reader = http.NoBody if requestObj != nil { outData, err := json.Marshal(requestObj) @@ -214,7 +213,7 @@ func maybeReconcileFleetEnrollment(params Params, result *reconciler.Results) st return "" } - kbConnectionSettings, err := extractClientConnectionSettings(params.Agent, params.Client, commonv1.KibanaAssociationType) + kbConnectionSettings, err := extractClientConnectionSettings(params.Context, params.Agent, params.Client, commonv1.KibanaAssociationType) if err != nil { result.WithError(err) return "" From 1c78ce8d7c0116e1fcf7bd2c4b5e8a9bb77c83c7 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Wed, 13 Jul 2022 09:25:12 +0200 Subject: [PATCH 22/26] doc changes --- .../agent-fleet.asciidoc | 34 ++++++++----------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/docs/orchestrating-elastic-stack-applications/agent-fleet.asciidoc b/docs/orchestrating-elastic-stack-applications/agent-fleet.asciidoc index 9f9ed227ee..bfbbd846a4 100644 --- a/docs/orchestrating-elastic-stack-applications/agent-fleet.asciidoc +++ b/docs/orchestrating-elastic-stack-applications/agent-fleet.asciidoc @@ -312,7 +312,7 @@ spec: [id="{p}-elastic-agent-fleet-configuration-setting-referenced-resources"] === Set referenced resources -Both Fleet Server and Elastic Agent in Fleet mode can facilitate the Fleet setup. Fleet Server can set up Fleet in Kibana (which otherwise requires manual steps) and enroll itself in the default Fleet Server policy. Elastic Agent can enroll itself in the default Elastic Agent policy. To allow ECK to set this up, provide a reference to ECK-managed Kibana through `kibanaRef` configuration element. +Both Fleet Server and Elastic Agent in Fleet mode can be automatically setup with Fleet by ECK. The ECK operator can set up Fleet in Kibana (which otherwise requires manual steps) and enroll Fleet Server in the default Fleet Server policy. Elastic Agent can be automatically enrolled in the default Elastic Agent policy. To allow ECK to set this up, provide a reference to a ECK-managed Kibana through the `kibanaRef` configuration element. [source,yaml,subs="attributes,+macros"] ---- @@ -325,7 +325,7 @@ spec: name: kibana ---- -ECK can also facilitate the connection between Elastic Agents and ECK-managed Fleet Server. To allow ECK to set this up, provide a reference to Fleet Server through `fleetServerRef` configuration element. +ECK can also facilitate the connection between Elastic Agents and a ECK-managed Fleet Server. To allow ECK to set this up, provide a reference to Fleet Server through the `fleetServerRef` configuration element. [source,yaml,subs="attributes,+macros"] ---- @@ -339,7 +339,7 @@ spec: ---- -Set `elasticsearchRefs` element in your Fleet Server to point to the Elasticsearch cluster that will manage Fleet. Leave `elasticsearchRefs` empty or unset for any Elastic Agent running in Fleet mode as the Elasticsearch cluster to target will come from Kibana `xpack.fleet.agents.elasticsearch.hosts` configuration element. +Set the `elasticsearchRefs` element in your Fleet Server to point to the Elasticsearch cluster that will manage Fleet. Leave `elasticsearchRefs` empty or unset it for any Elastic Agent running in Fleet mode as the Elasticsearch cluster to target will come from Kibana's `xpack.fleet.agents.elasticsearch.hosts` configuration element. NOTE: Currently, Elastic Agent in Fleet mode supports only a single output, so only a single Elasticsearch cluster can be referenced. @@ -359,9 +359,7 @@ By default, every reference targets all instances in your Elasticsearch, Kibana [id="{p}-elastic-agent-fleet-configuration-custom-configuration"] === Customize Elastic Agent configuration -In contrast to what happens with Elastic Agent as standalone, the configuration is managed through Fleet, and it cannot be defined through `config` or `configRef` elements. - -You can only configure the setup part of the Fleet Server and Elastic Agent. You can override each of the environment variables that agents consume, as documented in link:https://www.elastic.co/guide/en/fleet/current/agent-environment-variables.html[Elastic Agent environment variables]. This allows different setups where components are deployed both in local Kubernetes cluster and externally. +In contrast to Elastic Agents in standalone mode, the configuration is managed through Fleet, and it cannot be defined through `config` or `configRef` elements. [id="{p}-elastic-agent-fleet-configuration-upgrade-specification"] === Upgrade the Elastic Agent specification @@ -470,27 +468,23 @@ To deploy Elastic Agent in clusters with the Pod Security Policy admission contr By default, ECK creates a Service for Fleet Server that Elastic Agents can connect through. You can customize it using the `http` configuration element. Check more information on how to link:k8s-services.html[make changes] to the Service and link:k8s-tls-certificates.html[customize] the TLS configuration. -[id="{p}-elastic-agent-fleet-configuration-override-default-fleet-configuration-settings"] -=== Override default Fleet configuration settings - -ECK uses environment variables to control how Elastic Agent and Fleet Server should be configured. Sometimes, it might be required to override some of these settings. For example, if Kibana TLS certificate is signed by a well-known root and can't include `kibana-kb-http.namespace.svc` as a SAN, `KIBANA_FLEET_HOST` can be overriden to point to the URL that the certificate specifies. To do that, specify environment variable, as shown in the following example. +[id="{p}-elastic-agent-control-fleet-policy-selection"] +=== Control Fleet policy selection +ECK uses the default policy to enroll Elastic Agents in Fleet and the default Fleet Server policy to enroll Fleet Server. A different policy can be chosen by using the `policyID` attribute in the Elastic Agent resource: [source,yaml] ---- -... + +apiVersion: agent.k8s.elastic.co/v1alpha1 +kind: Agent +metadata: + name: fleet-server-sample spec: - deployment: - podTemplate: - spec: - containers: - - name: agent - env: - - name: KIBANA_FLEET_HOST - value: "https://kibana.example.com:443" + policyID: my-custom-policy ... ---- -Check the Elastic Agent link:https://www.elastic.co/guide/en/fleet/current/agent-environment-variables.html[docs] to get a list of all the environment variables that can be used. +Please note that the environment variables related to policy selection mentioned in the Elastic Agent link:https://www.elastic.co/guide/en/fleet/current/agent-environment-variables.html[docs] like `FLEET_SERVER_POLICY_ID` will not take effect if enrollment is managed by the ECK operator. [id="{p}-elastic-agent-fleet-configuration-examples"] From 786c969744a8bb43859401f7efee92d9a8623b92 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Mon, 18 Jul 2022 11:46:37 +0200 Subject: [PATCH 23/26] Set FLEET_SERVER_POLICY_ID --- pkg/controller/agent/fleet.go | 35 +++++++++++++++--------------- pkg/controller/agent/fleet_test.go | 25 +++++++++++++-------- pkg/controller/agent/pod.go | 25 ++++++++++++--------- pkg/controller/agent/pod_test.go | 26 +++++++++++++--------- 4 files changed, 64 insertions(+), 47 deletions(-) diff --git a/pkg/controller/agent/fleet.go b/pkg/controller/agent/fleet.go index 57dbba8171..930b7a7949 100644 --- a/pkg/controller/agent/fleet.go +++ b/pkg/controller/agent/fleet.go @@ -45,6 +45,10 @@ type EnrollmentAPIKey struct { PolicyID string `json:"policy_id,omitempty"` } +func (e EnrollmentAPIKey) isEmpty() bool { + return !e.Active && e.ID == "" && e.APIKey == "" && e.PolicyID == "" +} + // PolicyList is a wrapper for a list of agent policies as returned by the Fleet API. type PolicyList struct { Items []Policy `json:"items"` @@ -198,25 +202,25 @@ func (f fleetAPI) setupFleet(ctx context.Context) error { return f.request(ctx, http.MethodPost, "setup", nil, nil) } -func maybeReconcileFleetEnrollment(params Params, result *reconciler.Results) string { +func maybeReconcileFleetEnrollment(params Params, result *reconciler.Results) EnrollmentAPIKey { if !params.Agent.Spec.KibanaRef.IsDefined() { - return "" + return EnrollmentAPIKey{} } reachable, err := isKibanaReachable(params.Context, params.Client, params.Agent.Spec.KibanaRef.WithDefaultNamespace(params.Agent.Namespace).NamespacedName()) if err != nil { result.WithError(err) - return "" + return EnrollmentAPIKey{} } if !reachable { result.WithResult(reconcile.Result{Requeue: true}) - return "" + return EnrollmentAPIKey{} } kbConnectionSettings, err := extractClientConnectionSettings(params.Context, params.Agent, params.Client, commonv1.KibanaAssociationType) if err != nil { result.WithError(err) - return "" + return EnrollmentAPIKey{} } token, err := reconcileEnrollmentToken( @@ -242,24 +246,19 @@ func isKibanaReachable(ctx context.Context, client k8s.Client, kibanaNSN types.N return true, nil } -func reconcileEnrollmentToken( - ctx context.Context, - agent agentv1alpha1.Agent, - client k8s.Client, - api fleetAPI, -) (string, error) { +func reconcileEnrollmentToken(ctx context.Context, agent agentv1alpha1.Agent, client k8s.Client, api fleetAPI) (EnrollmentAPIKey, error) { // do we have an existing token that we have rolled out previously? tokenName, exists := agent.Annotations[FleetTokenAnnotation] if !exists { // setup fleet to create default policies (and tokens) if err := api.setupFleet(ctx); err != nil { - return "", err + return EnrollmentAPIKey{}, err } } // what policy should we enroll this agent in? policyID, err := reconcilePolicyID(ctx, agent, api) if err != nil { - return "", err + return EnrollmentAPIKey{}, err } if exists { // get the enrollment token identified by the annotation @@ -269,18 +268,18 @@ func reconcileEnrollmentToken( goto CREATE } if err != nil { - return "", err + return EnrollmentAPIKey{}, err } // if the token is valid and for the right policy we are done here if key.Active && key.PolicyID == policyID { - return key.APIKey, nil + return key, nil } } CREATE: key, err := api.createEnrollmentAPIKey(ctx, policyID) if err != nil { - return "", err + return EnrollmentAPIKey{}, err } // this potentially creates conflicts we could introduce reconciler state similar to the ES controller and handle it // on the top level but we would then potentially create redundant enrollment tokens in the Fleet API @@ -291,9 +290,9 @@ CREATE: err = client.Update(ctx, &agent) if err != nil { // we have failed to store the token id in an annotation let's try to remove the token again - return "", k8serrors.NewAggregate([]error{err, api.deleteEnrollmentAPIKey(ctx, key.ID)}) + return EnrollmentAPIKey{}, k8serrors.NewAggregate([]error{err, api.deleteEnrollmentAPIKey(ctx, key.ID)}) } - return key.APIKey, nil + return key, nil } func reconcilePolicyID(ctx context.Context, agent agentv1alpha1.Agent, api fleetAPI) (string, error) { diff --git a/pkg/controller/agent/fleet_test.go b/pkg/controller/agent/fleet_test.go index 81131c590b..aabf1e59dd 100644 --- a/pkg/controller/agent/fleet_test.go +++ b/pkg/controller/agent/fleet_test.go @@ -6,6 +6,7 @@ package agent import ( "context" + "encoding/json" "io/ioutil" "net/http" "strings" @@ -28,6 +29,12 @@ var ( ) func Test_reconcileEnrollmentToken(t *testing.T) { + asObject := func(raw string) EnrollmentAPIKey { + var r EnrollmentAPIKeyResult + require.NoError(t, json.Unmarshal([]byte(raw), &r)) + return r.Item + } + type args struct { agent v1alpha1.Agent client *k8s.Client @@ -36,7 +43,7 @@ func Test_reconcileEnrollmentToken(t *testing.T) { tests := []struct { name string args args - want string + want EnrollmentAPIKey wantErr bool }{ { @@ -54,7 +61,7 @@ func Test_reconcileEnrollmentToken(t *testing.T) { {"GET", "/api/fleet/enrollment_api_keys/some-token-id"}: {code: 200, body: enrollmentKeySample}, }), }, - want: "some-token", + want: asObject(enrollmentKeySample), wantErr: false, }, { @@ -77,7 +84,7 @@ func Test_reconcileEnrollmentToken(t *testing.T) { {"POST", "/api/fleet/enrollment_api_keys"}: {code: 200, body: enrollmentKeySample}, }), }, - want: "some-token", + want: asObject(enrollmentKeySample), wantErr: false, }, { @@ -98,7 +105,7 @@ func Test_reconcileEnrollmentToken(t *testing.T) { {"POST", "/api/fleet/enrollment_api_keys"}: {code: 200, body: enrollmentKeySample}, }), }, - want: "some-token", + want: asObject(enrollmentKeySample), wantErr: false, }, { @@ -119,7 +126,7 @@ func Test_reconcileEnrollmentToken(t *testing.T) { {"POST", "/api/fleet/enrollment_api_keys"}: {code: 200, body: enrollmentKeySample}, }), }, - want: "some-token", + want: asObject(enrollmentKeySample), wantErr: false, }, { @@ -137,7 +144,7 @@ func Test_reconcileEnrollmentToken(t *testing.T) { {"POST", "/api/fleet/enrollment_api_keys"}: {code: 200, body: enrollmentKeySample}, }), }, - want: "some-token", + want: asObject(enrollmentKeySample), wantErr: false, }, { @@ -156,7 +163,7 @@ func Test_reconcileEnrollmentToken(t *testing.T) { {"GET", "/api/fleet/agent_policies"}: {code: 500}, }), }, - want: "", + want: EnrollmentAPIKey{}, wantErr: true, }, { @@ -179,7 +186,7 @@ func Test_reconcileEnrollmentToken(t *testing.T) { {"GET", "/api/fleet/enrollment_api_keys/some-token-id"}: {code: 200, body: fleetServerKeySample}, }), }, - want: "fleet-token", + want: asObject(fleetServerKeySample), wantErr: false, }, { @@ -207,7 +214,7 @@ func Test_reconcileEnrollmentToken(t *testing.T) { {"DELETE", "/api/fleet/enrollment_api_keys/some-token-id"}: {code: 200}, }), }, - want: "", + want: EnrollmentAPIKey{}, wantErr: true, }, } diff --git a/pkg/controller/agent/pod.go b/pkg/controller/agent/pod.go index 37b55c3ab6..f81d9d86a1 100644 --- a/pkg/controller/agent/pod.go +++ b/pkg/controller/agent/pod.go @@ -70,6 +70,7 @@ const ( FleetServerElasticsearchUsername = "FLEET_SERVER_ELASTICSEARCH_USERNAME" FleetServerElasticsearchPassword = "FLEET_SERVER_ELASTICSEARCH_PASSWORD" //nolint:gosec FleetServerElasticsearchCA = "FLEET_SERVER_ELASTICSEARCH_CA" + FleetServerPolicyID = "FLEET_SERVER_POLICY_ID" FleetServerServiceToken = "FLEET_SERVER_SERVICE_TOKEN" //nolint:gosec ubiSharedCAPath = "/etc/pki/ca-trust/source/anchors/" @@ -111,7 +112,7 @@ var ( } ) -func buildPodTemplate(params Params, fleetCerts *certificates.CertificatesSecret, fleetToken string, configHash hash.Hash32) (corev1.PodTemplateSpec, error) { +func buildPodTemplate(params Params, fleetCerts *certificates.CertificatesSecret, fleetToken EnrollmentAPIKey, configHash hash.Hash32) (corev1.PodTemplateSpec, error) { defer tracing.Span(¶ms.Context)() spec := ¶ms.Agent.Spec builder := defaults.NewPodTemplateBuilder(params.GetPodTemplate(), ContainerName) @@ -176,7 +177,7 @@ func buildPodTemplate(params Params, fleetCerts *certificates.CertificatesSecret return builder.PodTemplate, nil } -func amendBuilderForFleetMode(params Params, fleetCerts *certificates.CertificatesSecret, fleetToken string, builder *defaults.PodTemplateBuilder, configHash hash.Hash) (*defaults.PodTemplateBuilder, error) { +func amendBuilderForFleetMode(params Params, fleetCerts *certificates.CertificatesSecret, fleetToken EnrollmentAPIKey, builder *defaults.PodTemplateBuilder, configHash hash.Hash) (*defaults.PodTemplateBuilder, error) { esAssociation, err := getRelatedEsAssoc(params) if err != nil { return nil, err @@ -218,7 +219,7 @@ func amendBuilderForFleetMode(params Params, fleetCerts *certificates.Certificat return builder, nil } -func applyEnvVars(params Params, fleetToken string, builder *defaults.PodTemplateBuilder) (*defaults.PodTemplateBuilder, error) { +func applyEnvVars(params Params, fleetToken EnrollmentAPIKey, builder *defaults.PodTemplateBuilder) (*defaults.PodTemplateBuilder, error) { fleetModeEnvVars, err := getFleetModeEnvVars(params.Agent, params.Client, fleetToken) if err != nil { return nil, err @@ -425,10 +426,10 @@ func certificatesDir(association commonv1.Association) string { ) } -func getFleetModeEnvVars(agent agentv1alpha1.Agent, client k8s.Client, fleetToken string) (map[string]string, error) { +func getFleetModeEnvVars(agent agentv1alpha1.Agent, client k8s.Client, fleetToken EnrollmentAPIKey) (map[string]string, error) { result := map[string]string{} - for _, f := range []func(agentv1alpha1.Agent, k8s.Client, string) (map[string]string, error){ + for _, f := range []func(agentv1alpha1.Agent, k8s.Client, EnrollmentAPIKey) (map[string]string, error){ getFleetSetupKibanaEnvVars, getFleetSetupFleetEnvVars, getFleetSetupFleetServerEnvVars, @@ -443,23 +444,23 @@ func getFleetModeEnvVars(agent agentv1alpha1.Agent, client k8s.Client, fleetToke return result, nil } -func getFleetSetupKibanaEnvVars(agent agentv1alpha1.Agent, _ k8s.Client, fleetToken string) (map[string]string, error) { +func getFleetSetupKibanaEnvVars(agent agentv1alpha1.Agent, _ k8s.Client, fleetToken EnrollmentAPIKey) (map[string]string, error) { if !agent.Spec.KibanaRef.IsDefined() { return map[string]string{}, nil } - if fleetToken == "" { + if fleetToken.isEmpty() { return nil, errors.New("fleet enrollment token must not be empty, potential programmer error") } envVars := map[string]string{ - FleetEnrollmentToken: fleetToken, + FleetEnrollmentToken: fleetToken.APIKey, } return envVars, nil } -func getFleetSetupFleetEnvVars(agent agentv1alpha1.Agent, client k8s.Client, _ string) (map[string]string, error) { +func getFleetSetupFleetEnvVars(agent agentv1alpha1.Agent, client k8s.Client, fleetToken EnrollmentAPIKey) (map[string]string, error) { fleetCfg := map[string]string{} if agent.Spec.KibanaRef.IsDefined() { @@ -480,6 +481,10 @@ func getFleetSetupFleetEnvVars(agent agentv1alpha1.Agent, client k8s.Client, _ s fleetCfg[FleetURL] = fleetURL fleetCfg[FleetCA] = path.Join(FleetCertsMountPath, certificates.CAFileName) + // Fleet Server needs a policy ID to bootstrap itself unless a policy marked as default is used. + if agent.Spec.KibanaRef.IsDefined() && !fleetToken.isEmpty() { + fleetCfg[FleetServerPolicyID] = fleetToken.PolicyID + } } else if agent.Spec.FleetServerRef.IsDefined() { assoc, err := association.SingleAssociationOfType(agent.GetAssociations(), commonv1.FleetServerAssociationType) if err != nil { @@ -501,7 +506,7 @@ func getFleetSetupFleetEnvVars(agent agentv1alpha1.Agent, client k8s.Client, _ s return fleetCfg, nil } -func getFleetSetupFleetServerEnvVars(agent agentv1alpha1.Agent, client k8s.Client, _ string) (map[string]string, error) { +func getFleetSetupFleetServerEnvVars(agent agentv1alpha1.Agent, client k8s.Client, _ EnrollmentAPIKey) (map[string]string, error) { if !agent.Spec.FleetServerEnabled { return map[string]string{}, nil } diff --git a/pkg/controller/agent/pod_test.go b/pkg/controller/agent/pod_test.go index 42be434acb..a0fca4adea 100644 --- a/pkg/controller/agent/pod_test.go +++ b/pkg/controller/agent/pod_test.go @@ -177,7 +177,7 @@ func Test_amendBuilderForFleetMode(t *testing.T) { builder := generateBuilder() hash := sha256.New224() - gotBuilder, gotErr := amendBuilderForFleetMode(tt.params, fleetCerts, "", builder, hash) + gotBuilder, gotErr := amendBuilderForFleetMode(tt.params, fleetCerts, EnrollmentAPIKey{}, builder, hash) require.Nil(t, gotErr) require.NotNil(t, gotBuilder) @@ -232,11 +232,16 @@ func Test_applyEnvVars(t *testing.T) { podTemplateBuilderWithFleetTokenSet := generateBuilder() podTemplateBuilderWithFleetTokenSet = podTemplateBuilderWithFleetTokenSet.WithEnv(corev1.EnvVar{Name: "FLEET_ENROLLMENT_TOKEN", Value: "custom"}) + testToken := EnrollmentAPIKey{ + APIKey: "test-token", + PolicyID: "policy-id", + } + f := false for _, tt := range []struct { name string params Params - fleetToken string + fleetToken EnrollmentAPIKey podTemplateBuilder *defaults.PodTemplateBuilder wantContainer corev1.Container wantSecretData map[string][]byte @@ -247,7 +252,7 @@ func Test_applyEnvVars(t *testing.T) { Agent: agent, Client: k8s.NewFakeClient(), }, - fleetToken: "test-token", + fleetToken: testToken, podTemplateBuilder: generateBuilder(), wantContainer: corev1.Container{ Name: "agent", @@ -272,7 +277,7 @@ func Test_applyEnvVars(t *testing.T) { Agent: agent, Client: k8s.NewFakeClient(), }, - fleetToken: "test-token", + fleetToken: testToken, podTemplateBuilder: podTemplateBuilderWithFleetTokenSet, wantContainer: corev1.Container{ Name: "agent", @@ -309,7 +314,7 @@ func Test_applyEnvVars(t *testing.T) { }, ), }, - fleetToken: "test-token", + fleetToken: testToken, podTemplateBuilder: generateBuilder(), wantContainer: corev1.Container{ Name: "agent", @@ -335,6 +340,7 @@ func Test_applyEnvVars(t *testing.T) { Optional: &f, }}}, {Name: "FLEET_SERVER_ENABLE", Value: "true"}, + {Name: "FLEET_SERVER_POLICY_ID", Value: "policy-id"}, {Name: "FLEET_URL", Value: "https://agent-agent-http.default.svc:8220"}, }, }, @@ -781,7 +787,7 @@ func Test_getFleetSetupKibanaEnvVars(t *testing.T) { for _, tt := range []struct { name string agent agentv1alpha1.Agent - fleetToken string + fleetToken EnrollmentAPIKey wantErr bool wantEnvVars map[string]string }{ @@ -794,14 +800,14 @@ func Test_getFleetSetupKibanaEnvVars(t *testing.T) { { name: "kibana ref present, but no token", agent: agent, - fleetToken: "", + fleetToken: EnrollmentAPIKey{}, wantEnvVars: nil, wantErr: true, }, { name: "kibana ref present, token populated", agent: agent, - fleetToken: "test-token", + fleetToken: EnrollmentAPIKey{APIKey: "test-token"}, wantEnvVars: map[string]string{ "FLEET_ENROLLMENT_TOKEN": "test-token", }, @@ -982,7 +988,7 @@ func Test_getFleetSetupFleetEnvVars(t *testing.T) { }, } { t.Run(tt.name, func(t *testing.T) { - gotEnvVars, gotErr := getFleetSetupFleetEnvVars(tt.agent, tt.client, "") + gotEnvVars, gotErr := getFleetSetupFleetEnvVars(tt.agent, tt.client, EnrollmentAPIKey{}) require.Equal(t, tt.wantEnvVars, gotEnvVars) require.Equal(t, tt.wantErr, gotErr != nil) @@ -1102,7 +1108,7 @@ func Test_getFleetSetupFleetServerEnvVars(t *testing.T) { }, } { t.Run(tt.name, func(t *testing.T) { - gotEnvVars, gotErr := getFleetSetupFleetServerEnvVars(tt.agent, tt.client, "") + gotEnvVars, gotErr := getFleetSetupFleetServerEnvVars(tt.agent, tt.client, EnrollmentAPIKey{}) require.Equal(t, tt.wantEnvVars, gotEnvVars) require.Equal(t, tt.wantErr, gotErr != nil) From 9ed4efdb0c4d55771b44fabab389ce9b91defee7 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Mon, 18 Jul 2022 16:17:56 +0200 Subject: [PATCH 24/26] doc updates for FLEET_SERVER_POLICY_ID --- .../agent-fleet.asciidoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/orchestrating-elastic-stack-applications/agent-fleet.asciidoc b/docs/orchestrating-elastic-stack-applications/agent-fleet.asciidoc index bfbbd846a4..caba63e530 100644 --- a/docs/orchestrating-elastic-stack-applications/agent-fleet.asciidoc +++ b/docs/orchestrating-elastic-stack-applications/agent-fleet.asciidoc @@ -312,7 +312,7 @@ spec: [id="{p}-elastic-agent-fleet-configuration-setting-referenced-resources"] === Set referenced resources -Both Fleet Server and Elastic Agent in Fleet mode can be automatically setup with Fleet by ECK. The ECK operator can set up Fleet in Kibana (which otherwise requires manual steps) and enroll Fleet Server in the default Fleet Server policy. Elastic Agent can be automatically enrolled in the default Elastic Agent policy. To allow ECK to set this up, provide a reference to a ECK-managed Kibana through the `kibanaRef` configuration element. +Both Fleet Server and Elastic Agent in Fleet mode can be automatically set up with Fleet by ECK. The ECK operator can set up Fleet in Kibana (which otherwise requires manual steps) and enroll Fleet Server in the default Fleet Server policy. Elastic Agent can be automatically enrolled in the default Elastic Agent policy. To allow ECK to set this up, provide a reference to a ECK-managed Kibana through the `kibanaRef` configuration element. [source,yaml,subs="attributes,+macros"] ---- @@ -364,7 +364,7 @@ In contrast to Elastic Agents in standalone mode, the configuration is managed t [id="{p}-elastic-agent-fleet-configuration-upgrade-specification"] === Upgrade the Elastic Agent specification -You can upgrade the Elastic Agent version or change settings by editing the YAML specification file. ECK applies the changes by performing a rolling restart of the Agent's Pods. Depending on the settings that you used, ECK configures an agent to set up Fleet in Kibana, enrolls itself in Fleet, or restarts Elastic Agent on certificate rollover. +You can upgrade the Elastic Agent version or change settings by editing the YAML specification file. ECK applies the changes by performing a rolling restart of the Agent's Pods. Depending on the settings that you used, ECK will set up Fleet in Kibana, enrolls the agent in Fleet, or restarts Elastic Agent on certificate rollover. [id="{p}-elastic-agent-fleet-configuration-chose-the-deployment-model"] === Choose the deployment model @@ -484,7 +484,7 @@ spec: ... ---- -Please note that the environment variables related to policy selection mentioned in the Elastic Agent link:https://www.elastic.co/guide/en/fleet/current/agent-environment-variables.html[docs] like `FLEET_SERVER_POLICY_ID` will not take effect if enrollment is managed by the ECK operator. +Please note that the environment variables related to policy selection mentioned in the Elastic Agent link:https://www.elastic.co/guide/en/fleet/current/agent-environment-variables.html[docs] like `FLEET_SERVER_POLICY_ID` will be managed by the ECK operator. [id="{p}-elastic-agent-fleet-configuration-examples"] From 1af696cff95b5e0761e5398fee28bbe769796c65 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Mon, 18 Jul 2022 17:48:20 +0200 Subject: [PATCH 25/26] Do not replace default cert pool when no custom certs present --- pkg/controller/common/http/http_client.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/controller/common/http/http_client.go b/pkg/controller/common/http/http_client.go index cba5b7db60..17b74d46b6 100644 --- a/pkg/controller/common/http/http_client.go +++ b/pkg/controller/common/http/http_client.go @@ -26,14 +26,9 @@ import ( // match Kubernetes internal service name, but only the user-facing public endpoint // - set APM spans with each request func Client(dialer net.Dialer, caCerts []*x509.Certificate, timeout time.Duration) *http.Client { - certPool := x509.NewCertPool() - for _, c := range caCerts { - certPool.AddCert(c) - } transportConfig := http.Transport{ TLSClientConfig: &tls.Config{ - RootCAs: certPool, MinVersion: tls.VersionTLS12, // this is the default as of Go 1.18 we are just restating this here for clarity. // We use our own certificate verification because we permit users to provide their own certificates, which may not @@ -50,6 +45,15 @@ func Client(dialer net.Dialer, caCerts []*x509.Certificate, timeout time.Duratio }, } + // only replace default cert pool if we have certificates to trust + if caCerts != nil { + certPool := x509.NewCertPool() + for _, c := range caCerts { + certPool.AddCert(c) + } + transportConfig.TLSClientConfig.RootCAs = certPool + } + transportConfig.TLSClientConfig.VerifyPeerCertificate = func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { if verifiedChains != nil { return errors.New("tls: non-nil verifiedChains argument breaks crypto/tls.Config.VerifyPeerCertificate contract") From e8647b11fb7500a0c708dd130beb9e045f650ebf Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Mon, 18 Jul 2022 18:15:47 +0200 Subject: [PATCH 26/26] lint --- pkg/controller/common/http/http_client.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/controller/common/http/http_client.go b/pkg/controller/common/http/http_client.go index 17b74d46b6..1229a338e3 100644 --- a/pkg/controller/common/http/http_client.go +++ b/pkg/controller/common/http/http_client.go @@ -26,7 +26,6 @@ import ( // match Kubernetes internal service name, but only the user-facing public endpoint // - set APM spans with each request func Client(dialer net.Dialer, caCerts []*x509.Certificate, timeout time.Duration) *http.Client { - transportConfig := http.Transport{ TLSClientConfig: &tls.Config{ MinVersion: tls.VersionTLS12, // this is the default as of Go 1.18 we are just restating this here for clarity.