Add cronjob metadata by default (#30637)

CHANGELOG.next.asciidoc

@@ -90,6 +90,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...main[Check the HEAD dif
 - Add FIPS configuration option for all AWS API calls. {pull}[28899]
 - Add support for non-unique Kafka headers for output messages. {pull}30369[30369]
 - Add action_input_type for the .fleet-actions-results {pull}30562[30562]
+- Add cronjob metadata by default {pull}30637[30637]
 
 *Auditbeat*
 

deploy/kubernetes/filebeat-kubernetes.yaml

@@ -188,6 +188,10 @@ rules:
   resources:
     - replicasets
   verbs: ["get", "list", "watch"]
+- apiGroups: ["batch"]
+  resources:
+    - jobs
+  verbs: ["get", "list", "watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role

deploy/kubernetes/filebeat/filebeat-role.yaml

@@ -18,6 +18,10 @@ rules:
   resources:
     - replicasets
   verbs: ["get", "list", "watch"]
+- apiGroups: ["batch"]
+  resources:
+    - jobs
+  verbs: ["get", "list", "watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role

deploy/kubernetes/heartbeat-kubernetes.yaml

@@ -185,6 +185,10 @@ rules:
   resources:
     - replicasets
   verbs: ["get", "list", "watch"]
+- apiGroups: ["batch"]
+  resources:
+    - jobs
+  verbs: ["get", "list", "watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role

deploy/kubernetes/heartbeat/heartbeat-role.yaml

@@ -16,6 +16,10 @@ rules:
   resources:
     - replicasets
   verbs: ["get", "list", "watch"]
+- apiGroups: ["batch"]
+  resources:
+    - jobs
+  verbs: ["get", "list", "watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role

libbeat/common/kubernetes/metadata/config.go

@@ -35,6 +35,7 @@ type AddResourceMetadataConfig struct {
 	Node       *common.Config `config:"node"`
 	Namespace  *common.Config `config:"namespace"`
 	Deployment bool           `config:"deployment"`
+	CronJob    bool           `config:"cronjob"`
 }
 
 // InitDefaults initializes the defaults for the config.
@@ -56,5 +57,6 @@ func GetDefaultResourceMetadataConfig() *AddResourceMetadataConfig {
 		Node:       metaCfg,
 		Namespace:  metaCfg,
 		Deployment: true,
+		CronJob:    true,
 	}
 }

libbeat/common/kubernetes/metadata/pod.go

@@ -88,7 +88,6 @@ func (p *pod) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) common.
 	out := p.resource.GenerateK8s("pod", obj, opts...)
 
 	// check if Pod is handled by a ReplicaSet which is controlled by a Deployment
-	// TODO: same happens with CronJob vs Job. The hierarcy there is CronJob->Job->Pod
 	if p.addResourceMetadata.Deployment {
 		rsName, _ := out.GetValue("replicaset.name")
 		if rsName, ok := rsName.(string); ok {
@@ -99,6 +98,17 @@ func (p *pod) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) common.
 		}
 	}
 
+	// check if Pod is handled by a Job which is controlled by a CronJob
+	if p.addResourceMetadata.CronJob {
+		jobName, _ := out.GetValue("job.name")
+		if jobName, ok := jobName.(string); ok {
+			dep := p.getCronjobOfJob(jobName, po.GetNamespace())
+			if dep != "" {
+				out.Put("cronjob.name", dep)
+			}
+		}
+	}
+
 	if p.node != nil {
 		meta := p.node.GenerateFromName(po.Spec.NodeName, WithMetadata("node"))
 		if meta != nil {
@@ -162,3 +172,24 @@ func (p *pod) getRSDeployment(rsName string, ns string) string {
 	}
 	return ""
 }
+
+// getCronjobOfJob return the name of the Cronjob object that
+// owns the Job with the given name under the given Namespace
+func (p *pod) getCronjobOfJob(jobName string, ns string) string {
+	if p.client == nil {
+		return ""
+	}
+	cronjob, err := p.client.BatchV1().Jobs(ns).Get(context.TODO(), jobName, metav1.GetOptions{})
+	if err != nil {
+		return ""
+	}
+	for _, ref := range cronjob.GetOwnerReferences() {
+		if ref.Controller != nil && *ref.Controller {
+			switch ref.Kind {
+			case "CronJob":
+				return ref.Name
+			}
+		}
+	}
+	return ""
+}

libbeat/docs/shared-autodiscover.asciidoc

@@ -178,7 +178,9 @@ endif::[]
   while annotations are not added by default. This settings are useful when labels' and annotations'
   storing requires special handling to avoid overloading the storage output. The enrichment of `node` or `namespace` metadata
   can be individually disabled by setting `enabled: false`. If resource is `pod` and it is created from a `deployment`, by default
-  the deployment name is added, this can be disabled by set to `false`.
+  the deployment name is added, this can be disabled by setting `deployment: false`.
+  If resource is `pod` and it is created from a `cronjob`, by default
+  the cronjob name is added, this can be disabled by setting `cronjob: false`.
   Example:
 
 ["source","yaml",subs="attributes"]
@@ -190,6 +192,7 @@ endif::[]
           include_labels: ["nodelabel2"]
           include_annotations: ["nodeannotation1"]
         deployment: false
+        cronjob: false
 -------------------------------------------------------------------------------------
 
 `unique`:: (Optional) Defaults to `false`. Marking an autodiscover provider as unique results into

metricbeat/docs/modules/kubernetes.asciidoc

@@ -241,6 +241,7 @@ metricbeat.modules:
   #     include_labels: ["nodelabel2"]
   #     include_annotations: ["nodeannotation1"]
   #   deployment: false
+  #   cronjob: false
   # Kubernetes client QPS and burst can be configured additionally
   #kube_client_options:
   #  qps: 5

metricbeat/metricbeat.reference.yml

@@ -516,6 +516,7 @@ metricbeat.modules:
   #     include_labels: ["nodelabel2"]
   #     include_annotations: ["nodeannotation1"]
   #   deployment: false
+  #   cronjob: false
   # Kubernetes client QPS and burst can be configured additionally
   #kube_client_options:
   #  qps: 5

metricbeat/module/kubernetes/_meta/config.reference.yml

@@ -33,6 +33,7 @@
   #     include_labels: ["nodelabel2"]
   #     include_annotations: ["nodeannotation1"]
   #   deployment: false
+  #   cronjob: false
   # Kubernetes client QPS and burst can be configured additionally
   #kube_client_options:
   #  qps: 5

metricbeat/module/kubernetes/_meta/config.yml

@@ -35,6 +35,7 @@
   #     include_labels: ["nodelabel2"]
   #     include_annotations: ["nodeannotation1"]
   #   deployment: false
+  #   cronjob: false
   # Kubernetes client QPS and burst can be configured additionally
   #kube_client_options:
   #  qps: 5

metricbeat/module/kubernetes/kubernetes.yml

@@ -111,19 +111,23 @@ spec:
 apiVersion: batch/v1
 kind: CronJob
 metadata:
-  name: basic-cronjob
+  name: hello
+  labels:
+    k8s-app: myjob
 spec:
-  schedule: "* * * * *"
+  schedule: "*/1 * * * *"
   jobTemplate:
     spec:
       template:
-        metadata:
-          name: basic-job
         spec:
           containers:
-          - name: hello
-            image: alpine:3
-            command: ["sh", "-c", "echo Hello!"]
+            - name: hello
+              image: busybox
+              imagePullPolicy: IfNotPresent
+              command:
+                - /bin/sh
+                - -c
+                - date; echo Hello from the Kubernetes cluster
           restartPolicy: Never
 ---
 apiVersion: v1

metricbeat/module/kubernetes/state_cronjob/_meta/docs.asciidoc

@@ -1,6 +1,13 @@
 This is the `state_cronjob` metricset of the Kubernetes module.
 
-This metricset does not add metadata by default and hence in order to
-add metadata for this one need to configure the metricset with `add_metadata: true`
-and uncomment the proper `apiGroup` in the `ClusterRole`. Metadata are only available
-for versions of k8s >= v1.21.
+This metricset adds metadata by default only for versions of k8s >= v1.21.
+For older versions the APIs are not compatible and one need to configure the
+metricset with `add_metadata: false` and remove the proper `apiGroup` in the `ClusterRole`:
+
+["source","yaml",subs="attributes"]
+-------------------------------------------------------------------------------------
+- apiGroups: [ "batch" ]
+  resources:
+  - cronjobs
+-------------------------------------------------------------------------------------
+

metricbeat/module/kubernetes/state_cronjob/state_cronjob.go

@@ -59,15 +59,11 @@ func NewCronJobMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) {
 		return nil, fmt.Errorf("must be child of kubernetes module")
 	}
 
-	config := util.GetDefaultDisabledMetaConfig()
-	if err := base.Module().UnpackConfig(&config); err != nil {
-		return nil, fmt.Errorf("error loading config of kubernetes module")
-	}
-
 	ms := CronJobMetricSet{
 		BaseMetricSet: base,
 		prometheus:    prometheus,
 		mod:           mod,
+		enricher:      util.NewResourceMetadataEnricher(base, &kubernetes.CronJob{}, false),
 		mapping: &p.MetricsMapping{
 			Metrics: map[string]p.MetricMap{
 				"kube_cronjob_info":                           p.InfoMetric(),
@@ -86,10 +82,7 @@ func NewCronJobMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) {
 			},
 		},
 	}
-	if config.AddMetadata {
-		ms.enricher = util.NewResourceMetadataEnricher(
-			base, &kubernetes.CronJob{}, false)
-	}
+
 	return &ms, nil
 }
 

metricbeat/module/kubernetes/state_cronjob/state_cronjob_integration_test.go

@@ -30,7 +30,7 @@ import (
 )
 
 func TestFetchMetricset(t *testing.T) {
-	config := test.GetKubeStateMetricsConfigWithMetaDisabled(t, "state_cronjob")
+	config := test.GetKubeStateMetricsConfig(t, "state_cronjob")
 	metricSet := mbtest.NewFetcher(t, config)
 	events, errs := metricSet.FetchEvents()
 	if len(errs) > 0 {

metricbeat/module/kubernetes/test/integration.go

@@ -49,18 +49,6 @@ func GetKubeStateMetricsConfig(t *testing.T, metricSetName string) map[string]in
 	}
 }
 
-// GetKubeStateMetricsConfigWithMetaDisabled function returns configuration for talking to kube-state-metrics.
-func GetKubeStateMetricsConfigWithMetaDisabled(t *testing.T, metricSetName string) map[string]interface{} {
-	t.Helper()
-	return map[string]interface{}{
-		"module":       "kubernetes",
-		"metricsets":   []string{metricSetName},
-		"host":         "${NODE_NAME}",
-		"hosts":        []string{"kube-state-metrics:8080"},
-		"add_metadata": false,
-	}
-}
-
 // GetKubeletConfig function returns configuration for talking to Kubelet API.
 func GetKubeletConfig(t *testing.T, metricSetName string) map[string]interface{} {
 	t.Helper()

metricbeat/modules.d/kubernetes.yml.disabled

@@ -38,6 +38,7 @@
   #     include_labels: ["nodelabel2"]
   #     include_annotations: ["nodeannotation1"]
   #   deployment: false
+  #   cronjob: false
   # Kubernetes client QPS and burst can be configured additionally
   #kube_client_options:
   #  qps: 5

x-pack/metricbeat/metricbeat.reference.yml

@@ -904,6 +904,7 @@ metricbeat.modules:
   #     include_labels: ["nodelabel2"]
   #     include_annotations: ["nodeannotation1"]
   #   deployment: false
+  #   cronjob: false
   # Kubernetes client QPS and burst can be configured additionally
   #kube_client_options:
   #  qps: 5