[Filebeat] Improve ECS categorization field mappings in kibana module (elastic#16652)

* Improve ECS categorization field mappings in kibana module

- event.kind
- event.outcome
- event.type
- convert pipeline to yaml

Closes elastic#16168

Author: leehinman

CHANGELOG.next.asciidoc

@@ -150,6 +150,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improve ECS field mappings in aws module. {issue}16154[16154] {pull}16307[16307]
 - Improve ECS categorization field mappings in googlecloud module. {issue}16030[16030] {pull}16500[16500]
 - Improve ECS field mappings in haproxy module. {issue}16162[16162] {pull}16529[16529]
+- Improve ECS categorization field mappings in kibana module. {issue}16168[16168] {pull}16652[16652]
 - Improve the decode_cef processor by reducing the number of memory allocations. {pull}16587[16587]
 - Add `cloudfoundry` input to send events from Cloud Foundry. {pull}16586[16586]
 - Improve ECS categorization field mappings in iis module. {issue}16165[16165] {pull}16618[16618]

filebeat/module/kibana/log/ingest/pipeline.json

@@ -0,0 +1,111 @@
+description: Pipeline for parsing Kibana logs
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'
+processors:
+- rename:
+    field: '@timestamp'
+    target_field: event.created
+- rename:
+    field: json
+    target_field: kibana.log.meta
+- date:
+    field: kibana.log.meta.@timestamp
+    formats:
+    - ISO8601
+    target_field: '@timestamp'
+- remove:
+    field: kibana.log.meta.@timestamp
+- rename:
+    field: kibana.log.meta.message
+    target_field: message
+- rename:
+    field: kibana.log.meta.state
+    target_field: kibana.log.state
+    ignore_missing: true
+- rename:
+    field: kibana.log.meta.pid
+    target_field: process.pid
+- rename:
+    field: kibana.log.meta.tags
+    target_field: kibana.log.tags
+- rename:
+    field: kibana.log.meta.res.statusCode
+    target_field: http.response.status_code
+    ignore_missing: true
+- rename:
+    field: kibana.log.meta.res.responseTime
+    target_field: temp.duration
+    ignore_missing: true
+- script:
+    lang: painless
+    source: ctx.event.duration = Math.round(ctx.temp.duration * params.scale)
+    params:
+      scale: 1000000
+    if: ctx.temp?.duration != null
+- remove:
+    field: temp.duration
+    ignore_missing: true
+- rename:
+    field: kibana.log.meta.res.contentLength
+    target_field: http.response.body.bytes
+    ignore_missing: true
+- rename:
+    field: kibana.log.meta.req.method
+    target_field: http.request.method
+    ignore_missing: true
+- rename:
+    field: kibana.log.meta.req.headers.referer
+    target_field: http.request.referrer
+    ignore_missing: true
+- rename:
+    field: kibana.log.meta.req.headers.user-agent
+    target_field: user_agent.original
+    ignore_missing: true
+- rename:
+    field: kibana.log.meta.req.remoteAddress
+    target_field: source.address
+    ignore_missing: true
+- set:
+    field: source.ip
+    value: '{{source.address}}'
+    if: ctx.source?.address != null
+- rename:
+    field: kibana.log.meta.req.url
+    target_field: url.original
+    ignore_missing: true
+- remove:
+    field: kibana.log.meta.req.referer
+    ignore_missing: true
+- remove:
+    field: kibana.log.meta.statusCode
+    ignore_missing: true
+- remove:
+    field: kibana.log.meta.method
+    ignore_missing: true
+- append:
+    field: service.name
+    value: kibana
+- set:
+    field: event.kind
+    value: event
+- script:
+    lang: painless
+    source: >-
+      if (ctx?.kibana?.log?.state != null) {
+        if (ctx.kibana.log.state == "red") {
+          ctx.event.type = "error";
+        } else {
+          ctx.event.type = "info";
+        }
+      }
+
+- set:
+    field: event.outcome
+    value: success
+    if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400"
+- set:
+    field: event.outcome
+    value: failure
+    if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400"

filebeat/module/kibana/log/ingest/pipeline.yml

@@ -5,5 +5,5 @@ var:
     default:
       - /var/log/kibana/kibana.stdout
 
-ingest_pipeline: ingest/pipeline.json
+ingest_pipeline: ingest/pipeline.yml
 input: config/log.yml