Remove ECS fields from gcp's custom fields definition (#675)

* Remove clour.instance.name, service.type and dns.answers from gcp's custom fields definition

* Update the Wazuh Common Schema

* Update CHANGELOG.md

* Update push_schema.sh to commit everything under the docs/ folder

* Update the Wazuh Common Schema

---------

Co-authored-by: Wazuh Indexer Bot <github_devel_xdrsiem_indexer@wazuh.com>
Co-authored-by: Alex Ruiz <alejandro.ruiz.becerra@wazuh.com>

CHANGELOG.md

@@ -79,6 +79,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Update `DEVELOPER_GUIDE.md` to use JDK 21 [(#538)](https://github.com/wazuh/wazuh-indexer-plugins/pull/538)
 - Fix WCS generator modules detection issues [(#620)](https://github.com/wazuh/wazuh-indexer-plugins/pull/620)
 - Fix verify_integrations script to read the integrations from module_list.txt [(#640)](https://github.com/wazuh/wazuh-indexer-plugins/pull/640)
+- Remove ECS fields from gcp's custom fields definition [(#675)](https://github.com/wazuh/wazuh-indexer-plugins/pull/675)
 
 ### Security
 - Reduce risk of GITHUB_TOKEN exposure [(#484)](https://github.com/wazuh/wazuh-indexer-plugins/pull/484)

ecs/generator/push_schema.sh

@@ -62,7 +62,7 @@ function push_changes() {
   echo
   echo "---> Pushing changes to the repository..."
   git add plugins/setup/src/main/resources/*.json
-  git add ecs/**/docs/fields.csv
+  git add ecs/**/docs/*
   git add ecs/module_list.txt
   if [[ $(git status --porcelain --untracked-files=no | wc -l) -gt 0 ]]; then
     git status --short --untracked-files=no

ecs/stateless/cloud-services/gcp/docs/ecs_flat.yml

@@ -1511,16 +1511,13 @@ cloud.instance.id:
   type: keyword
 cloud.instance.name:
   dashed_name: cloud-instance-name
-  description: Provides the normalized instance name derived by removing any leading
-    segments from the VM instance name
-  example: example-name
+  description: Instance name of the host machine.
   flat_name: cloud.instance.name
   ignore_above: 1024
-  level: custom
-  name: cloud.instance.name
+  level: extended
+  name: instance.name
   normalize: []
-  short: Provides the normalized instance name derived by removing any leading segments
-    from the VM instance name
+  short: Instance name of the host machine.
   type: keyword
 cloud.machine.type:
   dashed_name: cloud-machine-type
@@ -3326,14 +3323,23 @@ dll.pe.sections.virtual_size:
   type: long
 dns.answers:
   dashed_name: dns-answers
-  description: The DNS class of the resource record
+  description: 'An array containing an object for each answer section returned by
+    the server.
+
+    The main keys that should be present in these objects are defined by ECS. Records
+    that have more information may contain more keys than what ECS defines.
+
+    Not all DNS data sources give all details about DNS answers. At minimum, answer
+    objects must contain the `data` key. If more information is available, map as
+    much of it to ECS as possible, and add any additional fields to the answer objects
+    as custom fields.'
   flat_name: dns.answers
-  level: custom
-  name: dns.answers
+  level: extended
+  name: answers
   normalize:
   - array
-  short: The DNS class of the resource record
-  type: nested
+  short: Array of DNS answers.
+  type: object
 dns.answers.class:
   dashed_name: dns-answers-class
   description: The class of DNS data contained in this resource record.
@@ -19465,14 +19471,20 @@ service.target.version:
   type: keyword
 service.type:
   dashed_name: service-type
-  description: Indicates the type of service that generated the event
-  example: example-type
+  description: 'The type of the service data is collected from.
+
+    The type can be used to group and correlate logs and metrics from one service
+    type.
+
+    Example: If logs or metrics are collected from Elasticsearch, `service.type` would
+    be `elasticsearch`.'
+  example: elasticsearch
   flat_name: service.type
   ignore_above: 1024
-  level: custom
-  name: service.type
+  level: core
+  name: type
   normalize: []
-  short: Indicates the type of service that generated the event
+  short: The type of the service.
   type: keyword
 service.version:
   dashed_name: service-version

ecs/stateless/cloud-services/gcp/docs/fields.csv

@@ -116,7 +116,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 9.1.0,true,cloud,cloud.account.name,keyword,extended,,elastic-dev,The cloud account name.
 9.1.0,true,cloud,cloud.availability_zone,keyword,extended,,us-east-1c,"Availability zone in which this host, resource, or service is located."
 9.1.0,true,cloud,cloud.instance.id,keyword,extended,,i-1234567890abcdef0,Instance ID of the host machine.
-9.1.0,true,cloud,cloud.instance.name,keyword,custom,,example-name,Provides the normalized instance name derived by removing any leading segments from the VM instance name
+9.1.0,true,cloud,cloud.instance.name,keyword,extended,,,Instance name of the host machine.
 9.1.0,true,cloud,cloud.machine.type,keyword,extended,,t2.medium,Machine type of the host machine.
 9.1.0,true,cloud,cloud.origin.account.id,keyword,extended,,666777888999,The cloud account or organization id.
 9.1.0,true,cloud,cloud.origin.account.name,keyword,extended,,elastic-dev,The cloud account name.
@@ -255,7 +255,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 9.1.0,true,dll,dll.pe.sections.physical_size,long,extended,,,PE Section List physical size.
 9.1.0,true,dll,dll.pe.sections.var_entropy,long,extended,,,Variance for Shannon entropy calculation from the section.
 9.1.0,true,dll,dll.pe.sections.virtual_size,long,extended,,,PE Section List virtual size. This is always the same as `physical_size`.
-9.1.0,true,dns,dns.answers,nested,custom,array,,The DNS class of the resource record
+9.1.0,true,dns,dns.answers,object,extended,array,,Array of DNS answers.
 9.1.0,true,dns,dns.answers.class,keyword,extended,,IN,The class of DNS data contained in this resource record.
 9.1.0,true,dns,dns.answers.data,keyword,extended,,10.10.10.10,The data describing the resource.
 9.1.0,true,dns,dns.answers.name,keyword,extended,,www.example.com,The domain name to which this resource record pertains.
@@ -1501,7 +1501,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 9.1.0,true,service,service.target.state,keyword,core,,,Current state of the service.
 9.1.0,true,service,service.target.type,keyword,core,,elasticsearch,The type of the service.
 9.1.0,true,service,service.target.version,keyword,core,,3.2.4,Version of the service.
-9.1.0,true,service,service.type,keyword,custom,,example-type,Indicates the type of service that generated the event
+9.1.0,true,service,service.type,keyword,core,,elasticsearch,The type of the service.
 9.1.0,true,service,service.version,keyword,core,,3.2.4,Version of the service.
 9.1.0,true,source,source.address,keyword,extended,,,Source network address.
 9.1.0,true,source,source.as.number,long,extended,,15169,Unique number allocated to the autonomous system.

ecs/stateless/cloud-services/gcp/fields/custom/gcp.yml

@@ -856,13 +856,6 @@
     description: Contains the original virtual machine instance name prior to any
       normalization
     example: example-vminstancename
-  - name: cloud.instance.name
-    type: keyword
-    level: custom
-    description: >-
-      Provides the normalized instance name derived by removing any leading segments
-      from the VM instance name
-    example: example-name
   - name: gcp_dns.vmprojectid
     type: keyword
     level: custom
@@ -891,11 +884,6 @@
     description: Indicates whether the DNS query was authenticated based on the authAnswer
       flag
     example: true
-  - name: service.type
-    type: keyword
-    level: custom
-    description: Indicates the type of service that generated the event
-    example: example-type
   - name: metricset.name
     type: keyword
     level: custom
@@ -1131,12 +1119,6 @@
     description: Name of the destination Virtual Private Cloud network where traffic
       is received
     example: example-vpc_name
-  - name: dns.answers
-    type: nested
-    level: custom
-    description: The DNS class of the resource record
-    normalize:
-    - array
   - name: gcp_audit.event_provider
     type: keyword
     level: custom

plugins/setup/src/main/resources/templates/streams/cloud-services-gcp.json

@@ -1280,7 +1280,7 @@
                   "type": "keyword"
                 }
               },
-              "type": "nested"
+              "type": "object"
             },
             "header_flags": {
               "ignore_above": 1024,