From 3e58e4188a9e2b1a7a4f5d1e518544c8dba443da Mon Sep 17 00:00:00 2001 From: Fede Galland Date: Mon, 1 Dec 2025 12:46:38 -0300 Subject: [PATCH] Remove ECS fields from gcp's custom fields definition (#675) * Remove clour.instance.name, service.type and dns.answers from gcp's custom fields definition * Update the Wazuh Common Schema * Update CHANGELOG.md * Update push_schema.sh to commit everything under the docs/ folder * Update the Wazuh Common Schema --------- Co-authored-by: Wazuh Indexer Bot Co-authored-by: Alex Ruiz --- CHANGELOG.md | 1 + ecs/generator/push_schema.sh | 2 +- .../cloud-services/gcp/docs/ecs_flat.yml | 46 ++++++++++++------- .../cloud-services/gcp/docs/fields.csv | 6 +-- .../cloud-services/gcp/fields/custom/gcp.yml | 18 -------- .../templates/streams/cloud-services-gcp.json | 2 +- 6 files changed, 35 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 651732fc..4b3e2815 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,6 +79,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Update `DEVELOPER_GUIDE.md` to use JDK 21 [(#538)](https://github.com/wazuh/wazuh-indexer-plugins/pull/538) - Fix WCS generator modules detection issues [(#620)](https://github.com/wazuh/wazuh-indexer-plugins/pull/620) - Fix verify_integrations script to read the integrations from module_list.txt [(#640)](https://github.com/wazuh/wazuh-indexer-plugins/pull/640) +- Remove ECS fields from gcp's custom fields definition [(#675)](https://github.com/wazuh/wazuh-indexer-plugins/pull/675) ### Security - Reduce risk of GITHUB_TOKEN exposure [(#484)](https://github.com/wazuh/wazuh-indexer-plugins/pull/484) diff --git a/ecs/generator/push_schema.sh b/ecs/generator/push_schema.sh index 5765a4f5..1d8d78b9 100644 --- a/ecs/generator/push_schema.sh +++ b/ecs/generator/push_schema.sh @@ -62,7 +62,7 @@ function push_changes() { echo echo "---> Pushing changes to the repository..." git add plugins/setup/src/main/resources/*.json - git add ecs/**/docs/fields.csv + git add ecs/**/docs/* git add ecs/module_list.txt if [[ $(git status --porcelain --untracked-files=no | wc -l) -gt 0 ]]; then git status --short --untracked-files=no diff --git a/ecs/stateless/cloud-services/gcp/docs/ecs_flat.yml b/ecs/stateless/cloud-services/gcp/docs/ecs_flat.yml index 71337567..23cb6f98 100644 --- a/ecs/stateless/cloud-services/gcp/docs/ecs_flat.yml +++ b/ecs/stateless/cloud-services/gcp/docs/ecs_flat.yml @@ -1511,16 +1511,13 @@ cloud.instance.id: type: keyword cloud.instance.name: dashed_name: cloud-instance-name - description: Provides the normalized instance name derived by removing any leading - segments from the VM instance name - example: example-name + description: Instance name of the host machine. flat_name: cloud.instance.name ignore_above: 1024 - level: custom - name: cloud.instance.name + level: extended + name: instance.name normalize: [] - short: Provides the normalized instance name derived by removing any leading segments - from the VM instance name + short: Instance name of the host machine. type: keyword cloud.machine.type: dashed_name: cloud-machine-type @@ -3326,14 +3323,23 @@ dll.pe.sections.virtual_size: type: long dns.answers: dashed_name: dns-answers - description: The DNS class of the resource record + description: 'An array containing an object for each answer section returned by + the server. + + The main keys that should be present in these objects are defined by ECS. Records + that have more information may contain more keys than what ECS defines. + + Not all DNS data sources give all details about DNS answers. At minimum, answer + objects must contain the `data` key. If more information is available, map as + much of it to ECS as possible, and add any additional fields to the answer objects + as custom fields.' flat_name: dns.answers - level: custom - name: dns.answers + level: extended + name: answers normalize: - array - short: The DNS class of the resource record - type: nested + short: Array of DNS answers. + type: object dns.answers.class: dashed_name: dns-answers-class description: The class of DNS data contained in this resource record. @@ -19465,14 +19471,20 @@ service.target.version: type: keyword service.type: dashed_name: service-type - description: Indicates the type of service that generated the event - example: example-type + description: 'The type of the service data is collected from. + + The type can be used to group and correlate logs and metrics from one service + type. + + Example: If logs or metrics are collected from Elasticsearch, `service.type` would + be `elasticsearch`.' + example: elasticsearch flat_name: service.type ignore_above: 1024 - level: custom - name: service.type + level: core + name: type normalize: [] - short: Indicates the type of service that generated the event + short: The type of the service. type: keyword service.version: dashed_name: service-version diff --git a/ecs/stateless/cloud-services/gcp/docs/fields.csv b/ecs/stateless/cloud-services/gcp/docs/fields.csv index 982a97e1..1bac7e4f 100644 --- a/ecs/stateless/cloud-services/gcp/docs/fields.csv +++ b/ecs/stateless/cloud-services/gcp/docs/fields.csv @@ -116,7 +116,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 9.1.0,true,cloud,cloud.account.name,keyword,extended,,elastic-dev,The cloud account name. 9.1.0,true,cloud,cloud.availability_zone,keyword,extended,,us-east-1c,"Availability zone in which this host, resource, or service is located." 9.1.0,true,cloud,cloud.instance.id,keyword,extended,,i-1234567890abcdef0,Instance ID of the host machine. -9.1.0,true,cloud,cloud.instance.name,keyword,custom,,example-name,Provides the normalized instance name derived by removing any leading segments from the VM instance name +9.1.0,true,cloud,cloud.instance.name,keyword,extended,,,Instance name of the host machine. 9.1.0,true,cloud,cloud.machine.type,keyword,extended,,t2.medium,Machine type of the host machine. 9.1.0,true,cloud,cloud.origin.account.id,keyword,extended,,666777888999,The cloud account or organization id. 9.1.0,true,cloud,cloud.origin.account.name,keyword,extended,,elastic-dev,The cloud account name. @@ -255,7 +255,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 9.1.0,true,dll,dll.pe.sections.physical_size,long,extended,,,PE Section List physical size. 9.1.0,true,dll,dll.pe.sections.var_entropy,long,extended,,,Variance for Shannon entropy calculation from the section. 9.1.0,true,dll,dll.pe.sections.virtual_size,long,extended,,,PE Section List virtual size. This is always the same as `physical_size`. -9.1.0,true,dns,dns.answers,nested,custom,array,,The DNS class of the resource record +9.1.0,true,dns,dns.answers,object,extended,array,,Array of DNS answers. 9.1.0,true,dns,dns.answers.class,keyword,extended,,IN,The class of DNS data contained in this resource record. 9.1.0,true,dns,dns.answers.data,keyword,extended,,10.10.10.10,The data describing the resource. 9.1.0,true,dns,dns.answers.name,keyword,extended,,www.example.com,The domain name to which this resource record pertains. @@ -1501,7 +1501,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 9.1.0,true,service,service.target.state,keyword,core,,,Current state of the service. 9.1.0,true,service,service.target.type,keyword,core,,elasticsearch,The type of the service. 9.1.0,true,service,service.target.version,keyword,core,,3.2.4,Version of the service. -9.1.0,true,service,service.type,keyword,custom,,example-type,Indicates the type of service that generated the event +9.1.0,true,service,service.type,keyword,core,,elasticsearch,The type of the service. 9.1.0,true,service,service.version,keyword,core,,3.2.4,Version of the service. 9.1.0,true,source,source.address,keyword,extended,,,Source network address. 9.1.0,true,source,source.as.number,long,extended,,15169,Unique number allocated to the autonomous system. diff --git a/ecs/stateless/cloud-services/gcp/fields/custom/gcp.yml b/ecs/stateless/cloud-services/gcp/fields/custom/gcp.yml index 9a5f82a7..b190bae1 100644 --- a/ecs/stateless/cloud-services/gcp/fields/custom/gcp.yml +++ b/ecs/stateless/cloud-services/gcp/fields/custom/gcp.yml @@ -856,13 +856,6 @@ description: Contains the original virtual machine instance name prior to any normalization example: example-vminstancename - - name: cloud.instance.name - type: keyword - level: custom - description: >- - Provides the normalized instance name derived by removing any leading segments - from the VM instance name - example: example-name - name: gcp_dns.vmprojectid type: keyword level: custom @@ -891,11 +884,6 @@ description: Indicates whether the DNS query was authenticated based on the authAnswer flag example: true - - name: service.type - type: keyword - level: custom - description: Indicates the type of service that generated the event - example: example-type - name: metricset.name type: keyword level: custom @@ -1131,12 +1119,6 @@ description: Name of the destination Virtual Private Cloud network where traffic is received example: example-vpc_name - - name: dns.answers - type: nested - level: custom - description: The DNS class of the resource record - normalize: - - array - name: gcp_audit.event_provider type: keyword level: custom diff --git a/plugins/setup/src/main/resources/templates/streams/cloud-services-gcp.json b/plugins/setup/src/main/resources/templates/streams/cloud-services-gcp.json index e4581581..d339fcb2 100644 --- a/plugins/setup/src/main/resources/templates/streams/cloud-services-gcp.json +++ b/plugins/setup/src/main/resources/templates/streams/cloud-services-gcp.json @@ -1280,7 +1280,7 @@ "type": "keyword" } }, - "type": "nested" + "type": "object" }, "header_flags": { "ignore_above": 1024,