Use stricter field limits for the WCS indices (#589)

* Use stricter field limits for the WCS indices

* Update the Wazuh Common Schema

* Bring down 'max_docvalue_fields_search'

* Add changelog entry

* Adjust limits

* Update the Wazuh Common Schema

* Add script to count and update total fiels and apply those changes

* Improve count_and_update_total_fields script and document it

* Add Changelog entry and fix a typo

* Set stricter limits for cloud-services indices

* Revert wrong value for max_docvalue_fields setting for the system activity index

---------

Signed-off-by: Jorge Sánchez <jorge.sanchez@wazuh.com>
Signed-off-by: Álex Ruiz Becerra <alejandro.ruiz.becerra@wazuh.com>
Co-authored-by: Wazuh Indexer Bot <github_devel_xdrsiem_indexer@wazuh.com>
Co-authored-by: Jorge Sanchez <jorge.sanchez@wazuh.com>
This commit is contained in:
Álex Ruiz Becerra 2025-10-23 13:16:28 +02:00 committed by GitHub
parent bd17b549be
commit 85465e3e87
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
36 changed files with 228 additions and 32 deletions

View File

@ -39,6 +39,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Add browser-extensions and services inventory indices to documentation [(#574)](https://github.com/wazuh/wazuh-indexer-plugins/pull/574)
- Update index templates with agent fields [(#578)](https://github.com/wazuh/wazuh-indexer-plugins/pull/578)
- Rename indices from *-5.x-* to *-v5-* [(#597)](https://github.com/wazuh/wazuh-indexer-plugins/pull/597)
- Use stricter field limits for the WCS indices [(#589)](https://github.com/wazuh/wazuh-indexer-plugins/pull/589)
### Deprecated
-

View File

@ -9,6 +9,7 @@ The generation of the Wazuh Common Schema is automated using a set of scripts an
- [update_module_list.sh](./update_module_list.sh): generates the [module_list.txt](../module_list.txt) file, by scanning the [ecs/](..) folder. Run this script whenever a new module is added.
- [images/Dockerfile](./images/Dockerfile): Dockerfile to build the image used for the schema generation. Clones the ECS repository, which contains the main tooling.
- [images/generator.sh](./images/generator.sh): our actual schema generation script. It is executed inside the container. Contains post-processing steps to make the templates compatible with OpenSearch and to adapt them to our needs.
- [count_and_update_total_fields.sh](./count_and_update_total_fields.sh): counts fields in a generated index template and proposes (or applies with --apply) an updated mapping.total_fields.limit rounded up to the next 500.
### Requirements
@ -31,6 +32,11 @@ However, it can also be run locally. To do so, follow these steps.
./generate_schema.sh
```
3. Update the number of total field of each module:
``` bash
./count_and_update_total_fields.sh all --apply
```
The scripts can be invoked from any location. When successful, all the generated files will be copied to their corresponding folders.
A new `mappings` folder will be created inside the module's folder, containing all the generated files.

View File

@ -0,0 +1,189 @@
#!/usr/bin/env bash
# Count fields in a generated index template and update mapping.total_fields.limit
# Usage:
# ./count_and_update_total_fields.sh <module|all> [--apply]
# If --apply is not passed the script runs in dry-run mode and prints proposed values.
set -euo pipefail
if [[ $# -lt 1 ]]; then
echo "Usage: $0 <module|all> [--apply]" >&2
exit 1
fi
ARG1="$1"
APPLY=false
if [[ ${2:-} == "--apply" ]] || [[ ${3:-} == "--apply" ]]; then
APPLY=true
fi
# If ARG1 is 'all' we will read module names from ecs/module_list.txt
PROCESS_ALL=false
if [[ "$ARG1" == "all" ]]; then
PROCESS_ALL=true
fi
# Navigate to repository root
function navigate_to_project_root() {
local repo_root_marker=".github"
local script_path
script_path=$(dirname "$(realpath "$0")")
while [[ "$script_path" != "/" ]] && [[ ! -d "$script_path/$repo_root_marker" ]]; do
script_path=$(dirname "$script_path")
done
if [[ "$script_path" == "/" ]]; then
echo "Error: Unable to find the repository root." >&2
exit 1
fi
cd "$script_path"
}
navigate_to_project_root
REPO_ROOT="$(pwd)"
# process a single module name
process_module() {
local MODULE="$1"
MODULE_LIST_FILE="$REPO_ROOT/ecs/module_list.txt"
INDEX_TEMPLATE_BASENAME="index-template-${MODULE}.json"
if [[ "$MODULE" == stateless-* ]]; then
short=${MODULE#stateless-}
INDEX_TEMPLATE_BASENAME="index-template-${short}.json"
fi
if [[ -f "$MODULE_LIST_FILE" ]]; then
match=$(grep -E "\[${MODULE//./\.}\]=" "$MODULE_LIST_FILE" || true)
if [[ -n "$match" ]]; then
rhs=$(echo "$match" | sed -E 's/^[^=]*=//')
rhs=$(echo "$rhs" | tr -d ' "')
if [[ -n "$rhs" ]]; then
INDEX_TEMPLATE_BASENAME="$rhs"
fi
fi
fi
INDEX_TEMPLATE_PATH="plugins/setup/src/main/resources/$INDEX_TEMPLATE_BASENAME"
TEMPLATE_SETTINGS="ecs/${MODULE}/fields/template-settings.json"
TEMPLATE_SETTINGS_LEGACY="ecs/${MODULE}/fields/template-settings-legacy.json"
if ! command -v jq &> /dev/null; then
echo "Error: 'jq' is required but not installed." >&2
exit 1
fi
if [[ -z "$MODULE" ]]; then
echo "Usage: $0 <module|all> [--apply]" >&2
return 1
fi
if [[ ! -f "$REPO_ROOT/$INDEX_TEMPLATE_PATH" ]]; then
echo "Warning: Index template not found at $INDEX_TEMPLATE_PATH" >&2
return 0
fi
# jq filter to count fields
JQ_FILTER='def count_fields: (keys_unsorted | length) + ( map( if type == "object" then (.properties | select(.) | count_fields) // 0 + (.fields | select(.) | count_fields) // 0 else 0 end ) | add ); .mappings.properties | count_fields'
TOTAL_FIELDS=$(jq -r "$JQ_FILTER" "$REPO_ROOT/$INDEX_TEMPLATE_PATH" 2> /tmp/jq_error.log) || {
echo "Error: Could not parse JSON or find .mappings.properties in $INDEX_TEMPLATE_PATH" >&2
cat /tmp/jq_error.log >&2 || true
rm -f /tmp/jq_error.log
return 1
}
rm -f /tmp/jq_error.log
# compute next multiple of 500
PROPOSED=$(( ( (TOTAL_FIELDS + 499) / 500 ) * 500 ))
cat <<EOF
Module: $MODULE
Index template: $INDEX_TEMPLATE_PATH
Total fields: $TOTAL_FIELDS
Proposed mapping.total_fields.limit: $PROPOSED
EOF
if ! $APPLY; then
echo "Dry-run mode. To apply the change add --apply" >&2
return 0
fi
# Update JSON files in place using jq
update_file() {
local file="$1"
if [[ ! -f "$REPO_ROOT/$file" ]]; then
echo "Skipping missing file: $file" >&2
return
fi
if jq -e '.template? and .template.settings? and .template.settings["mapping.total_fields.limit"]' "$REPO_ROOT/$file" > /dev/null 2>&1; then
tmpfile=$(mktemp)
last_hex=$(tail -c1 "$REPO_ROOT/$file" 2> /dev/null | od -An -t x1 | tr -d ' \t\n' || true)
jq ".template.settings[\"mapping.total_fields.limit\"] = $PROPOSED" "$REPO_ROOT/$file" > "$tmpfile"
if [[ -n "$last_hex" && "$last_hex" != "0a" ]]; then
perl -0777 -pe 's/\n\z//' "$tmpfile" > "${tmpfile}.fix" && mv "${tmpfile}.fix" "$tmpfile"
fi
mv "$tmpfile" "$REPO_ROOT/$file"
echo "Updated $file -> $PROPOSED"
elif jq -e '.settings? and .settings["mapping.total_fields.limit"]' "$REPO_ROOT/$file" > /dev/null 2>&1; then
tmpfile=$(mktemp)
last_hex=$(tail -c1 "$REPO_ROOT/$file" 2> /dev/null | od -An -t x1 | tr -d ' \t\n' || true)
jq ".settings[\"mapping.total_fields.limit\"] = $PROPOSED" "$REPO_ROOT/$file" > "$tmpfile"
if [[ -n "$last_hex" && "$last_hex" != "0a" ]]; then
perl -0777 -pe 's/\n\z//' "$tmpfile" > "${tmpfile}.fix" && mv "${tmpfile}.fix" "$tmpfile"
fi
mv "$tmpfile" "$REPO_ROOT/$file"
echo "Updated $file -> $PROPOSED"
else
echo "No mapping.total_fields.limit key found in $file. Skipping." >&2
fi
}
update_file "$TEMPLATE_SETTINGS"
update_file "$TEMPLATE_SETTINGS_LEGACY"
update_file "$INDEX_TEMPLATE_PATH"
echo "Done. Files updated."
}
# If PROCESS_ALL, read module_list and process only stateless-* modules
if $PROCESS_ALL; then
MODULE_LIST_FILE="$REPO_ROOT/ecs/module_list.txt"
if [[ ! -f "$MODULE_LIST_FILE" ]]; then
echo "Error: $MODULE_LIST_FILE not found" >&2
exit 1
fi
modules_block=$(awk 'BEGIN{inside=0} /module_to_file=\(/ {inside=1; next} inside && /^\)/ {exit} inside {print}' "$MODULE_LIST_FILE" || true)
if [[ -z "$modules_block" ]]; then
echo "Error: Could not find module_to_file(...) block in $MODULE_LIST_FILE" >&2
exit 1
fi
mapfile -t MODULES < <(echo "$modules_block" | grep -oP '\[\K[^\]]+(?=\])' || true)
# Keep only stateless- modules
filtered=()
for m in "${MODULES[@]:-}"; do
if [[ "$m" == stateless-* ]]; then
filtered+=("$m")
fi
done
MODULES=("${filtered[@]}")
if [[ ${#MODULES[@]} -eq 0 ]]; then
echo "No stateless-* modules found in $MODULE_LIST_FILE" >&2
exit 1
fi
for m in "${MODULES[@]}"; do
process_module "$m"
done
exit 0
fi
# Otherwise process the single module provided as ARG1
MODULE="$ARG1"
process_module "$MODULE" || exit $?

0
ecs/generator/generate_schema.sh Normal file → Executable file
View File

View File

@ -5,7 +5,7 @@
"order": 1,
"settings": {
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-access-management",
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 5500,
"mapping.nested_fields.limit": 100,
"index": {
"number_of_shards": "3",

View File

@ -6,7 +6,7 @@
"template": {
"settings": {
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-access-management",
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 5500,
"mapping.nested_fields.limit": 100,
"index": {
"number_of_shards": "3",

View File

@ -5,7 +5,7 @@
"order": 1,
"settings": {
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-applications",
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 6000,
"mapping.nested_fields.limit": 100,
"index": {
"number_of_shards": "3",

View File

@ -6,7 +6,7 @@
"template": {
"settings": {
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-applications",
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 6000,
"mapping.nested_fields.limit": 100,
"index": {
"number_of_shards": "3",

View File

@ -5,7 +5,7 @@
"order": 10,
"settings": {
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-cloud-services-aws",
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 6500,
"mapping.nested_fields.limit": 250,
"index": {
"number_of_shards": "3",

View File

@ -6,7 +6,7 @@
"template": {
"settings": {
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-cloud-services-aws",
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 6500,
"mapping.nested_fields.limit": 250,
"index": {
"number_of_shards": "3",

View File

@ -5,7 +5,7 @@
"order": 10,
"settings": {
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-cloud-services-azure",
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 6000,
"mapping.nested_fields.limit": 100,
"index": {
"number_of_shards": "3",

View File

@ -6,7 +6,7 @@
"template": {
"settings": {
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-cloud-services-azure",
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 6000,
"mapping.nested_fields.limit": 100,
"index": {
"number_of_shards": "3",

View File

@ -5,7 +5,7 @@
"order": 10,
"settings": {
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-cloud-services-gcp",
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 6000,
"mapping.nested_fields.limit": 100,
"index": {
"number_of_shards": "3",

View File

@ -6,7 +6,7 @@
"template": {
"settings": {
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-cloud-services-gcp",
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 6000,
"mapping.nested_fields.limit": 100,
"index": {
"number_of_shards": "3",

View File

@ -5,7 +5,7 @@
"order": 1,
"settings": {
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-cloud-services",
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 5500,
"mapping.nested_fields.limit": 100,
"index": {
"number_of_shards": "3",

View File

@ -6,7 +6,7 @@
"template": {
"settings": {
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-cloud-services",
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 5500,
"mapping.nested_fields.limit": 100,
"index": {
"number_of_shards": "3",

View File

@ -5,7 +5,7 @@
"order": 1,
"settings": {
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-network-activity",
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 7000,
"mapping.nested_fields.limit": 100,
"index": {
"number_of_shards": "3",

View File

@ -6,7 +6,7 @@
"template": {
"settings": {
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-network-activity",
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 7000,
"mapping.nested_fields.limit": 100,
"index": {
"number_of_shards": "3",

View File

@ -5,7 +5,7 @@
"order": 1,
"settings": {
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-other",
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 5500,
"mapping.nested_fields.limit": 100,
"index": {
"number_of_shards": "3",

View File

@ -6,7 +6,7 @@
"template": {
"settings": {
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-other",
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 5500,
"mapping.nested_fields.limit": 100,
"index": {
"number_of_shards": "3",

View File

@ -5,7 +5,7 @@
"order": 1,
"settings": {
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-security",
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 6000,
"mapping.nested_fields.limit": 100,
"index": {
"number_of_shards": "3",

View File

@ -6,7 +6,7 @@
"template": {
"settings": {
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-security",
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 6000,
"mapping.nested_fields.limit": 100,
"index": {
"number_of_shards": "3",

View File

@ -5,7 +5,7 @@
"order": 1,
"settings": {
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-system-activity",
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 6000,
"mapping.nested_fields.limit": 100,
"index": {
"number_of_shards": "3",

View File

@ -6,7 +6,7 @@
"template": {
"settings": {
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-system-activity",
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 6000,
"mapping.nested_fields.limit": 100,
"index": {
"number_of_shards": "3",

View File

@ -3,7 +3,7 @@
"order": "<priority>",
"settings": {
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-<integration-name>",
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 6400,
"mapping.nested_fields.limit": 100,
"index": {
"number_of_shards": "3",

View File

@ -4,7 +4,7 @@
"template": {
"settings": {
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-<integration-name>",
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 6400,
"mapping.nested_fields.limit": 100,
"index": {
"number_of_shards": "3",

View File

@ -19502,7 +19502,7 @@
"refresh_interval": "2s"
},
"mapping.nested_fields.limit": 100,
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 5500,
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-access-management"
}
}

View File

@ -21500,7 +21500,7 @@
"refresh_interval": "2s"
},
"mapping.nested_fields.limit": 100,
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 6000,
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-applications"
}
}

View File

@ -23159,7 +23159,7 @@
"refresh_interval": "2s"
},
"mapping.nested_fields.limit": 250,
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 6500,
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-cloud-services-aws"
}
}

View File

@ -21781,7 +21781,7 @@
"refresh_interval": "2s"
},
"mapping.nested_fields.limit": 100,
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 6000,
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-cloud-services-azure"
}
}

View File

@ -21052,7 +21052,7 @@
"refresh_interval": "2s"
},
"mapping.nested_fields.limit": 100,
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 6000,
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-cloud-services-gcp"
}
}

View File

@ -19658,7 +19658,7 @@
"refresh_interval": "2s"
},
"mapping.nested_fields.limit": 100,
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 5500,
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-cloud-services"
}
}

View File

@ -24992,7 +24992,7 @@
"refresh_interval": "2s"
},
"mapping.nested_fields.limit": 100,
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 7000,
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-network-activity"
}
}

View File

@ -19502,7 +19502,7 @@
"refresh_interval": "2s"
},
"mapping.nested_fields.limit": 100,
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 5500,
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-other"
}
}

View File

@ -21429,7 +21429,7 @@
"refresh_interval": "2s"
},
"mapping.nested_fields.limit": 100,
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 6000,
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-security"
}
}

View File

@ -21285,7 +21285,7 @@
"refresh_interval": "2s"
},
"mapping.nested_fields.limit": 100,
"mapping.total_fields.limit": 10000,
"mapping.total_fields.limit": 6000,
"plugins.index_state_management.rollover_alias": "wazuh-events-v5-system-activity"
}
}