Use stricter field limits for the WCS indices (#589)

* Use stricter field limits for the WCS indices

* Update the Wazuh Common Schema

* Bring down 'max_docvalue_fields_search'

* Add changelog entry

* Adjust limits

* Update the Wazuh Common Schema

* Add script to count and update total fiels and apply those changes

* Improve count_and_update_total_fields script and document it

* Add Changelog entry and fix a typo

* Set stricter limits for cloud-services indices

* Revert wrong value for max_docvalue_fields setting for the system activity index

---------

Signed-off-by: Jorge Sánchez <jorge.sanchez@wazuh.com>
Signed-off-by: Álex Ruiz Becerra <alejandro.ruiz.becerra@wazuh.com>
Co-authored-by: Wazuh Indexer Bot <github_devel_xdrsiem_indexer@wazuh.com>
Co-authored-by: Jorge Sanchez <jorge.sanchez@wazuh.com>

CHANGELOG.md

@@ -39,6 +39,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add browser-extensions and services inventory indices to documentation [(#574)](https://github.com/wazuh/wazuh-indexer-plugins/pull/574)
 - Update index templates with agent fields [(#578)](https://github.com/wazuh/wazuh-indexer-plugins/pull/578)
 - Rename indices from *-5.x-* to *-v5-* [(#597)](https://github.com/wazuh/wazuh-indexer-plugins/pull/597)
+- Use stricter field limits for the WCS indices [(#589)](https://github.com/wazuh/wazuh-indexer-plugins/pull/589)
 
 ### Deprecated
 -

ecs/generator/README.md

@@ -9,6 +9,7 @@ The generation of the Wazuh Common Schema is automated using a set of scripts an
 - [update_module_list.sh](./update_module_list.sh): generates the [module_list.txt](../module_list.txt) file, by scanning the [ecs/](..) folder. Run this script whenever a new module is added.
 - [images/Dockerfile](./images/Dockerfile): Dockerfile to build the image used for the schema generation. Clones the ECS repository, which contains the main tooling.
 - [images/generator.sh](./images/generator.sh): our actual schema generation script. It is executed inside the container. Contains post-processing steps to make the templates compatible with OpenSearch and to adapt them to our needs.
+- [count_and_update_total_fields.sh](./count_and_update_total_fields.sh): counts fields in a generated index template and proposes (or applies with --apply) an updated mapping.total_fields.limit rounded up to the next 500.
 
 ### Requirements
 
@@ -31,6 +32,11 @@ However, it can also be run locally. To do so, follow these steps.
     ./generate_schema.sh
     ```
 
+3. Update the number of total field of each module:
+    ``` bash
+    ./count_and_update_total_fields.sh all --apply
+    ```
+
 The scripts can be invoked from any location. When successful, all the generated files will be copied to their corresponding folders.
 
 A new `mappings` folder will be created inside the module's folder, containing all the generated files.

ecs/generator/count_and_update_total_fields.sh

@@ -0,0 +1,189 @@
+#!/usr/bin/env bash
+
+# Count fields in a generated index template and update mapping.total_fields.limit
+# Usage:
+#   ./count_and_update_total_fields.sh <module|all> [--apply]
+# If --apply is not passed the script runs in dry-run mode and prints proposed values.
+
+set -euo pipefail
+
+if [[ $# -lt 1 ]]; then
+  echo "Usage: $0 <module|all> [--apply]" >&2
+  exit 1
+fi
+
+ARG1="$1"
+APPLY=false
+if [[ ${2:-} == "--apply" ]] || [[ ${3:-} == "--apply" ]]; then
+  APPLY=true
+fi
+
+# If ARG1 is 'all' we will read module names from ecs/module_list.txt
+PROCESS_ALL=false
+if [[ "$ARG1" == "all" ]]; then
+  PROCESS_ALL=true
+fi
+
+# Navigate to repository root
+function navigate_to_project_root() {
+  local repo_root_marker=".github"
+  local script_path
+  script_path=$(dirname "$(realpath "$0")")
+
+  while [[ "$script_path" != "/" ]] && [[ ! -d "$script_path/$repo_root_marker" ]]; do
+    script_path=$(dirname "$script_path")
+  done
+
+  if [[ "$script_path" == "/" ]]; then
+    echo "Error: Unable to find the repository root." >&2
+    exit 1
+  fi
+
+  cd "$script_path"
+}
+
+navigate_to_project_root
+REPO_ROOT="$(pwd)"
+
+# process a single module name
+process_module() {
+  local MODULE="$1"
+
+  MODULE_LIST_FILE="$REPO_ROOT/ecs/module_list.txt"
+  INDEX_TEMPLATE_BASENAME="index-template-${MODULE}.json"
+  if [[ "$MODULE" == stateless-* ]]; then
+    short=${MODULE#stateless-}
+    INDEX_TEMPLATE_BASENAME="index-template-${short}.json"
+  fi
+  if [[ -f "$MODULE_LIST_FILE" ]]; then
+    match=$(grep -E "\[${MODULE//./\.}\]=" "$MODULE_LIST_FILE" || true)
+    if [[ -n "$match" ]]; then
+      rhs=$(echo "$match" | sed -E 's/^[^=]*=//')
+      rhs=$(echo "$rhs" | tr -d ' "')
+      if [[ -n "$rhs" ]]; then
+        INDEX_TEMPLATE_BASENAME="$rhs"
+      fi
+    fi
+  fi
+
+  INDEX_TEMPLATE_PATH="plugins/setup/src/main/resources/$INDEX_TEMPLATE_BASENAME"
+  TEMPLATE_SETTINGS="ecs/${MODULE}/fields/template-settings.json"
+  TEMPLATE_SETTINGS_LEGACY="ecs/${MODULE}/fields/template-settings-legacy.json"
+
+  if ! command -v jq &> /dev/null; then
+    echo "Error: 'jq' is required but not installed." >&2
+    exit 1
+  fi
+
+  if [[ -z "$MODULE" ]]; then
+    echo "Usage: $0 <module|all> [--apply]" >&2
+    return 1
+  fi
+
+  if [[ ! -f "$REPO_ROOT/$INDEX_TEMPLATE_PATH" ]]; then
+    echo "Warning: Index template not found at $INDEX_TEMPLATE_PATH" >&2
+    return 0
+  fi
+
+  # jq filter to count fields
+  JQ_FILTER='def count_fields: (keys_unsorted | length) + ( map( if type == "object" then (.properties | select(.) | count_fields) // 0 + (.fields | select(.) | count_fields) // 0 else 0 end ) | add ); .mappings.properties | count_fields'
+
+  TOTAL_FIELDS=$(jq -r "$JQ_FILTER" "$REPO_ROOT/$INDEX_TEMPLATE_PATH" 2> /tmp/jq_error.log) || {
+    echo "Error: Could not parse JSON or find .mappings.properties in $INDEX_TEMPLATE_PATH" >&2
+    cat /tmp/jq_error.log >&2 || true
+    rm -f /tmp/jq_error.log
+    return 1
+  }
+  rm -f /tmp/jq_error.log
+
+  # compute next multiple of 500
+  PROPOSED=$(( ( (TOTAL_FIELDS + 499) / 500 ) * 500 ))
+
+  cat <<EOF
+Module: $MODULE
+Index template: $INDEX_TEMPLATE_PATH
+Total fields: $TOTAL_FIELDS
+Proposed mapping.total_fields.limit: $PROPOSED
+EOF
+
+  if ! $APPLY; then
+    echo "Dry-run mode. To apply the change add --apply" >&2
+    return 0
+  fi
+
+  # Update JSON files in place using jq
+  update_file() {
+    local file="$1"
+    if [[ ! -f "$REPO_ROOT/$file" ]]; then
+      echo "Skipping missing file: $file" >&2
+      return
+    fi
+    if jq -e '.template? and .template.settings? and .template.settings["mapping.total_fields.limit"]' "$REPO_ROOT/$file" > /dev/null 2>&1; then
+      tmpfile=$(mktemp)
+      last_hex=$(tail -c1 "$REPO_ROOT/$file" 2> /dev/null | od -An -t x1 | tr -d ' \t\n' || true)
+      jq ".template.settings[\"mapping.total_fields.limit\"] = $PROPOSED" "$REPO_ROOT/$file" > "$tmpfile"
+      if [[ -n "$last_hex" && "$last_hex" != "0a" ]]; then
+        perl -0777 -pe 's/\n\z//' "$tmpfile" > "${tmpfile}.fix" && mv "${tmpfile}.fix" "$tmpfile"
+      fi
+      mv "$tmpfile" "$REPO_ROOT/$file"
+      echo "Updated $file -> $PROPOSED"
+    elif jq -e '.settings? and .settings["mapping.total_fields.limit"]' "$REPO_ROOT/$file" > /dev/null 2>&1; then
+      tmpfile=$(mktemp)
+      last_hex=$(tail -c1 "$REPO_ROOT/$file" 2> /dev/null | od -An -t x1 | tr -d ' \t\n' || true)
+      jq ".settings[\"mapping.total_fields.limit\"] = $PROPOSED" "$REPO_ROOT/$file" > "$tmpfile"
+      if [[ -n "$last_hex" && "$last_hex" != "0a" ]]; then
+        perl -0777 -pe 's/\n\z//' "$tmpfile" > "${tmpfile}.fix" && mv "${tmpfile}.fix" "$tmpfile"
+      fi
+      mv "$tmpfile" "$REPO_ROOT/$file"
+      echo "Updated $file -> $PROPOSED"
+    else
+      echo "No mapping.total_fields.limit key found in $file. Skipping." >&2
+    fi
+  }
+
+  update_file "$TEMPLATE_SETTINGS"
+  update_file "$TEMPLATE_SETTINGS_LEGACY"
+  update_file "$INDEX_TEMPLATE_PATH"
+
+  echo "Done. Files updated."
+}
+
+# If PROCESS_ALL, read module_list and process only stateless-* modules
+if $PROCESS_ALL; then
+  MODULE_LIST_FILE="$REPO_ROOT/ecs/module_list.txt"
+  if [[ ! -f "$MODULE_LIST_FILE" ]]; then
+    echo "Error: $MODULE_LIST_FILE not found" >&2
+    exit 1
+  fi
+
+  modules_block=$(awk 'BEGIN{inside=0} /module_to_file=\(/ {inside=1; next} inside && /^\)/ {exit} inside {print}' "$MODULE_LIST_FILE" || true)
+  if [[ -z "$modules_block" ]]; then
+    echo "Error: Could not find module_to_file(...) block in $MODULE_LIST_FILE" >&2
+    exit 1
+  fi
+
+  mapfile -t MODULES < <(echo "$modules_block" | grep -oP '\[\K[^\]]+(?=\])' || true)
+  # Keep only stateless- modules
+  filtered=()
+  for m in "${MODULES[@]:-}"; do
+    if [[ "$m" == stateless-* ]]; then
+      filtered+=("$m")
+    fi
+  done
+  MODULES=("${filtered[@]}")
+
+  if [[ ${#MODULES[@]} -eq 0 ]]; then
+    echo "No stateless-* modules found in $MODULE_LIST_FILE" >&2
+    exit 1
+  fi
+
+  for m in "${MODULES[@]}"; do
+    process_module "$m"
+  done
+  exit 0
+fi
+
+# Otherwise process the single module provided as ARG1
+MODULE="$ARG1"
+process_module "$MODULE" || exit $?
+

ecs/stateless-access-management/fields/template-settings-legacy.json

@@ -5,7 +5,7 @@
   "order": 1,
   "settings": {
     "plugins.index_state_management.rollover_alias": "wazuh-events-v5-access-management",
-    "mapping.total_fields.limit": 10000,
+    "mapping.total_fields.limit": 5500,
     "mapping.nested_fields.limit": 100,
     "index": {
       "number_of_shards": "3",

ecs/stateless-access-management/fields/template-settings.json

@@ -6,7 +6,7 @@
   "template": {
     "settings": {
       "plugins.index_state_management.rollover_alias": "wazuh-events-v5-access-management",
-      "mapping.total_fields.limit": 10000,
+      "mapping.total_fields.limit": 5500,
       "mapping.nested_fields.limit": 100,
       "index": {
         "number_of_shards": "3",

ecs/stateless-applications/fields/template-settings-legacy.json

@@ -5,7 +5,7 @@
   "order": 1,
   "settings": {
     "plugins.index_state_management.rollover_alias": "wazuh-events-v5-applications",
-    "mapping.total_fields.limit": 10000,
+    "mapping.total_fields.limit": 6000,
     "mapping.nested_fields.limit": 100,
     "index": {
       "number_of_shards": "3",

ecs/stateless-applications/fields/template-settings.json

@@ -6,7 +6,7 @@
   "template": {
     "settings": {
       "plugins.index_state_management.rollover_alias": "wazuh-events-v5-applications",
-      "mapping.total_fields.limit": 10000,
+      "mapping.total_fields.limit": 6000,
       "mapping.nested_fields.limit": 100,
       "index": {
         "number_of_shards": "3",

ecs/stateless-cloud-services-aws/fields/template-settings-legacy.json

@@ -5,7 +5,7 @@
   "order": 10,
   "settings": {
     "plugins.index_state_management.rollover_alias": "wazuh-events-v5-cloud-services-aws",
-    "mapping.total_fields.limit": 10000,
+    "mapping.total_fields.limit": 6500,
     "mapping.nested_fields.limit": 250,
     "index": {
       "number_of_shards": "3",

ecs/stateless-cloud-services-aws/fields/template-settings.json

@@ -6,7 +6,7 @@
   "template": {
     "settings": {
       "plugins.index_state_management.rollover_alias": "wazuh-events-v5-cloud-services-aws",
-      "mapping.total_fields.limit": 10000,
+      "mapping.total_fields.limit": 6500,
       "mapping.nested_fields.limit": 250,
       "index": {
         "number_of_shards": "3",

ecs/stateless-cloud-services-azure/fields/template-settings-legacy.json

@@ -5,7 +5,7 @@
   "order": 10,
   "settings": {
     "plugins.index_state_management.rollover_alias": "wazuh-events-v5-cloud-services-azure",
-    "mapping.total_fields.limit": 10000,
+    "mapping.total_fields.limit": 6000,
     "mapping.nested_fields.limit": 100,
     "index": {
       "number_of_shards": "3",

ecs/stateless-cloud-services-azure/fields/template-settings.json

@@ -6,7 +6,7 @@
   "template": {
     "settings": {
       "plugins.index_state_management.rollover_alias": "wazuh-events-v5-cloud-services-azure",
-      "mapping.total_fields.limit": 10000,
+      "mapping.total_fields.limit": 6000,
       "mapping.nested_fields.limit": 100,
       "index": {
         "number_of_shards": "3",

ecs/stateless-cloud-services-gcp/fields/template-settings-legacy.json

@@ -5,7 +5,7 @@
   "order": 10,
   "settings": {
     "plugins.index_state_management.rollover_alias": "wazuh-events-v5-cloud-services-gcp",
-    "mapping.total_fields.limit": 10000,
+    "mapping.total_fields.limit": 6000,
     "mapping.nested_fields.limit": 100,
     "index": {
       "number_of_shards": "3",

ecs/stateless-cloud-services-gcp/fields/template-settings.json

@@ -6,7 +6,7 @@
   "template": {
     "settings": {
       "plugins.index_state_management.rollover_alias": "wazuh-events-v5-cloud-services-gcp",
-      "mapping.total_fields.limit": 10000,
+      "mapping.total_fields.limit": 6000,
       "mapping.nested_fields.limit": 100,
       "index": {
         "number_of_shards": "3",

ecs/stateless-cloud-services/fields/template-settings-legacy.json

@@ -5,7 +5,7 @@
   "order": 1,
   "settings": {
     "plugins.index_state_management.rollover_alias": "wazuh-events-v5-cloud-services",
-    "mapping.total_fields.limit": 10000,
+    "mapping.total_fields.limit": 5500,
     "mapping.nested_fields.limit": 100,
     "index": {
       "number_of_shards": "3",

ecs/stateless-cloud-services/fields/template-settings.json

@@ -6,7 +6,7 @@
   "template": {
     "settings": {
       "plugins.index_state_management.rollover_alias": "wazuh-events-v5-cloud-services",
-      "mapping.total_fields.limit": 10000,
+      "mapping.total_fields.limit": 5500,
       "mapping.nested_fields.limit": 100,
       "index": {
         "number_of_shards": "3",

ecs/stateless-network-activity/fields/template-settings-legacy.json

@@ -5,7 +5,7 @@
   "order": 1,
   "settings": {
     "plugins.index_state_management.rollover_alias": "wazuh-events-v5-network-activity",
-    "mapping.total_fields.limit": 10000,
+    "mapping.total_fields.limit": 7000,
     "mapping.nested_fields.limit": 100,
     "index": {
       "number_of_shards": "3",

ecs/stateless-network-activity/fields/template-settings.json

@@ -6,7 +6,7 @@
   "template": {
     "settings": {
       "plugins.index_state_management.rollover_alias": "wazuh-events-v5-network-activity",
-      "mapping.total_fields.limit": 10000,
+      "mapping.total_fields.limit": 7000,
       "mapping.nested_fields.limit": 100,
       "index": {
         "number_of_shards": "3",

ecs/stateless-other/fields/template-settings-legacy.json

@@ -5,7 +5,7 @@
   "order": 1,
   "settings": {
     "plugins.index_state_management.rollover_alias": "wazuh-events-v5-other",
-    "mapping.total_fields.limit": 10000,
+    "mapping.total_fields.limit": 5500,
     "mapping.nested_fields.limit": 100,
     "index": {
       "number_of_shards": "3",

ecs/stateless-other/fields/template-settings.json

@@ -6,7 +6,7 @@
   "template": {
     "settings": {
       "plugins.index_state_management.rollover_alias": "wazuh-events-v5-other",
-      "mapping.total_fields.limit": 10000,
+      "mapping.total_fields.limit": 5500,
       "mapping.nested_fields.limit": 100,
       "index": {
         "number_of_shards": "3",

ecs/stateless-security/fields/template-settings-legacy.json

@@ -5,7 +5,7 @@
   "order": 1,
   "settings": {
     "plugins.index_state_management.rollover_alias": "wazuh-events-v5-security",
-    "mapping.total_fields.limit": 10000,
+    "mapping.total_fields.limit": 6000,
     "mapping.nested_fields.limit": 100,
     "index": {
       "number_of_shards": "3",

ecs/stateless-security/fields/template-settings.json

@@ -6,7 +6,7 @@
   "template": {
     "settings": {
       "plugins.index_state_management.rollover_alias": "wazuh-events-v5-security",
-      "mapping.total_fields.limit": 10000,
+      "mapping.total_fields.limit": 6000,
       "mapping.nested_fields.limit": 100,
       "index": {
         "number_of_shards": "3",

ecs/stateless-system-activity/fields/template-settings-legacy.json

@@ -5,7 +5,7 @@
   "order": 1,
   "settings": {
     "plugins.index_state_management.rollover_alias": "wazuh-events-v5-system-activity",
-    "mapping.total_fields.limit": 10000,
+    "mapping.total_fields.limit": 6000,
     "mapping.nested_fields.limit": 100,
     "index": {
       "number_of_shards": "3",

ecs/stateless-system-activity/fields/template-settings.json

@@ -6,7 +6,7 @@
   "template": {
     "settings": {
       "plugins.index_state_management.rollover_alias": "wazuh-events-v5-system-activity",
-      "mapping.total_fields.limit": 10000,
+      "mapping.total_fields.limit": 6000,
       "mapping.nested_fields.limit": 100,
       "index": {
         "number_of_shards": "3",

ecs/stateless-template/fields/template-settings-legacy.json

@@ -3,7 +3,7 @@
   "order": "<priority>",
   "settings": {
     "plugins.index_state_management.rollover_alias": "wazuh-events-v5-<integration-name>",
-    "mapping.total_fields.limit": 10000,
+    "mapping.total_fields.limit": 6400,
     "mapping.nested_fields.limit": 100,
     "index": {
       "number_of_shards": "3",

ecs/stateless-template/fields/template-settings.json

@@ -4,7 +4,7 @@
   "template": {
     "settings": {
       "plugins.index_state_management.rollover_alias": "wazuh-events-v5-<integration-name>",
-      "mapping.total_fields.limit": 10000,
+      "mapping.total_fields.limit": 6400,
       "mapping.nested_fields.limit": 100,
       "index": {
         "number_of_shards": "3",

plugins/setup/src/main/resources/index-template-access-management.json

@@ -19502,7 +19502,7 @@
       "refresh_interval": "2s"
     },
     "mapping.nested_fields.limit": 100,
-    "mapping.total_fields.limit": 10000,
+    "mapping.total_fields.limit": 5500,
     "plugins.index_state_management.rollover_alias": "wazuh-events-v5-access-management"
   }
 }

plugins/setup/src/main/resources/index-template-applications.json

@@ -21500,7 +21500,7 @@
       "refresh_interval": "2s"
     },
     "mapping.nested_fields.limit": 100,
-    "mapping.total_fields.limit": 10000,
+    "mapping.total_fields.limit": 6000,
     "plugins.index_state_management.rollover_alias": "wazuh-events-v5-applications"
   }
 }

plugins/setup/src/main/resources/index-template-cloud-services-aws.json

@@ -23159,7 +23159,7 @@
       "refresh_interval": "2s"
     },
     "mapping.nested_fields.limit": 250,
-    "mapping.total_fields.limit": 10000,
+    "mapping.total_fields.limit": 6500,
     "plugins.index_state_management.rollover_alias": "wazuh-events-v5-cloud-services-aws"
   }
 }

plugins/setup/src/main/resources/index-template-cloud-services-azure.json

@@ -21781,7 +21781,7 @@
       "refresh_interval": "2s"
     },
     "mapping.nested_fields.limit": 100,
-    "mapping.total_fields.limit": 10000,
+    "mapping.total_fields.limit": 6000,
     "plugins.index_state_management.rollover_alias": "wazuh-events-v5-cloud-services-azure"
   }
 }

plugins/setup/src/main/resources/index-template-cloud-services-gcp.json

@@ -21052,7 +21052,7 @@
       "refresh_interval": "2s"
     },
     "mapping.nested_fields.limit": 100,
-    "mapping.total_fields.limit": 10000,
+    "mapping.total_fields.limit": 6000,
     "plugins.index_state_management.rollover_alias": "wazuh-events-v5-cloud-services-gcp"
   }
 }

plugins/setup/src/main/resources/index-template-cloud-services.json

@@ -19658,7 +19658,7 @@
       "refresh_interval": "2s"
     },
     "mapping.nested_fields.limit": 100,
-    "mapping.total_fields.limit": 10000,
+    "mapping.total_fields.limit": 5500,
     "plugins.index_state_management.rollover_alias": "wazuh-events-v5-cloud-services"
   }
 }

plugins/setup/src/main/resources/index-template-network-activity.json

@@ -24992,7 +24992,7 @@
       "refresh_interval": "2s"
     },
     "mapping.nested_fields.limit": 100,
-    "mapping.total_fields.limit": 10000,
+    "mapping.total_fields.limit": 7000,
     "plugins.index_state_management.rollover_alias": "wazuh-events-v5-network-activity"
   }
 }

plugins/setup/src/main/resources/index-template-other.json

@@ -19502,7 +19502,7 @@
       "refresh_interval": "2s"
     },
     "mapping.nested_fields.limit": 100,
-    "mapping.total_fields.limit": 10000,
+    "mapping.total_fields.limit": 5500,
     "plugins.index_state_management.rollover_alias": "wazuh-events-v5-other"
   }
 }

plugins/setup/src/main/resources/index-template-security.json

@@ -21429,7 +21429,7 @@
       "refresh_interval": "2s"
     },
     "mapping.nested_fields.limit": 100,
-    "mapping.total_fields.limit": 10000,
+    "mapping.total_fields.limit": 6000,
     "plugins.index_state_management.rollover_alias": "wazuh-events-v5-security"
   }
 }

plugins/setup/src/main/resources/index-template-system-activity.json

@@ -21285,7 +21285,7 @@
       "refresh_interval": "2s"
     },
     "mapping.nested_fields.limit": 100,
-    "mapping.total_fields.limit": 10000,
+    "mapping.total_fields.limit": 6000,
     "plugins.index_state_management.rollover_alias": "wazuh-events-v5-system-activity"
   }
 }