From fed047b2972f804f3fbf6cd95409e208e2cf7146 Mon Sep 17 00:00:00 2001 From: Jesus Garcia Date: Fri, 14 Nov 2025 14:08:31 -0500 Subject: [PATCH] Adapt Wazuh log gathering playbooks to use consistent variable naming and improve directory management. Adapt workflows to new architecture Remove degree of dependency of inventory hostname --- .github/playbooks/gather_agent_logs.yml | 64 +++++++++++-------- .github/playbooks/gather_central_logs.yml | 62 ++++++++++++------ .github/workflows/aio.yml | 4 +- .github/workflows/distributed.yml | 15 ++--- .../tasks/config_files_setup.yml | 6 +- 5 files changed, 92 insertions(+), 59 deletions(-) diff --git a/.github/playbooks/gather_agent_logs.yml b/.github/playbooks/gather_agent_logs.yml index b4c2d539..82533434 100644 --- a/.github/playbooks/gather_agent_logs.yml +++ b/.github/playbooks/gather_agent_logs.yml @@ -22,16 +22,20 @@ - name: Linux | Set required facts for logs gathering (2/2) ansible.builtin.set_fact: - remote_log_file_path: "/tmp/{{ logs_prefix }}_{{ ansible_date_time.epoch }}" + remote_gathering_path: "/tmp/{{ logs_prefix }}_{{ ansible_date_time.epoch }}" - name: Linux | Remove existing Wazuh logs temporal directories ansible.builtin.file: - path: "{{ remote_log_file_path }}" + path: "{{ remote_gathering_path }}" state: absent + - name: Set fact for logs directory + ansible.builtin.set_fact: + remote_logs_path: "{{ remote_gathering_path }}/logs" + - name: Linux | Create temporal directories for Wazuh logs ansible.builtin.file: - path: "{{ remote_log_file_path }}" + path: "{{ remote_logs_path }}" state: directory mode: '0755' @@ -41,47 +45,55 @@ - name: Linux | Wazuh Agent | Fetching logs (1/3) changed_when: false ansible.builtin.shell: | - journalctl -u wazuh-agent > {{ remote_log_file_path }}/{{ logs_prefix }}_journalctl.log + journalctl -u wazuh-agent > {{ remote_logs_path }}/{{ logs_prefix }}_journalctl.log - name: Linux | Wazuh Agent | Fetching logs (2/3) ansible.builtin.find: paths: /var/ossec/logs file_type: file - register: finded_files + register: wazuh_agent_logs - name: Linux | Wazuh Agent | Fetching logs (3/3) ansible.builtin.copy: src: "{{ item.path }}" - dest: "{{ remote_log_file_path }}/{{ logs_prefix }}_{{ item.path | basename }}" + dest: "{{ remote_logs_path }}/{{ logs_prefix }}_{{ item.path | basename }}" remote_src: true - loop: "{{ lookup('ansible.builtin.vars', 'ansible_facts').finded_files.files }}" + loop: "{{ wazuh_agent_logs.files | default([]) }}" loop_control: label: "{{ item.path | basename }}" - when: finded_files.files is defined and (finded_files.matched | int) > 0 + when: wazuh_agent_logs.files is defined and (wazuh_agent_logs.matched | int) > 0 - name: Linux | Wazuh Agent | Fetching agent configuration ansible.builtin.copy: src: /var/ossec/etc/ossec.conf - dest: "{{ remote_log_file_path }}/{{ logs_prefix }}_ossec.conf" + dest: "{{ remote_logs_path }}/{{ logs_prefix }}_ossec.conf" remote_src: true mode: '0644' - name: Linux | Compress Wazuh logs - community.general.archive: - path: "{{ remote_log_file_path }}" - dest: "{{ remote_log_file_path }}/compressed_wazuh_agent_logs_{{ logs_prefix }}.tar.gz" - format: gz - mode: '0644' + block: + - name: Compress Wazuh logs (archive module) + community.general.archive: + path: "{{ remote_logs_path }}" + dest: "{{ remote_gathering_path }}/compressed_wazuh_agent_logs_{{ logs_prefix }}.tar.gz" + format: gz + mode: '0644' + rescue: + - name: Compress Wazuh logs (tar command) + ansible.builtin.shell: | + tar -czf {{ remote_gathering_path }}/compressed_wazuh_agent_logs_{{ logs_prefix }}.tar.gz -C {{ remote_logs_path }} . + args: + creates: "{{ remote_gathering_path }}/compressed_wazuh_agent_logs_{{ logs_prefix }}.tar.gz" - name: Linux | Copy Wazuh logs to local machine ansible.builtin.fetch: - src: "{{ remote_log_file_path }}/compressed_wazuh_agent_logs_{{ logs_prefix }}.tar.gz" + src: "{{ remote_gathering_path }}/compressed_wazuh_agent_logs_{{ logs_prefix }}.tar.gz" dest: "{{ local_log_file_path }}/" flat: true - name: Linux | Remove Wazuh logs temporal directory from remote machine ansible.builtin.file: - path: "{{ remote_log_file_path }}" + path: "{{ remote_gathering_path }}" state: absent - name: Gather Wazuh Agent logs from Windows hosts @@ -93,16 +105,16 @@ - name: Windows | Set required facts for logs gathering (2/2) ansible.builtin.set_fact: - remote_log_file_path: "C:\\Temp\\{{ logs_prefix }}_{{ ansible_date_time.epoch }}" + remote_gathering_path: "C:\\Temp\\{{ logs_prefix }}_{{ ansible_date_time.epoch }}" - name: Windows | Remove existing Wazuh logs temporal directories ansible.windows.win_file: - path: "{{ remote_log_file_path }}" + path: "{{ remote_gathering_path }}" state: absent - name: Windows | Create temporal directories for Wazuh logs ansible.windows.win_file: - path: "{{ remote_log_file_path }}" + path: "{{ remote_gathering_path }}" state: directory - name: Windows | Wazuh Agent @@ -110,7 +122,7 @@ block: - name: Windows | Wazuh Agent | Fetching logs (1/3) ansible.windows.win_shell: | - $LogPath = "{{ remote_log_file_path }}\\{{ logs_prefix }}_wevents.log"; + $LogPath = "{{ remote_gathering_path }}\\{{ logs_prefix }}_wevents.log"; Get-EventLog -LogName Application | Where-Object { $_.Source -eq 'Wazuh' } | Select-Object -ExpandProperty Message | @@ -125,7 +137,7 @@ - name: Windows | Wazuh Agent | Fetching logs (3/3) ansible.windows.win_copy: src: "{{ item.path }}" - dest: "{{ remote_log_file_path }}\\{{ logs_prefix }}_{{ item.path | basename }}" + dest: "{{ remote_gathering_path }}\\{{ logs_prefix }}_{{ item.path | basename }}" remote_src: true loop: "{{ wazuh_agent_logs.files }}" when: wazuh_agent_logs.files is defined and (wazuh_agent_logs.matched | int) > 0 @@ -133,22 +145,22 @@ - name: Windows | Wazuh Agent | Fetching agent configuration ansible.windows.win_copy: src: 'C:\\Program Files (x86)\\ossec-agent\\ossec.conf' - dest: "{{ remote_log_file_path }}\\{{ logs_prefix }}_ossec.conf" + dest: "{{ remote_gathering_path }}\\{{ logs_prefix }}_ossec.conf" remote_src: true - name: Windows | Compress Wazuh logs ansible.windows.win_shell: | - Compress-Archive -Path "{{ remote_log_file_path }}\\*" ` - -DestinationPath "{{ remote_log_file_path }}\\compressed_wazuh_agent_logs_{{ logs_prefix }}.zip" ` + Compress-Archive -Path "{{ remote_gathering_path }}\\*" ` + -DestinationPath "{{ remote_gathering_path }}\\compressed_wazuh_agent_logs_{{ logs_prefix }}.zip" ` -CompressionLevel Optimal - name: Windows | Copy Wazuh logs to local machine ansible.builtin.fetch: - src: "{{ remote_log_file_path }}\\compressed_wazuh_agent_logs_{{ logs_prefix }}.zip" + src: "{{ remote_gathering_path }}\\compressed_wazuh_agent_logs_{{ logs_prefix }}.zip" dest: "{{ local_log_file_path }}/" flat: true - name: Windows | Remove Wazuh logs from remote machine ansible.windows.win_file: - path: "{{ remote_log_file_path }}" + path: "{{ remote_gathering_path }}" state: absent diff --git a/.github/playbooks/gather_central_logs.yml b/.github/playbooks/gather_central_logs.yml index d877b84f..f5e223b3 100644 --- a/.github/playbooks/gather_central_logs.yml +++ b/.github/playbooks/gather_central_logs.yml @@ -5,22 +5,34 @@ strategy: free become: true tasks: - - name: Linux | Set required facts for logs gathering (1/2) + - name: Make sure local_log_file_path directory exists on local machine + delegate_to: localhost + become: false + ansible.builtin.file: + path: "{{ local_log_file_path }}" + state: directory + mode: '0755' + + - name: Set required facts for logs gathering (1/2) ansible.builtin.set_fact: logs_prefix: "{{ ansible_facts.distribution }}_{{ ansible_facts.distribution_major_version }}_{{ ansible_facts.architecture }}_{{ inventory_hostname }}" - - name: Linux | Set required facts for logs gathering (2/2) + - name: Set required facts for logs gathering (2/2) ansible.builtin.set_fact: - remote_log_file_path: "/tmp/wazuh_logs_{{ logs_prefix }}_{{ ansible_date_time.epoch }}" + remote_gathering_path: "/tmp/wazuh_logs_{{ logs_prefix }}_{{ ansible_date_time.epoch }}" - name: Remove existing Wazuh logs temporal directories ansible.builtin.file: - path: "{{ remote_log_file_path }}" + path: "{{ remote_gathering_path }}" state: absent + - name: Set fact for logs directory + ansible.builtin.set_fact: + remote_logs_path: "{{ remote_gathering_path }}/logs" + - name: Create temporal directories for Wazuh logs ansible.builtin.file: - path: "{{ remote_log_file_path }}/{{ item }}" + path: "{{ remote_logs_path }}/{{ item }}" state: directory mode: '0755' loop: @@ -44,7 +56,7 @@ - name: Wazuh Indexer | Fetching logs (1/3) changed_when: false ansible.builtin.shell: | - journalctl -u wazuh-indexer > {{ remote_log_file_path }}/wazuh-indexer/{{ logs_prefix }}_journalctl.log + journalctl -u wazuh-indexer > {{ remote_logs_path }}/wazuh-indexer/{{ logs_prefix }}_journalctl.log - name: Wazuh Indexer | Fetching logs (2/3) ansible.builtin.find: @@ -55,7 +67,7 @@ - name: Wazuh Indexer | Fetching logs (3/3) ansible.builtin.copy: src: "{{ item.path }}" - dest: "{{ remote_log_file_path }}/wazuh-indexer/{{ logs_prefix }}_{{ item.path | basename }}" + dest: "{{ remote_logs_path }}/wazuh-indexer/{{ logs_prefix }}_{{ item.path | basename }}" remote_src: true mode: '0644' loop: "{{ wi_indexer_logs.files | default([]) }}" @@ -66,7 +78,7 @@ - name: Wazuh Indexer | Fetching configuration ansible.builtin.copy: src: /etc/wazuh-indexer/opensearch.yml - dest: "{{ remote_log_file_path }}/wazuh-indexer/{{ logs_prefix }}_opensearch.yml" + dest: "{{ remote_logs_path }}/wazuh-indexer/{{ logs_prefix }}_opensearch.yml" remote_src: true mode: '0644' @@ -77,7 +89,7 @@ - name: Wazuh Server | Fetching logs (1/3) changed_when: false ansible.builtin.shell: | - journalctl -u wazuh-manager > {{ remote_log_file_path }}/wazuh-server/{{ logs_prefix }}_journalctl.log + journalctl -u wazuh-manager > {{ remote_logs_path }}/wazuh-server/{{ logs_prefix }}_journalctl.log - name: Wazuh Server | Fetching logs (2/3) ansible.builtin.find: @@ -88,7 +100,7 @@ - name: Wazuh Server | Fetching logs (3/3) ansible.builtin.copy: src: "{{ item.path }}" - dest: "{{ remote_log_file_path }}/wazuh-server/{{ logs_prefix }}_{{ item.path | basename }}" + dest: "{{ remote_logs_path }}/wazuh-server/{{ logs_prefix }}_{{ item.path | basename }}" remote_src: true mode: '0644' loop: "{{ wazuh_server_logs.files | default([]) }}" @@ -99,7 +111,7 @@ - name: Wazuh Server | Fetching configuration ansible.builtin.copy: src: /var/ossec/etc/ossec.conf - dest: "{{ remote_log_file_path }}/wazuh-server/{{ logs_prefix }}_ossec.conf" + dest: "{{ remote_logs_path }}/wazuh-server/{{ logs_prefix }}_ossec.conf" remote_src: true mode: '0644' @@ -110,38 +122,46 @@ - name: Wazuh Dashboard | Fetching logs changed_when: false ansible.builtin.shell: | - journalctl -u wazuh-dashboard > {{ remote_log_file_path }}/wazuh-dashboard/{{ logs_prefix }}_journalctl.log + journalctl -u wazuh-dashboard > {{ remote_logs_path }}/wazuh-dashboard/{{ logs_prefix }}_journalctl.log - name: Wazuh Dashboard | Fetching configuration (1/2) ansible.builtin.copy: src: /etc/wazuh-dashboard/opensearch_dashboards.yml - dest: "{{ remote_log_file_path }}/wazuh-dashboard/{{ logs_prefix }}_opensearch_dashboards.yml" + dest: "{{ remote_logs_path }}/wazuh-dashboard/{{ logs_prefix }}_opensearch_dashboards.yml" remote_src: true mode: '0644' - name: Wazuh Dashboard | Fetching configuration (2/2) ansible.builtin.copy: src: /usr/share/wazuh-dashboard/data/wazuh/config/wazuh.yml - dest: "{{ remote_log_file_path }}/wazuh-dashboard/{{ logs_prefix }}_wazuh.yml" + dest: "{{ remote_logs_path }}/wazuh-dashboard/{{ logs_prefix }}_wazuh.yml" remote_src: true mode: '0644' - name: Compress Wazuh logs - community.general.archive: - path: "{{ remote_log_file_path }}/*" - dest: "{{ remote_log_file_path }}/compressed_wazuh_logs_{{ logs_prefix }}.tar.gz" - format: gz - mode: '0644' + block: + - name: Compress Wazuh logs (archive module) + community.general.archive: + path: "{{ remote_logs_path }}/*" + dest: "{{ remote_gathering_path }}/compressed_wazuh_logs_{{ logs_prefix }}.tar.gz" + format: gz + mode: '0644' + rescue: + - name: Compress Wazuh logs (tar command) + ansible.builtin.shell: | + tar -czf {{ remote_gathering_path }}/compressed_wazuh_logs_{{ logs_prefix }}.tar.gz -C {{ remote_logs_path }} . + args: + creates: "{{ remote_gathering_path }}/compressed_wazuh_logs_{{ logs_prefix }}.tar.gz" - name: Copy Wazuh logs to local machine ansible.builtin.fetch: - src: "{{ remote_log_file_path }}/compressed_wazuh_logs_{{ logs_prefix }}.tar.gz" + src: "{{ remote_gathering_path }}/compressed_wazuh_logs_{{ logs_prefix }}.tar.gz" dest: "{{ local_log_file_path }}/compressed_individual_logs/" flat: true - name: Remove Wazuh logs temporal directory from remote machine ansible.builtin.file: - path: "{{ remote_log_file_path }}" + path: "{{ remote_gathering_path }}" state: absent - name: Local actions diff --git a/.github/workflows/aio.yml b/.github/workflows/aio.yml index 49e34de9..2b407d3d 100644 --- a/.github/workflows/aio.yml +++ b/.github/workflows/aio.yml @@ -242,6 +242,8 @@ jobs: # Append the processed line to the output file echo "$presigned_url_line" >> "$PRESIGNED_OUTPUT_FILE" else + echo "$line" >> "$PRESIGNED_OUTPUT_FILE" + echo "Skipping line for presigning (no S3 URI found):" echo "$line" fi @@ -454,7 +456,7 @@ jobs: ${{ needs.setup-runner.outputs.VERBOSITY }} - name: Ansible Playbook run Wazuh agents run: | - ansible-playbook wazuh-ansible/wazuh-agent.yml \ + ansible-playbook wazuh-ansible/wazuh-agent.yml --extra-vars "source=custom" \ -i $ALLOCATOR_PATH/inventory_agents.ini \ -l all \ ${{ needs.setup-runner.outputs.VERBOSITY }} diff --git a/.github/workflows/distributed.yml b/.github/workflows/distributed.yml index f088ce04..1ed7f32e 100644 --- a/.github/workflows/distributed.yml +++ b/.github/workflows/distributed.yml @@ -47,7 +47,7 @@ env: ALLOCATOR_PATH: "/tmp/wazuh-ansible/allocator_instance" ARTIFACT_URLS_FILE_TEMP: "/tmp/wazuh-ansible/artifact_urls.yml" PRESIGNED_OUTPUT_FILE: "/tmp/wazuh-ansible/artifact_urls_presigned.yml" - INSTANCE_NAMES: "wi1 wi2 wi3 dashboard manager worker balancer" + INSTANCE_NAMES: "wi1 wi2 wi3 dashboard manager worker" permissions: id-token: write # JWT contents: read # actions/checkout @@ -244,6 +244,8 @@ jobs: # Append the processed line to the output file echo "$presigned_url_line" >> "$PRESIGNED_OUTPUT_FILE" else + echo "$line" >> "$PRESIGNED_OUTPUT_FILE" + echo "Skipping line for presigning (no S3 URI found):" echo "$line" fi @@ -347,7 +349,7 @@ jobs: instance_names=($INSTANCE_NAMES) inventory_file="$ALLOCATOR_PATH/inventory_all" ssh_key_path=${{ needs.setup-runner.outputs.PRIVATE_KEY_PATH }} - load_balancer_private_ip="" + manager_private_ip="" # Create all instances in parallel for i in ${!instance_names[@]}; do @@ -395,11 +397,9 @@ jobs: echo "dashboard ansible_host=$ansible_host private_ip=$private_ip ansible_ssh_user=$ansible_user" >> $inventory_file elif [[ $i -eq 4 ]]; then echo "manager ansible_host=$ansible_host private_ip=$private_ip ansible_ssh_user=$ansible_user" >> $inventory_file + manager_private_ip=$private_ip elif [[ $i -eq 5 ]]; then echo "worker ansible_host=$ansible_host private_ip=$private_ip ansible_ssh_user=$ansible_user" >> $inventory_file - elif [[ $i -eq 6 ]]; then - echo "balancer ansible_host=$ansible_host private_ip=$private_ip ansible_ssh_user=$ansible_user" >> $inventory_file - load_balancer_private_ip="$private_ip" fi done @@ -416,8 +416,7 @@ jobs: echo "ansible_ssh_private_key_file=${ssh_key_path}" >> $inventory_file # Set the Wazuh server IP in the Ansible playbook - echo "Load balancer private IP: $load_balancer_private_ip" - sed -i "s||$load_balancer_private_ip|g" wazuh-ansible/wazuh-agent.yml + sed -i "s||$manager_private_ip|g" wazuh-ansible/wazuh-agent.yml - name: Allocate Agent instances run: | # Define variables @@ -500,7 +499,7 @@ jobs: ${{ needs.setup-runner.outputs.VERBOSITY }} - name: Ansible Playbook run Wazuh agents run: | - ansible-playbook wazuh-ansible/wazuh-agent.yml \ + ansible-playbook wazuh-ansible/wazuh-agent.yml --extra-vars "source=custom" \ -i $ALLOCATOR_PATH/inventory_agents.ini \ -l all \ ${{ needs.setup-runner.outputs.VERBOSITY }} diff --git a/roles/wazuh-indexer/tasks/config_files_setup.yml b/roles/wazuh-indexer/tasks/config_files_setup.yml index da98fff8..7491022f 100644 --- a/roles/wazuh-indexer/tasks/config_files_setup.yml +++ b/roles/wazuh-indexer/tasks/config_files_setup.yml @@ -195,8 +195,8 @@ group: wazuh-indexer mode: "0400" with_items: - - { src: "{{ instances[inventory_hostname].name }}-key.pem", dest: "indexer-key.pem" } - - { src: "{{ instances[inventory_hostname].name }}.pem", dest: "indexer.pem" } + - { src: "{{ node_name }}-key.pem", dest: "indexer-key.pem" } + - { src: "{{ node_name }}.pem", dest: "indexer.pem" } when: - generate_certs @@ -212,7 +212,7 @@ ansible.builtin.lineinfile: path: /etc/wazuh-indexer/opensearch.yml regexp: '^node\.name:' - line: 'node.name: "{{ instances[inventory_hostname].name }}"' + line: 'node.name: "{{ node_name }}"' - name: OpenSearch Config | Configure cluster.initial_cluster_manager_nodes block: