From 26c006f6bf1b4518e3cd7efe7e1b8132f8fd0ff5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20S=C3=A1nchez?= Date: Tue, 9 Dec 2025 18:08:41 +0100 Subject: [PATCH] Add scheduled content update (#682) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Delete old code * Initial version of Offset based update * Add Post Update logic to allow testing * Optimize the code * Add missing JavaDocs and upgrades in the code * Delete old code in PluginSettings and add new settings * Add unit tests * Modify mappings to make them dynamic and allow more date formats * Add Changelog entry * Update documentation * Changes from code review * Delete extra ifs * Add TODOs for future work * Fix broken link * Apply changes from code review Co-authored-by: Kevin Ledesma Signed-off-by: Jorge Sánchez * Add missing 'this' Change default catalog sync interval to 60 minutes --------- Signed-off-by: Jorge Sánchez Co-authored-by: Kevin Ledesma Co-authored-by: Alex Ruiz --- CHANGELOG.md | 1 + docs/dev/plugins/content-manager.md | 127 +++-- .../modules/content-manager/architecture.md | 39 ++ integrations/amazon-security-lake/README.md | 1 - .../contentmanager/ContentManagerPlugin.java | 111 +++- .../contentmanager/client/CTIClient.java | 394 ------------- .../contentmanager/client/HttpClient.java | 182 ------ .../cti/catalog/CtiCatalog.java | 20 - .../cti/catalog/client/ApiClient.java | 37 +- .../cti/catalog/index/ConsumersIndex.java | 152 +---- .../cti/catalog/index/ContentIndex.java | 527 ++++++++---------- .../cti/catalog/model/Changes.java | 97 ++-- .../cti/catalog/model/ConsumerInfo.java | 234 -------- .../cti/catalog/model/Offset.java | 250 +++------ .../cti/catalog/model/Operation.java | 108 ++-- .../catalog/service/ConsumerServiceImpl.java | 4 - .../cti/catalog/service/ContentUpdater.java | 167 ------ .../cti/catalog/service/UpdateService.java | 16 + .../catalog/service/UpdateServiceImpl.java | 201 +++++++ .../cti/catalog/utils/JsonPatch.java | 70 ++- .../jobscheduler/jobs/CatalogSyncJob.java | 74 ++- .../RestDeleteSubscriptionAction.java | 12 +- .../services/RestGetSubscriptionAction.java | 12 +- .../services/RestPostSubscriptionAction.java | 12 +- .../rest/services/RestPostUpdateAction.java | 27 +- .../settings/PluginSettings.java | 339 +++-------- .../contentmanager/utils/ClusterInfo.java | 34 -- .../contentmanager/utils/Privileged.java | 49 -- .../utils/VisibleForTesting.java | 35 -- .../contentmanager/utils/XContentUtils.java | 80 --- .../cti-decoders-integrations-mappings.json | 7 +- .../mappings/cti-decoders-mappings.json | 2 +- .../mappings/cti-kvdbs-mappings.json | 5 +- .../cti-rules-integrations-mappings.json | 5 +- .../mappings/cti-rules-mappings.json | 8 +- .../contentmanager/client/CTIClientTests.java | 379 ------------- .../client/HttpClientTests.java | 127 ----- .../catalog/index/ConsumersIndexTests.java | 177 ++++++ .../cti/catalog/index/ContentIndexTests.java | 296 ++++++---- .../cti/catalog/service/ContentUpdaterIT.java | 316 ----------- .../catalog/service/ContentUpdaterTests.java | 179 ------ .../service/SnapshotServiceImplTests.java | 97 ++-- .../service/UpdateServiceImplTests.java | 289 ++++++++++ .../rest/RestPostUpdateActionTests.java | 51 +- 44 files changed, 1829 insertions(+), 3521 deletions(-) delete mode 100644 plugins/content-manager/src/main/java/com/wazuh/contentmanager/client/CTIClient.java delete mode 100644 plugins/content-manager/src/main/java/com/wazuh/contentmanager/client/HttpClient.java delete mode 100644 plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/CtiCatalog.java delete mode 100644 plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/model/ConsumerInfo.java delete mode 100644 plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/service/ContentUpdater.java create mode 100644 plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/service/UpdateService.java create mode 100644 plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/service/UpdateServiceImpl.java delete mode 100644 plugins/content-manager/src/main/java/com/wazuh/contentmanager/utils/Privileged.java delete mode 100644 plugins/content-manager/src/main/java/com/wazuh/contentmanager/utils/VisibleForTesting.java delete mode 100644 plugins/content-manager/src/main/java/com/wazuh/contentmanager/utils/XContentUtils.java delete mode 100644 plugins/content-manager/src/test/java/com/wazuh/contentmanager/client/CTIClientTests.java delete mode 100644 plugins/content-manager/src/test/java/com/wazuh/contentmanager/client/HttpClientTests.java create mode 100644 plugins/content-manager/src/test/java/com/wazuh/contentmanager/cti/catalog/index/ConsumersIndexTests.java delete mode 100644 plugins/content-manager/src/test/java/com/wazuh/contentmanager/cti/catalog/service/ContentUpdaterIT.java delete mode 100644 plugins/content-manager/src/test/java/com/wazuh/contentmanager/cti/catalog/service/ContentUpdaterTests.java create mode 100644 plugins/content-manager/src/test/java/com/wazuh/contentmanager/cti/catalog/service/UpdateServiceImplTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index e8a0151d..06cc2b57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Initialize consumers metadata index on start [(#668)](https://github.com/wazuh/wazuh-indexer-plugins/pull/668) - Add job scheduler basic logic [(#671)](https://github.com/wazuh/wazuh-indexer-plugins/pull/671) - Init content from snapshot [(#670)](https://github.com/wazuh/wazuh-indexer-plugins/pull/670) +- Add scheduled content update [(#682)](https://github.com/wazuh/wazuh-indexer-plugins/pull/682) ### Dependencies - Upgrade to Gradle 8.14.3 [(#649)](https://github.com/wazuh/wazuh-indexer-plugins/pull/649) diff --git a/docs/dev/plugins/content-manager.md b/docs/dev/plugins/content-manager.md index cdf90c3f..1a5429c5 100644 --- a/docs/dev/plugins/content-manager.md +++ b/docs/dev/plugins/content-manager.md @@ -7,14 +7,17 @@ This document describes how to extend and configure the Wazuh Indexer Content Ma ## 📋 Overview The Content Manager plugin handles: -- **Content synchronization** from the Wazuh CTI API -- **Snapshot initialization** for a zip file -- **Incremental updates** using offset-based change tracking +- **Authentication** Manages subscriptions and tokens with the CTI Console. +- **Job Scheduling** Periodically checks for updates using the OpenSearch Job Scheduler. +- **Content Synchronization** Keeps local indices in sync with the Wazuh CTI Catalog. +- **Snapshot Initialization** Downloads and indexes full content via zip snapshots. +- **Incremental Updates** Applies JSON Patch operations based on offsets. - **Context management** to maintain synchronization state -The plugin manages two main indices: -- `wazuh-ruleset`: Contains the actual security content (rules, decoders, etc.) +The plugin manages several indices: - `.cti-consumers`: Stores consumer information and synchronization state +- `.wazuh-content-manager-jobs`: Stores job scheduler metadata. +- Content Indices: Indices for specific content types following the naming `.--`. --- @@ -25,41 +28,53 @@ The plugin manages two main indices: #### 1. **ContentManagerPlugin** Main class located at: `/plugins/content-manager/src/main/java/com/wazuh/contentmanager/ContentManagerPlugin.java` -This is the entry point of the plugin. +This is the entry point of the plugin: +- Registers REST handlers for subscription and update management. +- Initializes the `CatalogSyncJob` and schedules it via the OpenSearch Job Scheduler. +- Initializes the `CtiConsole` for authentication management. -#### 2. **ContentIndex** -Located at: `/plugins/content-manager/src/main/java/com/wazuh/contentmanager/index/ContentIndex.java` +#### 2. **CatalogSyncJob** +Located at: `/plugins/content-manager/src/main/java/com/wazuh/contentmanager/jobscheduler/jobs/CatalogSyncJob.java` -Manages operations on the `wazuh-ruleset` index: -- Bulk indexing operations -- Document patching (add, update, delete) -- Query and retrieval operations +This class acts as the orchestrator (`JobExecutor`). It is responsible for: +- Executing the content synchronization logic +- Managing concurrency using semaphores to prevent overlapping jobs. +- Determining whether to trigger a Snapshot Initialization or an Incremental Update based on consumer offsets. -#### 3. **ConsumersIndex** -Located at: `/plugins/content-manager/src/main/java/com/wazuh/contentmanager/index/ConsumersIndex.java` +#### 3. **Services** +The logic is split into specialized services: -Manages the `.cti-consumers` index which stores: -- Consumer name -- Local offset (last successfully applied change) -- Remote offset (last available offset from the CTI API) -- Snapshot link from where the index was initialized +##### 3.1 **ConsumerService** +Located at: `/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/service/ConsumerServiceImpl.java` -#### 4. **ContentUpdater** -Located at: `/plugins/content-manager/src/main/java/com/wazuh/contentmanager/updater/ContentUpdater.java` +Retrieves `LocalConsumer` state from `.cti-consumers` and `RemoteConsumer` state from the CTI API. -Orchestrates the update process by: -- Fetching changes from the CTI API -- Applying changes incrementally -- Updating offset information -- Handling update failures +##### 3.2 **SnapshotService** +Located at: `/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/service/SnapshotServiceImpl.java` -#### 5. **SnapshotManager** -Located at: `/plugins/content-manager/src/main/java/com/wazuh/contentmanager/utils/SnapshotManager.java` +Handles downloading zip snapshots, unzipping, parsing JSON files, and bulk indexing content when a consumer is new or reset. -Handles initial content bootstrapping: -- Downloads snapshots from the CTI API -- Decompresses and indexes snapshot content -- Triggers after initialization or on offset reset +##### 3.3 **UpdateService** +Located at: `/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/service/UpdateServiceImpl.java` + +Fetches specific changes (offsets) from the CTI API and applies them using JSON Patch (`Operation` class). + +##### 3.4 **AuthService** +Located at: `/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/console/service/AuthServiceImpl.java` + +Manages the exchange of device codes for permanent access tokens. + +#### 4. **Indices Management** + +##### 4.1 **ConsumersIndex** +Located at: `/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/index/ConsumersIndex.java` + +Wraps operations for the `.cti-consumers` index. + +##### 4.2 **ContentIndex** +Located at: `/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/index/ContentIndex.java` + +Manages operations for content indices. --- @@ -67,6 +82,14 @@ Handles initial content bootstrapping: The plugin is configured through the `PluginSettings` class. Settings can be defined in `opensearch.yml`: +| Setting | Default | Description | +|-----------------------------------------|------------------------------------|------------------------------------------------------------------------------| +| `content_manager.cti.api` | `https://cti-pre.wazuh.com/api/v1` | Base URL for the Wazuh CTI API. | +| `content_manager.catalog.sync_interval` | `60` | Interval (in minutes) for the periodic synchronization job. | +| `content_manager.max_items_per_bulk` | `25` | Maximum number of documents per bulk request during snapshot initialization. | +| `content_manager.max_concurrent_bulks` | `5` | Maximum number of concurrent bulk requests. | +| `content_manager.client.timeout` | `10` | Timeout (in seconds) for HTTP and Indexing operations. | + ## 🔄 How Content Synchronization Works @@ -92,8 +115,37 @@ The update process follows these steps: ### 3. **Error Handling** -- **Recoverable errors**: Updates local_offset and retries later -- **Critical failures**: Resets local_offset to 0, triggering snapshot re-initialization +Resets local_offset to 0, triggering snapshot re-initialization + +## 📡 REST API + +### Subscription Management + +#### GET /subscription + +Retrieves the current subscription token. + +`GET /_plugins/content-manager/subscription` + +#### POST /subscription + +Creates or updates a subscription. + +`POST /_plugins/content-manager/subscription { "device_code": "...", "client_id": "...", "expires_in": 3600, "interval": 5 }` + +#### DELETE /subscription + +Deletes the current token/subscription. + +`DELETE /_plugins/content-manager/subscription` + +### Update Trigger + +#### POST /update + +Manually triggers the `CatalogSyncJob`. + +`POST /_plugins/content-manager/update` --- @@ -113,7 +165,7 @@ GET /.cti-consumers/_search ### Check Content Index ```bash -GET /wazuh-ruleset/_search +GET /.cti-rules/_search { "size": 10 } @@ -121,10 +173,10 @@ GET /wazuh-ruleset/_search ### Monitor Plugin Logs -Look for entries from `ContentManagerPlugin`, `ContentUpdater`, and `SnapshotManager` in the OpenSearch logs. +Look for entries from `ContentManagerPlugin`, `CatalogSyncJob`, `SnapshotServiceImpl` and `UpdateServiceImpl` in the OpenSearch logs. ```bash -tail -f logs/opensearch.log | grep -E "ContentManager|ContentUpdater|SnapshotManager" +tail -f logs/opensearch.log | grep -E "ContentManager|CatalogSyncJob|SnapshotServiceImpl|UpdateServiceImpl" ``` --- @@ -134,7 +186,6 @@ tail -f logs/opensearch.log | grep -E "ContentManager|ContentUpdater|SnapshotMan - The plugin only runs on **cluster manager nodes** - CTI API must be accessible for content synchronization - Offset-based synchronization ensures no content is missed -- Snapshot initialization provides a fast bootstrap mechanism --- diff --git a/docs/ref/modules/content-manager/architecture.md b/docs/ref/modules/content-manager/architecture.md index a2dc5cd6..06b9df17 100644 --- a/docs/ref/modules/content-manager/architecture.md +++ b/docs/ref/modules/content-manager/architecture.md @@ -1,2 +1,41 @@ # Architecture +The Content Manager plugin operates within the Wazuh Indexer environment. It is composed of several key components that handle REST API requests, background job scheduling, and content synchronization logic. + +## High-Level Components + +### 1. REST Layer +The plugin exposes a set of REST endpoints under `/_plugins/content-manager/` to manage subscriptions and trigger updates. These handlers interact with the `CtiConsole` and `CatalogSyncJob` to perform operations. + +### 2. CTI Console +The `CtiConsole` acts as the authentication manager. It handles the storage and retrieval of authentication tokens required to communicate with the remote Wazuh CTI Console API. + +### 3. Job Scheduler & Sync Job +The plugin implements the `JobSchedulerExtension` to register the `CatalogSyncJob`. This job runs periodically (configured via `content_manager.catalog.sync_interval`) to synchronize content. It manages synchronization for different contexts, such as `rules` and `decoders`. + +## Synchronization Services + +The core logic is divided into three services: + +* **Consumer Service (`ConsumerServiceImpl`)**: + * Manages the state of "Consumers" (entities that consume content, e.g., a Rules consumer). + * Compares the local state (stored in the `.cti-consumers` index) with the remote state from the CTI API. + * Decides whether to perform a Snapshot Initialization or a Differential Update. + +* **Snapshot Service (`SnapshotServiceImpl`)**: + * Used when a consumer is new or empty. + * Downloads a full ZIP snapshot from the CTI provider. + * Extracts the content and indexes it into specific indices (e.g., `.cti-rules`). + * Performs data enrichment, such as converting JSON payloads to YAML for decoders. + +* **Update Service (`UpdateServiceImpl`)**: + * Used when the local content is behind the remote content. + * Fetches a list of changes based on the current offset. + * Applies operations (CREATE, UPDATE, DELETE) to the content indices. + * Updates the consumer offset upon success. + +## Data Persistence + +The plugin uses system indices to store data: +* **`.cti-consumers`**: Stores metadata about the synchronization state (offsets, snapshot links) for each consumer. +* **Content Indices**: Stores the actual CTI content. \ No newline at end of file diff --git a/integrations/amazon-security-lake/README.md b/integrations/amazon-security-lake/README.md index 83f26bb9..92c2b50c 100644 --- a/integrations/amazon-security-lake/README.md +++ b/integrations/amazon-security-lake/README.md @@ -43,7 +43,6 @@ Wazuh uses rules to monitor the events and logs in your network to detect securi - https://documentation.wazuh.com/current/user-manual/ruleset/getting-started.html#github-repository - https://github.com/wazuh/wazuh/tree/main/ruleset -- https://github.com/wazuh/wazuh/blob/main/extensions/elasticsearch/7.x/wazuh-template.json ### Wazuh Security Events to Amazon Security Lake diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/ContentManagerPlugin.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/ContentManagerPlugin.java index d3d28ca0..61e60bd9 100644 --- a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/ContentManagerPlugin.java +++ b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/ContentManagerPlugin.java @@ -17,7 +17,6 @@ package com.wazuh.contentmanager; import com.wazuh.contentmanager.cti.catalog.index.ConsumersIndex; -import com.wazuh.contentmanager.cti.catalog.index.ContentIndex; import com.wazuh.contentmanager.cti.console.CtiConsole; import com.wazuh.contentmanager.jobscheduler.ContentJobParameter; import com.wazuh.contentmanager.jobscheduler.ContentJobRunner; @@ -73,12 +72,25 @@ public class ContentManagerPlugin extends Plugin implements ClusterPlugin, JobSc private CtiConsole ctiConsole; private Client client; private Environment environment; + private CatalogSyncJob catalogSyncJob; - // Rest API endpoints - public static final String PLUGINS_BASE_URI = "/_plugins/content-manager"; - public static final String SUBSCRIPTION_URI = PLUGINS_BASE_URI + "/subscription"; - public static final String UPDATE_URI = PLUGINS_BASE_URI + "/update"; - + /** + * Initializes the plugin components, including the CTI console, consumer index helpers, + * and the catalog synchronization job. + * + * @param client The OpenSearch client. + * @param clusterService The cluster service for managing cluster state. + * @param threadPool The thread pool for executing asynchronous tasks. + * @param resourceWatcherService Service for watching resource changes. + * @param scriptService Service for executing scripts. + * @param xContentRegistry Registry for XContent parsers. + * @param environment The node environment settings. + * @param nodeEnvironment The node environment information. + * @param namedWriteableRegistry Registry for named writeables. + * @param indexNameExpressionResolver Resolver for index name expressions. + * @param repositoriesServiceSupplier Supplier for the repositories service. + * @return A collection of constructed components (empty in this implementation as components are stored internally). + */ @Override public Collection createComponents( Client client, @@ -92,7 +104,7 @@ public class ContentManagerPlugin extends Plugin implements ClusterPlugin, JobSc NamedWriteableRegistry namedWriteableRegistry, IndexNameExpressionResolver indexNameExpressionResolver, Supplier repositoriesServiceSupplier) { - PluginSettings.getInstance(environment.settings(), clusterService); + PluginSettings.getInstance(environment.settings()); this.client = client; this.threadPool = threadPool; this.environment = environment; @@ -101,18 +113,21 @@ public class ContentManagerPlugin extends Plugin implements ClusterPlugin, JobSc // Content Manager 5.0 this.ctiConsole = new CtiConsole(); ContentJobRunner runner = ContentJobRunner.getInstance(); + + // Initialize CatalogSyncJob + this.catalogSyncJob = new CatalogSyncJob(this.client, this.consumersIndex, this.environment, this.threadPool); + // Register Executors - runner.registerExecutor(CatalogSyncJob.JOB_TYPE, new CatalogSyncJob(client, consumersIndex, environment, threadPool)); + runner.registerExecutor(CatalogSyncJob.JOB_TYPE, this.catalogSyncJob); return Collections.emptyList(); } /** - * The initialization requires the existence of the {@link ContentIndex#INDEX_NAME} index. For - * this reason, we use a ClusterStateListener to listen for the creation of this index by the - * "setup" plugin, to then proceed with the initialization. + * Triggers the internal {@link #start()} method if the current node is a Cluster Manager to + * initialize indices. It also ensures the periodic catalog sync job is scheduled. * - * @param localNode local Node info + * @param localNode The local node discovery information. */ @Override public void onNodeStarted(DiscoveryNode localNode) { @@ -125,6 +140,18 @@ public class ContentManagerPlugin extends Plugin implements ClusterPlugin, JobSc this.scheduleCatalogSyncJob(); } + /** + * Registers the REST handlers for the Content Manager API. + * + * @param settings The node settings. + * @param restController The REST controller. + * @param clusterSettings The cluster settings. + * @param indexScopedSettings The index scoped settings. + * @param settingsFilter The settings filter. + * @param indexNameExpressionResolver The index name resolver. + * @param nodesInCluster Supplier for nodes in the cluster. + * @return A list of REST handlers. + */ public List getRestHandlers( Settings settings, org.opensearch.rest.RestController restController, @@ -137,17 +164,11 @@ public class ContentManagerPlugin extends Plugin implements ClusterPlugin, JobSc new RestGetSubscriptionAction(this.ctiConsole), new RestPostSubscriptionAction(this.ctiConsole), new RestDeleteSubscriptionAction(this.ctiConsole), - new RestPostUpdateAction(this.ctiConsole) - ); + new RestPostUpdateAction(this.ctiConsole, this.catalogSyncJob)); } /** - * Initialize. The initialization consists of: - * - * 
-     * 1. create required indices if they do not exist.
-     * 2. initialize from a snapshot if the local consumer does not exist, or its offset is 0.
-     *
+ * Performs initialization tasks for the plugin. */ private void start() { try { @@ -173,7 +194,15 @@ public class ContentManagerPlugin extends Plugin implements ClusterPlugin, JobSc /** - * Schedules the Catalog Sync Job. + * Schedules the Catalog Sync Job within the OpenSearch Job Scheduler. + *

+ * This method performs two main checks asynchronously: + * + * - Ensures the job index ({@value #JOB_INDEX_NAME}) exists. + * - Ensures the specific job document ({@value #JOB_ID}) exists. + * + * If either is missing, it creates them. The job is configured to run based on the + * interval defined in PluginSettings. */ private void scheduleCatalogSyncJob() { this.threadPool.generic().execute(() -> { @@ -205,7 +234,10 @@ public class ContentManagerPlugin extends Plugin implements ClusterPlugin, JobSc ContentJobParameter job = new ContentJobParameter( "Catalog Sync Periodic Task", CatalogSyncJob.JOB_TYPE, - new IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES), + new IntervalSchedule( + Instant.now(), + PluginSettings.getInstance().getCatalogSyncInterval(), + ChronoUnit.MINUTES), true, Instant.now(), Instant.now() @@ -222,37 +254,56 @@ public class ContentManagerPlugin extends Plugin implements ClusterPlugin, JobSc }); } + /** + * Retrieves the list of settings defined by this plugin. + * + * @return A list of {@link Setting} objects including client timeout, API URL, bulk operation limits, and sync interval. + */ @Override public List> getSettings() { return Arrays.asList( - PluginSettings.CONSUMER_ID, - PluginSettings.CONTEXT_ID, PluginSettings.CLIENT_TIMEOUT, PluginSettings.CTI_API_URL, - PluginSettings.CTI_CLIENT_MAX_ATTEMPTS, - PluginSettings.CTI_CLIENT_SLEEP_TIME, - PluginSettings.JOB_MAX_DOCS, - PluginSettings.JOB_SCHEDULE, - PluginSettings.MAX_CHANGES, PluginSettings.MAX_CONCURRENT_BULKS, - PluginSettings.MAX_ITEMS_PER_BULK); + PluginSettings.MAX_ITEMS_PER_BULK, + PluginSettings.CATALOG_SYNC_INTERVAL); } + /** + * Returns the job type identifier for the Job Scheduler extension. + * + * @return The string identifier for content manager jobs. + */ @Override public String getJobType() { return "content-manager-job"; } + /** + * Returns the name of the index used to store job metadata. + * + * @return The job index name. + */ @Override public String getJobIndex() { return JOB_INDEX_NAME; } + /** + * Returns the runner instance responsible for executing the scheduled jobs. + * + * @return The {@link ContentJobRunner} singleton instance. + */ @Override public ScheduledJobRunner getJobRunner() { return ContentJobRunner.getInstance(); } + /** + * Returns the parser responsible for deserializing job parameters from XContent. + * + * @return A {@link ScheduledJobParser} for {@link ContentJobParameter}. + */ @Override public ScheduledJobParser getJobParser() { return (parser, id, jobDocVersion) -> ContentJobParameter.parse(parser); diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/client/CTIClient.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/client/CTIClient.java deleted file mode 100644 index fe237ffc..00000000 --- a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/client/CTIClient.java +++ /dev/null @@ -1,394 +0,0 @@ -/* - * Copyright (C) 2024, Wazuh Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package com.wazuh.contentmanager.client; - -import org.apache.hc.client5.http.HttpHostConnectException; -import org.apache.hc.client5.http.async.methods.*; -import org.apache.hc.client5.http.classic.methods.HttpGet; -import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; -import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; -import org.apache.hc.client5.http.impl.classic.HttpClients; -import org.apache.hc.core5.http.*; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.env.Environment; - -import java.io.*; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.time.Duration; -import java.time.ZonedDateTime; -import java.util.*; - -import com.wazuh.contentmanager.cti.catalog.model.Changes; -import com.wazuh.contentmanager.cti.catalog.model.ConsumerInfo; -import com.wazuh.contentmanager.settings.PluginSettings; -import com.wazuh.contentmanager.utils.VisibleForTesting; -import com.wazuh.contentmanager.utils.XContentUtils; -import reactor.util.annotation.NonNull; - -/** - * CTIClient is a singleton class responsible for interacting with the Cyber Threat Intelligence - * (CTI) API. It extends {@link HttpClient} to manage HTTP requests. - * - *

This client provides methods to fetch CTI catalog data and retrieve content changes based on - * query parameters. - */ -public class CTIClient extends HttpClient { - private static final Logger log = LogManager.getLogger(CTIClient.class); - - private final String CONSUMER_INFO_ENDPOINT; - private final String CONSUMER_CHANGES_ENDPOINT; - - private static CTIClient INSTANCE; - - private final PluginSettings pluginSettings; - - /** Enum representing the query parameters used in CTI API requests. */ - public enum QueryParameters { - /** The starting offset parameter TO_OFFSET - FROM_OFFSET must be >1001 */ - FROM_OFFSET("from_offset"), - /** The destination offset parameter */ - TO_OFFSET("to_offset"), - /** Include empties */ - WITH_EMPTIES("with_empties"); - - private final String value; - - QueryParameters(String value) { - this.value = value; - } - - /** - * Returns the string representation of the query parameter. - * - * @return The query parameter key as a string. - */ - public String getValue() { - return this.value; - } - } - - /** - * Initializes a new instance of the {@code CTIClient} class. - * - *

This constructor creates the CTIClient object by setting up API endpoints using - * configuration values from the {@link PluginSettings} singleton instance. The endpoints include: - * - Consumer information endpoint - Consumer changes endpoint - * - *

The base URI for requests is derived from the CTI base URL provided via {@link - * PluginSettings}. The constructed endpoints are validated to ensure they form valid URIs. - */ - public CTIClient() { - super(URI.create(PluginSettings.getInstance().getCtiBaseUrl())); - - this.pluginSettings = PluginSettings.getInstance(); - this.CONSUMER_INFO_ENDPOINT = - "/catalog/contexts/" - + this.pluginSettings.getContextId() - + "/consumers/" - + this.pluginSettings.getConsumerId(); - - // In order to validate the URI created - try { - URI.create(this.CONSUMER_INFO_ENDPOINT + "/changes"); - } catch (IllegalArgumentException e) { - log.error("Invalid URI for CTI API Changes endpoint: {}", e.getMessage()); - } - this.CONSUMER_CHANGES_ENDPOINT = this.CONSUMER_INFO_ENDPOINT + "/changes"; - } - - /** - * Retrieves the singleton instance of {@code CTIClient}. - * - * @return The singleton instance of {@code CTIClient}. - */ - public static synchronized CTIClient getInstance() { - if (INSTANCE == null) { - INSTANCE = new CTIClient(); - } - return INSTANCE; - } - - /** - * This constructor is only used on tests. - * - * @param CTIBaseURL base URL of the CTI API (mocked). - * @param pluginSettings plugin settings (mocked). - */ - @VisibleForTesting - CTIClient(String CTIBaseURL, PluginSettings pluginSettings) { - super(URI.create(CTIBaseURL)); - this.pluginSettings = pluginSettings; - this.CONSUMER_INFO_ENDPOINT = - "/catalog/contexts/" - + this.pluginSettings.getContextId() - + "/consumers/" - + this.pluginSettings.getConsumerId(); - this.CONSUMER_CHANGES_ENDPOINT = this.CONSUMER_INFO_ENDPOINT + "/changes"; - } - - /** - * Fetches content changes from the CTI API using the provided query parameters. - * - * @param fromOffset The starting offset (inclusive) for fetching changes. - * @param toOffset The ending offset (exclusive) for fetching changes. - * @param withEmpties A flag indicating whether to include empty values (Optional). - * @return {@link Changes} instance with the current changes. - */ - public Changes getChanges(long fromOffset, long toOffset, boolean withEmpties) { - Map params = - CTIClient.contextQueryParameters(fromOffset, toOffset, withEmpties); - SimpleHttpResponse response = - this.sendRequest( - Method.GET, - this.CONSUMER_CHANGES_ENDPOINT, - null, - params, - null, - this.pluginSettings.getCtiClientMaxAttempts()); - // Fail fast - if (response == null) { - log.error("No reply from [{}]", this.CONSUMER_CHANGES_ENDPOINT); - return new Changes(); - } - if (!Arrays.asList(HttpStatus.SC_OK, HttpStatus.SC_SUCCESS).contains(response.getCode())) { - log.error("Request to [{}] failed: {}", this.CONSUMER_CHANGES_ENDPOINT, response.getBody()); - return new Changes(); - } - log.debug("[{}] replied with status [{}]", this.CONSUMER_CHANGES_ENDPOINT, response.getCode()); - try { - return Changes.parse(XContentUtils.createJSONParser(response.getBodyBytes())); - } catch (IOException | IllegalArgumentException e) { - log.error("Failed to parse changes: {}", e.getMessage()); - return new Changes(); - } - } - - /** - * Fetches the entire CTI catalog from the API. - * - * @return A {@link ConsumerInfo} object containing the catalog information. - * @throws HttpHostConnectException server unreachable. - * @throws IOException error parsing response. - */ - public ConsumerInfo getConsumerInfo() throws HttpHostConnectException, IOException { - // spotless:off - SimpleHttpResponse response = this.sendRequest( - Method.GET, - this.CONSUMER_INFO_ENDPOINT, - null, - null, - null, - this.pluginSettings.getCtiClientMaxAttempts() - ); - // spotless:on - if (response == null) { - throw new HttpHostConnectException("No reply from [" + this.CONSUMER_INFO_ENDPOINT + "]"); - } - log.debug("[{}] replied with status [{}]", this.CONSUMER_INFO_ENDPOINT, response.getCode()); - return ConsumerInfo.parse(XContentUtils.createJSONParser(response.getBodyBytes())); - } - - /** - * Builds a map of query parameters for the API request to fetch context changes. - * - * @param fromOffset The starting offset (inclusive). - * @param toOffset The ending offset (exclusive). - * @param withEmpties A flag indicating whether to include empty values. If null or empty, it will - * be ignored. - * @return A map containing the query parameters. - */ - public static Map contextQueryParameters( - long fromOffset, long toOffset, boolean withEmpties) { - Map params = new HashMap<>(); - params.put(QueryParameters.FROM_OFFSET.getValue(), String.valueOf(fromOffset)); - params.put(QueryParameters.TO_OFFSET.getValue(), String.valueOf(toOffset)); - params.put(QueryParameters.WITH_EMPTIES.getValue(), String.valueOf(withEmpties)); - return params; - } - - /** - * Send a request to the CTI API and handles the HTTP response based on the provided status code. - * - *

Implements a retry strategy based on {@code attemptsLeft} - * - * @param method The HTTP method to use for the request. - * @param endpoint The endpoint to append to the base API URI. - * @param body The request body (optional, applicable for POST/PUT). - * @param params The query parameters (optional). - * @param header The headers to include in the request (optional). - * @param attemptsLeft number of retries left. - * @return SimpleHttpResponse or null. - */ - protected SimpleHttpResponse sendRequest( - @NonNull Method method, - @NonNull String endpoint, - String body, - Map params, - Header header, - int attemptsLeft) { - ZonedDateTime cooldown = null; - SimpleHttpResponse response = null; - while (attemptsLeft > 0) { - // Check if in cooldown - if (cooldown != null && ZonedDateTime.now().isBefore(cooldown)) { - long waitTime = Duration.between(ZonedDateTime.now(), cooldown).getSeconds(); - log.info("In cooldown, waiting {} seconds", waitTime); - try { - Thread.sleep(waitTime * 1000); // Wait before retrying - } catch (InterruptedException e) { - log.error("Interrupted while waiting for cooldown", e); - Thread.currentThread().interrupt(); // Reset interrupt status - } - } - - int currentAttempt = this.pluginSettings.getCtiClientMaxAttempts() - attemptsLeft + 1; - log.debug( - "Sending {} request to [{}]. Attempt {}/{}.", - method, - endpoint, - currentAttempt, - this.pluginSettings.getCtiClientMaxAttempts()); - // WARN Changing this to sendRequest makes the test fail. - response = this.doHttpClientSendRequest(method, endpoint, body, params, header); - if (response == null) { - return null; // Handle null - } - - // Calculate timeout - int timeout = currentAttempt * this.pluginSettings.getCtiClientSleepTime(); - int statusCode = response.getCode(); - switch (statusCode) { - case 200: - log.info("Operation succeeded: status code {}", statusCode); - log.debug("Response body: {}", response.getBodyText()); - return response; - - case 400: - log.error( - "Operation failed: status code {} - Error: {}", statusCode, response.getBodyText()); - return response; - - case 422: - log.error( - "Unprocessable Entity: status code {} - Error: {}", - statusCode, - response.getBodyText()); - return response; - - case 429: // Handling Too Many Requests - log.warn("Max requests limit reached: status code {}", statusCode); - try { - String retryAfterValue = response.getHeader("Retry-After").getValue(); - if (retryAfterValue != null) { - timeout = Integer.parseInt(retryAfterValue); - } - cooldown = ZonedDateTime.now().plusSeconds(timeout); // Set cooldown - log.info("Cooldown until {}", cooldown); - } catch (ProtocolException | NullPointerException e) { - log.warn("Retry-After header not present or invalid format: {}", e.getMessage()); - cooldown = ZonedDateTime.now().plusSeconds(timeout); // Default cooldown - } - break; - - case 500: // Handling Server Error - log.warn("Server Error: status code {} - Error: {}", statusCode, response.getBodyText()); - cooldown = ZonedDateTime.now().plusSeconds(60); // Set cooldown for server errors - break; - - default: - log.error("Unexpected status code: {}", statusCode); - return response; - } - attemptsLeft--; // Decrease remaining attempts - } - - log.error("All attempts exhausted for the request to CTI API."); - return response; // Return null if all attempts fail - } - - /*** - * Downloads the CTI snapshot. - * - * @param snapshotURI URI to the file to download. - * @param env environment. Required to resolve files' paths. - * @return The downloaded file's name - */ - public Path download(String snapshotURI, Environment env) { - try (CloseableHttpClient client = HttpClients.createDefault()) { - // Setup - final URI uri = new URI(snapshotURI); - final HttpGet request = new HttpGet(uri); - final String filename = uri.getPath().substring(uri.getPath().lastIndexOf('/') + 1); - final Path path = env.tmpDir().resolve(filename); - - // Download - log.info("Starting snapshot download from [{}]", uri); - try (CloseableHttpResponse response = client.execute(request)) { - if (response.getEntity() != null) { - // Write to disk - InputStream input = response.getEntity().getContent(); - try (OutputStream out = - new BufferedOutputStream( - Files.newOutputStream( - path, - StandardOpenOption.CREATE, - StandardOpenOption.WRITE, - StandardOpenOption.TRUNCATE_EXISTING))) { - - int bytesRead; - byte[] buffer = new byte[1024]; - while ((bytesRead = input.read(buffer)) != -1) { - out.write(buffer, 0, bytesRead); - } - } - } - } - log.info("Snapshot downloaded to [{}]", path); - return path; - } catch (URISyntaxException e) { - log.error("Failed to download snapshot. Invalid URL provided: {}", e.getMessage()); - } catch (IOException e) { - log.error("Snapshot download failed: {}", e.getMessage()); - } - return null; - } - - /** - * Sends an HTTP request to the specified endpoint using the provided method, body, parameters, - * and header. - * - *

This method is intentionally separated from the main logic to facilitate mocking in unit - * tests. - * - * @param method the HTTP method to use (e.g. GET, POST, PUT, etc.) - * @param endpoint the URL of the endpoint to send the request to - * @param body the request body, or null if no body is required - * @param params a map of query parameters to include in the request, or null if no parameters are - * required - * @param header the request header, or null if no header is required - * @return the response from the server, or null if an error occurs - */ - protected SimpleHttpResponse doHttpClientSendRequest( - Method method, String endpoint, String body, Map params, Header header) { - return super.sendRequest(method, endpoint, body, params, header); - } -} diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/client/HttpClient.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/client/HttpClient.java deleted file mode 100644 index f35430a6..00000000 --- a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/client/HttpClient.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Copyright (C) 2024, Wazuh Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package com.wazuh.contentmanager.client; - -import org.apache.hc.client5.http.async.methods.SimpleHttpRequest; -import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; -import org.apache.hc.client5.http.async.methods.SimpleRequestBuilder; -import org.apache.hc.client5.http.async.methods.SimpleRequestProducer; -import org.apache.hc.client5.http.async.methods.SimpleResponseConsumer; -import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient; -import org.apache.hc.client5.http.impl.async.HttpAsyncClients; -import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; -import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; -import org.apache.hc.core5.http.ContentType; -import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpHost; -import org.apache.hc.core5.http.Method; -import org.apache.hc.core5.ssl.SSLContextBuilder; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import javax.net.ssl.SSLContext; - -import java.io.IOException; -import java.net.URI; -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import com.wazuh.contentmanager.settings.PluginSettings; -import reactor.util.annotation.NonNull; - -/** - * HttpClient is a base class to handle HTTP requests to external APIs. It supports secure - * communication using SSL/TLS and manages an async HTTP client. - */ -public class HttpClient { - private static final Logger log = LogManager.getLogger(HttpClient.class); - - private static final Object LOCK = new Object(); - - /** - * Singleton instance of the HTTP client. - */ - protected static CloseableHttpAsyncClient httpClient; - - /** - * Base URI for API requests - */ - protected final URI apiUri; - - /** - * Constructs an HttpClient instance with the specified API URI. - * - * @param apiUri The base URI for API requests. - */ - protected HttpClient(@NonNull URI apiUri) { - log.debug("Client initialized pointing at [{}]", apiUri); - this.apiUri = apiUri; - startHttpAsyncClient(); - } - - /** - * Initializes and starts the HTTP asynchronous client if not already started. Ensures thread-safe - * initialization. - * - * @throws RuntimeException error initializing the HttpClient. - */ - private static void startHttpAsyncClient() throws RuntimeException { - synchronized (LOCK) { - if (httpClient == null) { - try { - SSLContext sslContext = - SSLContextBuilder.create() - .loadTrustMaterial(null, (chains, authType) -> true) - .build(); - - httpClient = - HttpAsyncClients.custom() - .setConnectionManager( - PoolingAsyncClientConnectionManagerBuilder.create() - .setTlsStrategy( - ClientTlsStrategyBuilder.create().setSslContext(sslContext).build()) - .build()) - .build(); - httpClient.start(); - } catch (NoSuchAlgorithmException | KeyStoreException | KeyManagementException e) { - log.error("Error initializing HTTP client: {}", e.getMessage()); - throw new RuntimeException("Failed to initialize HttpClient", e); - } - } - } - } - - /** - * Sends an HTTP request with the specified parameters. - * - * @param method The HTTP method (e.g., GET, POST, PUT, DELETE). - * @param endpoint The endpoint to append to the base API URI. - * @param requestBody The request body (optional, applicable for POST/PUT). - * @param queryParameters The query parameters (optional). - * @param headers The headers to include in the request (optional). - * @return A SimpleHttpResponse containing the response details. - */ - protected SimpleHttpResponse sendRequest( - @NonNull Method method, - String endpoint, - String requestBody, - Map queryParameters, - Header... headers) { - URI _apiUri; - if (httpClient == null) { - startHttpAsyncClient(); - } - if (endpoint != null) { - _apiUri = URI.create(this.apiUri.toString() + endpoint); - } else { - _apiUri = this.apiUri; - } - - try { - HttpHost httpHost = HttpHost.create(_apiUri); - log.debug("Sending {} request to [{}]", method, _apiUri); - - SimpleRequestBuilder builder = SimpleRequestBuilder.create(method); - if (requestBody != null) { - builder.setBody(requestBody, ContentType.APPLICATION_JSON); - } - if (queryParameters != null) { - queryParameters.forEach(builder::addParameter); - } - if (headers != null) { - builder.setHeaders(headers); - } - - SimpleHttpRequest request = builder.setHttpHost(httpHost).setPath(_apiUri.getPath()).build(); - log.debug("Request sent: [{}]", request); - return httpClient - .execute( - SimpleRequestProducer.create(request), - SimpleResponseConsumer.create(), - new HttpResponseCallback( - request, "Failed to execute outgoing " + method + " request")) - .get(PluginSettings.getInstance().getClientTimeout(), TimeUnit.SECONDS); - } catch (InterruptedException | ExecutionException | TimeoutException e) { - log.error("HTTP {} request failed: {}", method, e.getMessage()); - Thread.currentThread().interrupt(); - } catch (Exception e) { - log.error("Unexpected error in HTTP {} request: {}", method, e.getMessage()); - } - return null; - } - - /** - * Closes the underlying HTTP asynchronous client if it exists. Used in tests - * - * @throws IOException if an I/O error occurs while closing the client - */ - public void close() throws IOException { - if (httpClient != null) { - httpClient.close(); - } - } -} diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/CtiCatalog.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/CtiCatalog.java deleted file mode 100644 index 47007bd3..00000000 --- a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/CtiCatalog.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.wazuh.contentmanager.cti.catalog; - -import com.wazuh.contentmanager.cti.catalog.service.ConsumerService; - -/** - * Represents the CTI Catalog. - * Acts as a facade or entry point for catalog-related operations, primarily managing consumers. - */ -public class CtiCatalog { - - private ConsumerService consumerService; - /** - * Constructs a new CtiCatalog instance. - * - * @param consumerService The service used to manage local and remote consumers. - */ - public CtiCatalog(ConsumerService consumerService) { - this.consumerService = consumerService; - } -} diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/client/ApiClient.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/client/ApiClient.java index 97502fbc..55402e97 100644 --- a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/client/ApiClient.java +++ b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/client/ApiClient.java @@ -29,14 +29,15 @@ import java.util.concurrent.TimeoutException; */ public class ApiClient { - private static final String BASE_URI = "https://cti-pre.wazuh.com"; - private static final String API_PREFIX = "/api/v1"; + private final String baseUri; private CloseableHttpAsyncClient client; /** * Constructs an ApiClient instance and initializes the underlying HTTP client. */ public ApiClient() { + // Retrieve base URI from PluginSettings + this.baseUri = PluginSettings.getInstance().getCtiBaseUrl(); this.buildClient(); } @@ -87,7 +88,7 @@ public class ApiClient { * @return A string representing the full absolute URL for the resource. */ private String buildConsumerURI(String context, String consumer) { - return BASE_URI + API_PREFIX + "/catalog/contexts/" + context + "/consumers/" + consumer; + return this.baseUri + "/catalog/contexts/" + context + "/consumers/" + consumer; } /** @@ -115,4 +116,34 @@ public class ApiClient { return future.get(PluginSettings.getInstance().getClientTimeout(), TimeUnit.SECONDS); } + + /** + * Retrieves the changes for a specific consumer within a given context. + * + * @param context The context identifier. + * @param consumer The consumer identifier. + * @param fromOffset The starting offset (exclusive). + * @param toOffset The ending offset (inclusive). + * @return A {@link SimpleHttpResponse} containing the API response. + * @throws ExecutionException If the computation threw an exception. + * @throws InterruptedException If the current thread was interrupted while waiting. + * @throws TimeoutException If the wait timed out. + */ + public SimpleHttpResponse getChanges(String context, String consumer, long fromOffset, long toOffset) throws ExecutionException, InterruptedException, TimeoutException { + String uri = this.buildConsumerURI(context, consumer) + "/changes?from_offset=" + fromOffset + "&to_offset=" + toOffset; + + SimpleHttpRequest request = SimpleRequestBuilder + .get(uri) + .build(); + + final Future future = client.execute( + SimpleRequestProducer.create(request), + SimpleResponseConsumer.create(), + new HttpResponseCallback( + request, "Failed to send request to CTI service" + ) + ); + + return future.get(PluginSettings.getInstance().getClientTimeout(), TimeUnit.SECONDS); + } } diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/index/ConsumersIndex.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/index/ConsumersIndex.java index e2568608..4dd1e894 100644 --- a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/index/ConsumersIndex.java +++ b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/index/ConsumersIndex.java @@ -17,37 +17,29 @@ package com.wazuh.contentmanager.cti.catalog.index; import com.wazuh.contentmanager.cti.catalog.model.LocalConsumer; +import com.wazuh.contentmanager.settings.PluginSettings; +import com.wazuh.contentmanager.utils.ClusterInfo; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.action.DocWriteResponse; -import org.opensearch.action.get.GetRequest; -import org.opensearch.action.get.GetResponse; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.get.GetRequest; +import org.opensearch.action.get.GetResponse; import org.opensearch.action.index.IndexRequest; import org.opensearch.action.index.IndexResponse; import org.opensearch.common.action.ActionFuture; -import org.opensearch.transport.client.Client; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.settings.Settings; -import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.transport.client.Client; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.Locale; -import java.util.Map; -import java.util.NoSuchElementException; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import com.wazuh.contentmanager.cti.catalog.model.ConsumerInfo; -import com.wazuh.contentmanager.settings.PluginSettings; -import com.wazuh.contentmanager.utils.ClusterInfo; - /** Class to manage the Context index. */ -// TODO remove unused methods: all but setConsumer(), getConsumer() and createIndex() public class ConsumersIndex { private static final Logger log = LogManager.getLogger(ConsumersIndex.class); @@ -57,13 +49,6 @@ public class ConsumersIndex { private static final String MAPPING_PATH = "/mappings/consumers-mapping.json"; private final Client client; - - /** - * This instance of ConsumerInfo comprehends the internal state of this class. The ContextIndex - * class is responsible for maintaining its internal state update at all times. - */ - private ConsumerInfo consumerInfo; - private final PluginSettings pluginSettings; /** @@ -76,42 +61,6 @@ public class ConsumersIndex { this.pluginSettings = PluginSettings.getInstance(); } - /** - * Index CTI API consumer information. - * - * @param consumerInfo Model containing information parsed from the CTI API. - * @return the IndexResponse from the indexing operation, or null. - */ - public boolean index(ConsumerInfo consumerInfo) { - try { - IndexRequest indexRequest = - new IndexRequest() - .index(ConsumersIndex.INDEX_NAME) - .source( - consumerInfo.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) - .id(consumerInfo.getContext()); - - IndexResponse indexResponse = - this.client - .index(indexRequest) - .get(this.pluginSettings.getClientTimeout(), TimeUnit.SECONDS); - if (indexResponse.getResult() == DocWriteResponse.Result.CREATED - || indexResponse.getResult() == DocWriteResponse.Result.UPDATED) { - // Update consumer info (internal state). - this.consumerInfo = consumerInfo; - return true; - } - } catch (IOException e) { - log.error("Failed to create JSON content builder: {}", e.getMessage()); - } catch (InterruptedException | ExecutionException | TimeoutException e) { - log.error( - "Failed to index Consumer [{}] information due to: {}", - consumerInfo.getContext(), - e.getMessage()); - } - return false; - } - /** * Indexes a local consumer object into the cluster. * @@ -163,58 +112,6 @@ public class ConsumersIndex { return future.get(this.pluginSettings.getClientTimeout(), TimeUnit.SECONDS); } - /** TODO: Review ConsumerInfo class and adapt mappings accordingly */ - /** - * Searches for the given consumer within a context. - * - * @param context ID (name) of the context. - * @param consumer ID (name) of the consumer. - * @return the required consumer as an instance of {@link ConsumerInfo}, or null. - */ - @SuppressWarnings("unchecked") - public ConsumerInfo get(String context, String consumer) { - // Avoid faulty requests if the cluster is unstable. - if (!ClusterInfo.indexStatusCheck(this.client, ConsumersIndex.INDEX_NAME)) { - throw new RuntimeException("Index not ready"); - } - try { - GetResponse getResponse = - this.client - .get(new GetRequest(ConsumersIndex.INDEX_NAME, context).preference("_local")) - .get(this.pluginSettings.getClientTimeout(), TimeUnit.SECONDS); - - Map source = (Map) getResponse.getSourceAsMap().get(consumer); - if (source == null) { - throw new NoSuchElementException( - String.format( - Locale.ROOT, "Consumer [%s] not found in context [%s]", consumer, context)); - } - - // Update consumer info (internal state) - long offset = ConsumersIndex.asLong(source.get(ConsumerInfo.OFFSET)); - long lastOffset = ConsumersIndex.asLong(source.get(ConsumerInfo.LAST_OFFSET)); - String snapshot = (String) source.get(ConsumerInfo.LAST_SNAPSHOT_LINK); - this.consumerInfo = new ConsumerInfo(consumer, context, offset, lastOffset, snapshot); - log.info( - "Fetched consumer from the [{}] index: {}", ConsumersIndex.INDEX_NAME, this.consumerInfo); - } catch (InterruptedException | ExecutionException | TimeoutException e) { - log.error("Failed to fetch consumer [{}][{}]: {}", context, consumer, e.getMessage()); - } - - // May be null if the request fails and was not initialized on previously. - return this.consumerInfo; - } - - /** - * Utility method to parse an object value to primitive long. - * - * @param o the object to parse. - * @return the value as primitive long. - */ - private static long asLong(Object o) { - return o instanceof Number ? ((Number) o).longValue() : Long.parseLong(o.toString()); - } - /** * Checks whether the {@link ConsumersIndex#INDEX_NAME} index exists. * @@ -227,43 +124,44 @@ public class ConsumersIndex { /** * Creates the {@link ConsumersIndex#INDEX_NAME} index. - * - * @return */ - public CreateIndexResponse createIndex() throws ExecutionException, InterruptedException, TimeoutException { - Settings settings = Settings.builder() - .put("index.number_of_replicas", 0) - .put("hidden", true) - .build(); + public CreateIndexResponse createIndex() throws ExecutionException, InterruptedException, TimeoutException { + Settings settings = Settings.builder() + .put("index.number_of_replicas", 0) + .put("hidden", true) + .build(); - String mappings; - try { - mappings = this.loadMappingFromResources(); - } catch (IOException e) { - log.error("Could not read mappings for index [{}]", INDEX_NAME); - return null; - } + String mappings; + try { + mappings = this.loadMappingFromResources(); + } catch (IOException e) { + log.error("Could not read mappings for index [{}]", INDEX_NAME); + return null; + } - CreateIndexRequest request = new CreateIndexRequest() + CreateIndexRequest request = new CreateIndexRequest() .index(INDEX_NAME) .mapping(mappings) .settings(settings); - return this.client + return this.client .admin() .indices() .create(request) .get(this.pluginSettings.getClientTimeout(), TimeUnit.SECONDS); - } + } /** - * Loads the index mapping from the resources folder. + * Loads the index mapping from the 'resources' folder. * * @return the mapping as a JSON string. * @throws IOException if reading the resource fails. */ - private String loadMappingFromResources() throws IOException { + protected String loadMappingFromResources() throws IOException { try (InputStream is = this.getClass().getResourceAsStream(MAPPING_PATH)) { + if (is == null) { + throw new java.io.FileNotFoundException("Mapping file not found: " + MAPPING_PATH); + } return new String(is.readAllBytes(), StandardCharsets.UTF_8); } } diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/index/ContentIndex.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/index/ContentIndex.java index 51caa84a..66bea352 100644 --- a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/index/ContentIndex.java +++ b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/index/ContentIndex.java @@ -16,8 +16,16 @@ */ package com.wazuh.contentmanager.cti.catalog.index; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import com.wazuh.contentmanager.cti.catalog.model.Operation; +import com.wazuh.contentmanager.cti.catalog.utils.JsonPatch; +import com.wazuh.contentmanager.settings.PluginSettings; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchTimeoutException; @@ -32,91 +40,89 @@ import org.opensearch.action.get.GetRequest; import org.opensearch.action.get.GetResponse; import org.opensearch.action.index.IndexRequest; import org.opensearch.common.settings.Settings; -import org.opensearch.transport.client.Client; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.action.ActionListener; import org.opensearch.core.xcontent.ToXContent; -import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.mapper.StrictDynamicMappingException; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.reindex.BulkByScrollResponse; import org.opensearch.index.reindex.DeleteByQueryAction; import org.opensearch.index.reindex.DeleteByQueryRequestBuilder; +import org.opensearch.transport.client.Client; -import java.io.BufferedReader; -import java.io.FileReader; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; +import java.util.*; import java.util.concurrent.ExecutionException; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import com.wazuh.contentmanager.cti.catalog.model.Changes; -import com.wazuh.contentmanager.cti.catalog.model.Offset; -import com.wazuh.contentmanager.cti.catalog.model.Operation; -import com.wazuh.contentmanager.settings.PluginSettings; -import com.wazuh.contentmanager.cti.catalog.utils.JsonPatch; -import com.wazuh.contentmanager.utils.XContentUtils; - /** - * Manages operations for the Wazuh CTI Content Index. + * Manages operations for a specific Wazuh CTI Content Index. + *

+ * This class handles the lifecycle of the index (creation, deletion) as well as + * CRUD operations for documents, including specialized logic for parsing + * and sanitizing CTI content payloads. */ public class ContentIndex { - private static final String JSON_NAME_KEY = "name"; - private static final String JSON_OFFSET_KEY = "offset"; private static final Logger log = LogManager.getLogger(ContentIndex.class); - //TODO: Delete - public static final String INDEX_NAME = "wazuh-ruleset"; - private final Client client; private final PluginSettings pluginSettings; private final Semaphore semaphore; - private String indexName; - private String mappingsPath; - private String alias; + private final String indexName; + private final String mappingsPath; + private final String alias; + + private final ObjectMapper jsonMapper; + private final ObjectMapper yamlMapper; + private static final List DECODER_ORDER_KEYS = Arrays.asList( + "name", "metadata", "parents", "definitions", "check", + "parse|event.original", "parse|message", "normalize" + ); /** - * Constructs a ContentIndex manager with specific settings. + * Constructs a new ContentIndex manager. * - * @param client The OpenSearch client. + * @param client The OpenSearch client used to communicate with the cluster. * @param indexName The name of the index to manage. - * @param mappingsPath The classpath resource path to the index mappings file. + * @param mappingsPath The classpath resource path to the JSON mapping file. */ public ContentIndex(Client client, String indexName, String mappingsPath) { + this(client, indexName, mappingsPath, null); + } + + /** + * Constructs a new ContentIndex manager with an alias. + * + * @param client The OpenSearch client used to communicate with the cluster. + * @param indexName The name of the index to manage. + * @param mappingsPath The classpath resource path to the JSON mapping file. + * @param alias The alias to associate with the index (can be null). + */ + public ContentIndex(Client client, String indexName, String mappingsPath, String alias) { this.pluginSettings = PluginSettings.getInstance(); - this.semaphore = new Semaphore(pluginSettings.getMaximumConcurrentBulks()); + this.semaphore = new Semaphore(this.pluginSettings.getMaximumConcurrentBulks()); this.client = client; this.indexName = indexName; this.mappingsPath = mappingsPath; - } - - /** - * Constructs a ContentIndex manager with specific settings and an alias. - * - * @param client The OpenSearch client. - * @param indexName The name of the index to manage. - * @param mappingsPath The classpath resource path to the index mappings file. - * @param alias The alias to assign to the index. - */ - public ContentIndex(Client client, String indexName, String mappingsPath, String alias) { - this(client, indexName, mappingsPath); this.alias = alias; + this.jsonMapper = new ObjectMapper(); + this.yamlMapper = new ObjectMapper(new YAMLFactory()); } /** - * Creates the content index with specific settings and mappings. + * Creates the index in OpenSearch using the configured mappings and settings. + *

+ * Applies specific settings (hidden=true, replicas=0) and registers an alias if one is defined. * - * @return A {@link CreateIndexResponse} indicating success, or {@code null} if mappings could not be read. - * @throws ExecutionException If the creation request fails. - * @throws InterruptedException If the thread is interrupted while waiting for the response. - * @throws TimeoutException If the operation exceeds the configured client timeout. + * @return The response from the create index operation, or null if mappings could not be read. + * @throws ExecutionException If the client execution fails. + * @throws InterruptedException If the thread is interrupted while waiting. + * @throws TimeoutException If the operation exceeds the client timeout setting. */ public CreateIndexResponse createIndex() throws ExecutionException, InterruptedException, TimeoutException { Settings settings = Settings.builder() @@ -141,18 +147,97 @@ public class ContentIndex { request.alias(new Alias(this.alias)); } - return this.client - .admin() - .indices() - .create(request) - .get(this.pluginSettings.getClientTimeout(), TimeUnit.SECONDS); + return this.client.admin().indices().create(request).get(this.pluginSettings.getClientTimeout(), TimeUnit.SECONDS); } + /** + * Checks if a document with the specified ID exists in the index. + * + * @param id The ID of the document to check. + * @return true if the document exists, false otherwise. + */ + public boolean exists(String id) { + return this.client.prepareGet(this.indexName, id).setFetchSource(false).get().isExists(); + } /** - * Executes a bulk request using the semaphore. + * Indexes a new document or overwrites an existing one. + *

+ * The payload is pre-processed (sanitized and enriched) before being indexed. * - * @param bulkRequest The request to execute. + * @param id The unique identifier for the document. + * @param payload The JSON object representing the document content. + * @throws IOException If the indexing operation fails. + */ + public void create(String id, JsonObject payload) throws IOException { + this.processPayload(payload); + IndexRequest request = new IndexRequest(this.indexName) + .id(id) + .source(payload.toString(), XContentType.JSON); + + try { + this.client.index(request).get(this.pluginSettings.getClientTimeout(), TimeUnit.SECONDS); + } catch (Exception e) { + log.error("Failed to index document [{}]: {}", id, e.getMessage()); + throw new IOException(e); + } + } + + /** + * Updates an existing document by applying a list of patch operations. + * + * @param id The ID of the document to update. + * @param operations The list of operations to apply to the document. + * @throws Exception If the document does not exist, or if patching/indexing fails. + */ + public void update(String id, List operations) throws Exception { + // 1. Fetch + GetResponse response = this.client.get(new GetRequest(this.indexName, id)).get(this.pluginSettings.getClientTimeout(), TimeUnit.SECONDS); + if (!response.isExists()) { + throw new IOException("Document [" + id + "] not found for update."); + } + + // 2. Patch + JsonObject currentDoc = JsonParser.parseString(response.getSourceAsString()).getAsJsonObject(); + for (Operation op : operations) { + XContentBuilder builder = XContentFactory.jsonBuilder(); + op.toXContent(builder, ToXContent.EMPTY_PARAMS); + JsonObject opJson = JsonParser.parseString(builder.toString()).getAsJsonObject(); + JsonPatch.applyOperation(currentDoc, opJson); + } + + // 3. Process + this.processPayload(currentDoc); + + // 4. Index + IndexRequest request = new IndexRequest(this.indexName) + .id(id) + .source(currentDoc.toString(), XContentType.JSON); + this.client.index(request).get(this.pluginSettings.getClientTimeout(), TimeUnit.SECONDS); + } + + /** + * Asynchronously deletes a document from the index. + * + * @param id The ID of the document to delete. + */ + public void delete(String id) { + this.client.delete(new DeleteRequest(this.indexName, id), new ActionListener<>() { + @Override + public void onResponse(DeleteResponse response) { + log.debug("Deleted {} from {}", id, ContentIndex.this.indexName); + } + @Override + public void onFailure(Exception e) { + log.error("Failed to delete {}: {}", id, e.getMessage()); + } + }); + } + + /** + * Executes a bulk request asynchronously. + * + * @param bulkRequest The BulkRequest containing multiple index/delete operations. */ public void executeBulk(BulkRequest bulkRequest) { try { @@ -160,18 +245,15 @@ public class ContentIndex { this.client.bulk(bulkRequest, new ActionListener<>() { @Override public void onResponse(BulkResponse bulkResponse) { - semaphore.release(); + ContentIndex.this.semaphore.release(); if (bulkResponse.hasFailures()) { log.warn("Bulk indexing finished with failures: {}", bulkResponse.buildFailureMessage()); - } else { - log.debug("Bulk indexing successful. Indexed {} documents.", bulkResponse.getItems().length); } } - @Override public void onFailure(Exception e) { - semaphore.release(); - log.error("Bulk indexing failed completely: {}", e.getMessage()); + ContentIndex.this.semaphore.release(); + log.error("Bulk index operation failed: {}", e.getMessage()); } }); } catch (InterruptedException e) { @@ -181,252 +263,103 @@ public class ContentIndex { } /** - * Constructs a ContentIndex manager using default plugin settings. - * - * @param client the OpenSearch Client to interact with the cluster - */ - public ContentIndex(Client client) { - this.pluginSettings = PluginSettings.getInstance(); - this.semaphore = new Semaphore(pluginSettings.getMaximumConcurrentBulks()); - this.client = client; - } - - /** - * Constructs a ContentIndex manager with injected settings (testing). - * - * @param client Client. - * @param pluginSettings PluginSettings. - */ - public ContentIndex(Client client, PluginSettings pluginSettings) { - this.pluginSettings = pluginSettings; - this.semaphore = new Semaphore(pluginSettings.getMaximumConcurrentBulks()); - this.client = client; - } - - /** - * Searches for an element in the {@link ContentIndex#INDEX_NAME} by its ID. - * - * @param resourceId the ID of the element to retrieve. - * @return the element as a JsonObject instance. - * @throws InterruptedException if the operation is interrupted. - * @throws ExecutionException if an error occurs during execution. - * @throws TimeoutException if the operation times out. - * @throws IllegalArgumentException if the content is not found in the index. - */ - public JsonObject getById(String resourceId) - throws InterruptedException, ExecutionException, TimeoutException, IllegalArgumentException { - GetResponse response = - this.client - .get(new GetRequest(ContentIndex.INDEX_NAME, resourceId)) - .get(this.pluginSettings.getClientTimeout(), TimeUnit.SECONDS); - if (response.isExists()) { - return JsonParser.parseString(response.getSourceAsString()).getAsJsonObject(); - } - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "Document with ID [%s] not found in the [%s] index", - resourceId, - ContentIndex.INDEX_NAME)); - } - - /** - * Indexes a single Offset document synchronously. - * - * @param document {@link Offset} document to index. - * @throws StrictDynamicMappingException if the document does not match the index mappings. - * @throws ExecutionException if the index operation failed to execute. - * @throws InterruptedException if the index operation was interrupted. - * @throws TimeoutException if the index operation timed out. - * @throws IOException if XContentBuilder creation fails. - */ - public void index(Offset document) - throws StrictDynamicMappingException, - ExecutionException, - InterruptedException, - TimeoutException, - IOException { - IndexRequest indexRequest = - new IndexRequest() - .index(ContentIndex.INDEX_NAME) - .source(document.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) - .id(document.getResource()); - this.client.index(indexRequest).get(this.pluginSettings.getClientTimeout(), TimeUnit.SECONDS); - } - - /** - * Indexes a list of JSON documents in bulk asynchronously. - * - * @param documents list of JSON documents to be indexed. - */ - public void index(List documents) { - BulkRequest bulkRequest = new BulkRequest(ContentIndex.INDEX_NAME); - for (JsonObject document : documents) { - bulkRequest.add( - new IndexRequest() - .id(document.get(ContentIndex.JSON_NAME_KEY).getAsString()) - .source(document.toString(), XContentType.JSON)); - } - - this.client.bulk( - bulkRequest, - new ActionListener<>() { - @Override - public void onResponse(BulkResponse bulkResponse) { - semaphore.release(); - if (bulkResponse.hasFailures()) { - log.error("Bulk index operation failed: {}", bulkResponse.buildFailureMessage()); - } else { - log.debug("Bulk index operation succeeded in {} ms", bulkResponse.getTook().millis()); - } - } - - @Override - public void onFailure(Exception e) { - semaphore.release(); - log.error("Bulk index operation failed: {}", e.getMessage(), e); - } - }); - } - - /** - * Deletes a document from the index asynchronously. - * - * @param id ID of the document to delete. - */ - public void delete(String id) { - this.client.delete( - new DeleteRequest(ContentIndex.INDEX_NAME, id), - new ActionListener<>() { - @Override - public void onResponse(DeleteResponse response) { - log.info("Deleted CTI Catalog Content {} from index", id); - } - - @Override - public void onFailure(Exception e) { - log.error("Failed to delete CTI Catalog Content {}: {}", id, e.getMessage(), e); - } - }); - } - - /** - * Initializes the index from a local snapshot file. - * - * @param path path to the CTI snapshot JSON file to be indexed. - * @return The offset number of the last indexed resource of the snapshot, or 0 on error/empty. - */ - public long fromSnapshot(String path) { - long startTime = System.currentTimeMillis(); - - String line; - JsonObject json; - int lineCount = 0; - ArrayList items = new ArrayList<>(); - - try (BufferedReader reader = new BufferedReader(new FileReader(path, StandardCharsets.UTF_8))) { - while ((line = reader.readLine()) != null) { - json = JsonParser.parseString(line).getAsJsonObject(); - items.add(json); - lineCount++; - - // Index items (MAX_DOCUMENTS reached) - if (lineCount == this.pluginSettings.getMaxItemsPerBulk()) { - this.semaphore.acquire(); - this.index(items); - lineCount = 0; - items.clear(); - } - } - // Index remaining items (> MAX_DOCUMENTS) - if (lineCount > 0) { - this.semaphore.acquire(); - this.index(items); - } - } catch (InterruptedException e) { - items.clear(); - log.error("Processing snapshot file interrupted {}", e.getMessage()); - } catch (Exception e) { - items.clear(); - log.error("Generic exception indexing the snapshot: {}", e.getMessage()); - } - long estimatedTime = System.currentTimeMillis() - startTime; - log.info("Snapshot indexing finished successfully in {} ms", estimatedTime); - - return items.isEmpty() - ? 0 - : items.get(items.size() - 1).get(ContentIndex.JSON_OFFSET_KEY).getAsLong(); - } - - /** - * Applies a set of changes (create, update, delete) to the content index. - * - * @param changes content changes to apply. - * @throws RuntimeException if the patching process is interrupted or fails. - * @deprecated Use of this specific patch implementation may be replaced by newer synchronization methods. - */ - public void patch(Changes changes) { - ArrayList offsets = changes.get(); - if (offsets.isEmpty()) { - log.info("No changes to apply"); - return; - } - - log.info( - "Patching [{}] from offset [{}] to [{}]", - ContentIndex.INDEX_NAME, - changes.getFirst().getOffset(), - changes.getLast().getOffset()); - for (Offset change : offsets) { - String id = change.getResource(); - try { - log.debug("Processing offset [{}]", change.getOffset()); - switch (change.getType()) { - case CREATE: - log.debug("Creating new resource with ID [{}]", id); - this.index(change); - break; - case UPDATE: - log.debug("Updating resource with ID [{}]", id); - JsonObject content = this.getById(id); - for (Operation op : change.getOperations()) { - JsonPatch.applyOperation(content, XContentUtils.xContentObjectToJson(op)); - } - try (XContentParser parser = XContentUtils.createJSONParser(content)) { - this.index(Offset.parse(parser)); - } - break; - case DELETE: - log.debug("Deleting resource with ID [{}]", id); - this.delete(id); - break; - default: - throw new IllegalArgumentException("Unknown change type: " + change.getType()); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException("Interrupted while patching", e); - } catch (Exception e) { - log.error("Failed to patch [{}] due to {}", id, e.getMessage()); - throw new RuntimeException("Patch operation failed", e); - } - } - } - - /** - * Clears all documents from the {@link ContentIndex#INDEX_NAME} index using a "delete by query" operation. + * Deletes all documents in the index using a "match_all" query. */ public void clear() { try { - DeleteByQueryRequestBuilder deleteByQuery = - new DeleteByQueryRequestBuilder(this.client, DeleteByQueryAction.INSTANCE); + DeleteByQueryRequestBuilder deleteByQuery = new DeleteByQueryRequestBuilder(this.client, DeleteByQueryAction.INSTANCE); deleteByQuery.source(this.indexName).filter(QueryBuilders.matchAllQuery()); - BulkByScrollResponse response = deleteByQuery.get(); - log.debug( - "[{}] wiped. {} documents were removed", this.indexName, response.getDeleted()); + log.debug("[{}] wiped. {} documents removed", this.indexName, response.getDeleted()); } catch (OpenSearchTimeoutException e) { log.error("[{}] delete query timed out: {}", this.indexName, e.getMessage()); } } + + /** + * Orchestrates the enrichment and sanitization of a payload. + * + * @param payload The JSON payload to process. + */ + private void processPayload(JsonObject payload) { + if (payload.has("type") && "decoder".equalsIgnoreCase(payload.get("type").getAsString())) { + this.enrichDecoderWithYaml(payload); + } + if (payload.has("document")) { + this.preprocessDocument(payload.getAsJsonObject("document")); + } + } + + /** + * Generates a YAML representation for decoder documents. + * + * @param payload The payload containing the decoder definition. + */ + private void enrichDecoderWithYaml(JsonObject payload) { + try { + if (!payload.has("document")) return; + JsonNode docNode = this.jsonMapper.readTree(payload.get("document").toString()); + + if (docNode != null && docNode.isObject()) { + Map orderedDecoderMap = new LinkedHashMap<>(); + for (String key : DECODER_ORDER_KEYS) { + if (docNode.has(key)) orderedDecoderMap.put(key, docNode.get(key)); + } + Iterator> fields = docNode.fields(); + while (fields.hasNext()) { + Map.Entry field = fields.next(); + if (!DECODER_ORDER_KEYS.contains(field.getKey())) { + orderedDecoderMap.put(field.getKey(), field.getValue()); + } + } + payload.addProperty("decoder", this.yamlMapper.writeValueAsString(orderedDecoderMap)); + } + } catch (IOException e) { + log.error("Failed to convert decoder payload to YAML: {}", e.getMessage(), e); + } + } + + /** + * Sanitizes the document by removing internal or unnecessary fields. + *

+ * This removes fields like 'date', 'enabled', and internal metadata, and + * normalizes 'related' objects. + * + * @param document The document object to preprocess. + */ + private void preprocessDocument(JsonObject document) { + if (document.has("metadata") && document.get("metadata").isJsonObject()) { + JsonObject metadata = document.getAsJsonObject("metadata"); + if (metadata.has("custom_fields")) { + metadata.remove("custom_fields"); + } + if (metadata.has("dataset")) { + metadata.remove("dataset"); + } + } + + if (document.has("related")) { + JsonElement relatedElement = document.get("related"); + if (relatedElement.isJsonObject()) { + this.sanitizeRelatedObject(relatedElement.getAsJsonObject()); + } else if (relatedElement.isJsonArray()) { + JsonArray relatedArray = relatedElement.getAsJsonArray(); + for (JsonElement element : relatedArray) { + if (element.isJsonObject()) this.sanitizeRelatedObject(element.getAsJsonObject()); + } + } + } + } + + /** + * Normalizes a "related" object. + * + * @param relatedObj The related object to sanitize. + */ + private void sanitizeRelatedObject(JsonObject relatedObj) { + if (relatedObj.has("sigma_id")) { + relatedObj.add("id", relatedObj.get("sigma_id")); + relatedObj.remove("sigma_id"); + } + } } diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/model/Changes.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/model/Changes.java index d7887520..72a8e13a 100644 --- a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/model/Changes.java +++ b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/model/Changes.java @@ -16,7 +16,6 @@ */ package com.wazuh.contentmanager.cti.catalog.model; -import org.opensearch.core.common.ParsingException; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -26,94 +25,74 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -/** ToXContentObject model to parse and build CTI API changes query replies. */ +/** + * This class acts as a wrapper for a list of {@link Offset} objects. + */ public class Changes implements ToXContentObject { private static final String JSON_DATA_KEY = "data"; - private final ArrayList list; - - /** Constructor. */ - public Changes() { - this.list = new ArrayList<>(); - } + private final List list; /** - * Constructor. + * Constructs a new Changes object with the specified list of offsets. * - * @param list a List of Offset objects, each containing a JSON patch. + * @param list The list of {@link Offset} objects. If null, an empty list is initialized. */ public Changes(List list) { - this.list = new ArrayList<>(list); + this.list = list != null ? list : new ArrayList<>(); } /** - * Get the list of changes. + * Retrieves the list of changes. * - * @return A list of Offset objects + * @return The list of {@link Offset} objects. */ - public ArrayList get() { + public List get() { return this.list; } /** - * Get first element of the changes list. + * Parses an XContent stream to create a {@code Changes} instance. + *

+ * This method expects the parser to be positioned at the start of a JSON object. + * It looks for a field named "data" (defined by {@code JSON_DATA_KEY}), which + * must be an array of {@link Offset} objects. * - * @return first {@link Offset} element in the list, or null. + * @param parser The {@link XContentParser} to read from. + * @return A populated {@code Changes} object. + * @throws IOException If an I/O error occurs or the content structure is invalid. */ - public Offset getFirst() { - return !this.list.isEmpty() ? this.list.get(0) : null; - } - - /** - * Get last element of the changes list. - * - * @return last {@link Offset} element in the list, or null. - */ - public Offset getLast() { - return !this.list.isEmpty() ? this.list.get(this.list.size() - 1) : null; - } - - /** - * Parses the data[] object from the CTI API changes response body. - * - * @param parser The received parser object. - * @return a ContentChanges object with all inner array values parsed. - * @throws IOException rethrown from the inner parse() methods. - * @throws IllegalArgumentException rethrown from the inner parse() methods. - * @throws ParsingException rethrown from ensureExpectedToken(). - */ - public static Changes parse(XContentParser parser) - throws IOException, IllegalArgumentException, ParsingException { + public static Changes parse(XContentParser parser) throws IOException { List changes = new ArrayList<>(); - // Make sure we are at the start - XContentParserUtils.ensureExpectedToken( - XContentParser.Token.START_OBJECT, parser.nextToken(), parser); - // Check that we are indeed reading the "data" array - XContentParserUtils.ensureFieldName(parser, parser.nextToken(), JSON_DATA_KEY); - // Check we are at the start of the array - XContentParserUtils.ensureExpectedToken( - XContentParser.Token.START_ARRAY, parser.nextToken(), parser); - // Iterate over the array and add each Offset object to changes list - while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - changes.add(Offset.parse(parser)); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + if (JSON_DATA_KEY.equals(parser.currentName())) { + parser.nextToken(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + changes.add(Offset.parse(parser)); + } + } else { + parser.skipChildren(); + } } return new Changes(changes); } /** - * Outputs an XContentBuilder object ready to be printed or manipulated + * Serializes this object into an {@link XContentBuilder}. * - * @param builder the received builder object - * @param params Unused params - * @return an XContentBuilder object ready to be printed - * @throws IOException rethrown from Offset's toXContent + * @param builder The builder to write to. + * @param params Contextual parameters for the serialization. + * @return The builder instance for chaining. + * @throws IOException If an error occurs while writing to the builder. */ @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.startArray(Changes.JSON_DATA_KEY); - // For each Offset in the data field, add them to an XContentBuilder array + builder.startArray(JSON_DATA_KEY); for (Offset change : this.list) { - change.toXContent(builder, ToXContentObject.EMPTY_PARAMS); + change.toXContent(builder, params); } builder.endArray(); return builder.endObject(); diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/model/ConsumerInfo.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/model/ConsumerInfo.java deleted file mode 100644 index 071b92a5..00000000 --- a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/model/ConsumerInfo.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Copyright (C) 2024, Wazuh Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package com.wazuh.contentmanager.cti.catalog.model; - -import org.opensearch.core.xcontent.ToXContentObject; -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.core.xcontent.XContentParser; - -import java.io.IOException; - -/** ToXContentObject model to parse and build CTI API Catalog query replies */ -public class ConsumerInfo implements ToXContentObject { - private static final String ID = "id"; - private static final String CONTEXT = "context"; - private static final String NAME = "name"; - public static final String LAST_OFFSET = "last_offset"; - public static final String OFFSET = "offset"; - private static final String PATHS_FILTER = "paths_filter"; - public static final String LAST_SNAPSHOT_LINK = "last_snapshot_link"; - private static final String LAST_SNAPSHOT_OFFSET = "last_snapshot_offset"; - private static final String LAST_SNAPSHOT_AT = "last_snapshot_at"; - private static final String CHANGES_URL = "changes_url"; - private static final String INSERTED_AT = "inserted_at"; - private static final String DATA = "data"; - private static final String UPDATED_AT = "updated_at"; - private static final String OPERATIONS = "operations"; - private final String context; - private final String name; - private long offset; - private long lastOffset; - private String lastSnapshotLink; - - /** - * Constructor. - * - * @param name Name of the consumer - * @param context Name of the context - * @param offset The current offset number - * @param lastOffset The last offset number - * @param lastSnapshotLink URL link to the latest snapshot - */ - public ConsumerInfo( - String name, String context, long offset, long lastOffset, String lastSnapshotLink) { - this.name = name; - this.context = context; - this.setOffset(offset); - this.setLastOffset(lastOffset); - this.lastSnapshotLink = lastSnapshotLink; - } - - /** - * Parses the consumer's information within an XContentParser (reply from the CTI API). - * - * @param parser the incoming parser. - * @return a fully parsed ConsumerInfo object. - * @throws IOException rethrown from parse(). - * @throws IllegalArgumentException rethrown from parse(). - */ - public static ConsumerInfo parse(XContentParser parser) - throws IOException, IllegalArgumentException { - String context = null; - String name = null; - long lastOffset = 0L; - // We are initializing the offset to 0 - long offset = 0L; - String lastSnapshotLink = null; - while (parser.nextToken() != XContentParser.Token.END_OBJECT) { - if (parser.currentToken().equals(XContentParser.Token.FIELD_NAME)) { - String fieldName = parser.currentName(); - parser.nextToken(); - switch (fieldName) { - case DATA: - case ID: - case OPERATIONS: - case INSERTED_AT: - case UPDATED_AT: - case PATHS_FILTER: - case CHANGES_URL: - case LAST_SNAPSHOT_AT: - case LAST_SNAPSHOT_OFFSET: - break; - case NAME: - name = parser.text(); - break; - case CONTEXT: - context = parser.text(); - break; - case LAST_OFFSET: - lastOffset = parser.longValue(); - break; - case LAST_SNAPSHOT_LINK: - lastSnapshotLink = parser.text(); - break; - default: - parser.skipChildren(); - break; - } - } - } - return new ConsumerInfo(name, context, offset, lastOffset, lastSnapshotLink); - } - - /** - * Creates an XContentBuilder for the parsed object - * - * @param builder Incoming builder to add the fields to - * @param params Not used - * @return a valid XContentBuilder object ready to be turned into JSON - * @throws IOException rethrown from XContentBuilder methods - */ - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.startObject(this.name); - builder.field(ConsumerInfo.LAST_OFFSET, this.lastOffset); - builder.field(ConsumerInfo.LAST_SNAPSHOT_LINK, this.lastSnapshotLink); - builder.field(ConsumerInfo.OFFSET, this.offset); - builder.endObject(); - return builder.endObject(); - } - - /** - * {@link ConsumerInfo#context} getter. - * - * @return the consumer's context name. - */ - public String getContext() { - return this.context; - } - - /** - * {@link ConsumerInfo#name} getter. - * - * @return the consumer's name. - */ - public String getName() { - return this.name; - } - - /** - * Get the latest consumer's offset (as last fetched from the CTI API). - * - * @return Consumer's latest available offset. - */ - public long getLastOffset() { - return this.lastOffset; - } - - /** - * Get the consumer's offset (in Indexer). - * - * @return The consumer's offset. - */ - public long getOffset() { - return this.offset; - } - - /** - * Get the URL of the latest consumer's snapshot. - * - * @return URL string. - */ - public String getLastSnapshotLink() { - return this.lastSnapshotLink; - } - - /** - * {@link ConsumerInfo#offset} setter. - * - * @param offset new value (positive). - * @throws IllegalArgumentException when {@code offset < 0}. - */ - public void setOffset(long offset) throws IllegalArgumentException { - if (offset < 0) { - throw new IllegalArgumentException("Offset can't be negative"); - } - this.offset = offset; - } - - /** - * {@link ConsumerInfo#lastOffset} setter. - * - * @param offset new value (positive). - * @throws IllegalArgumentException when {@code offset < 0}. - */ - public void setLastOffset(long offset) throws IllegalArgumentException { - if (offset < 0) { - throw new IllegalArgumentException("Offset can't be negative"); - } - this.lastOffset = offset; - } - - /** - * {@link ConsumerInfo#lastSnapshotLink} setter. - * - * @param url new value (URL). - */ - public void setLastSnapshotLink(String url) { - this.lastSnapshotLink = url; - } - - @Override - public String toString() { - return "ConsumerInfo{" - + "context='" - + context - + '\'' - + ", name='" - + name - + '\'' - + ", offset=" - + offset - + ", lastOffset=" - + lastOffset - + ", lastSnapshotLink='" - + lastSnapshotLink - + '\'' - + '}'; - } -} diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/model/Offset.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/model/Offset.java index 08bd9b41..9b72f5a9 100644 --- a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/model/Offset.java +++ b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/model/Offset.java @@ -25,9 +25,11 @@ import java.io.IOException; import java.util.*; /** - * ToXContentObject model to parse and build CTI API changes. - * - *

This class represents an offset in the context of a content change operation. + * Data Transfer Object representing a change offset from the CTI API. + *

+ * This class encapsulates a single synchronization event, defining what action + * took place (Create, Update, Delete), which resource was affected, and the + * data associated with that change (either a full payload or a list of patch operations). */ public class Offset implements ToXContentObject { private static final String CONTEXT = "context"; @@ -37,6 +39,7 @@ public class Offset implements ToXContentObject { private static final String VERSION = "version"; private static final String OPERATIONS = "operations"; private static final String PAYLOAD = "payload"; + private final String context; private final long offset; private final String resource; @@ -45,35 +48,20 @@ public class Offset implements ToXContentObject { private final List operations; private final Map payload; - /** - * Type of change represented by the offset. Possible values are defined in catalog.md. - */ - public enum Type { - CREATE, - UPDATE, - DELETE - } + public enum Type { CREATE, UPDATE, DELETE } /** - * Constructor. + * Constructs a new Offset instance. * - * @param context Name of the context - * @param offset Offset number of the record - * @param resource Name of the resource - * @param type OperationType of operation to be performed - * @param version Version Number - * @param operations JSON Patch payload data - * @param payload JSON Patch payload data + * @param context The context or category of the content (e.g., catalog ID). + * @param offset The sequential ID of this event. Defaults to 0 if null. + * @param resource The unique identifier of the specific resource being modified. + * @param type The type of modification (CREATE, UPDATE, DELETE). + * @param version The version number of the resource. Defaults to 0 if null. + * @param operations A list of patch operations (typically used with UPDATE). + * @param payload The full resource content (typically used with CREATE). */ - public Offset( - String context, - Long offset, - String resource, - Offset.Type type, - Long version, - List operations, - Map payload) { + public Offset(String context, long offset, String resource, Type type, long version, List operations, Map payload) { this.context = context; this.offset = offset; this.resource = resource; @@ -84,204 +72,106 @@ public class Offset implements ToXContentObject { } /** - * Builds an Offset instance from the content of an XContentParser. + * Parses an XContent stream to create an {@code Offset} instance. * - * @param parser The XContentParser parser holding the data. - * @return A new Offset instance. - * @throws IOException if an I/O error occurs during parsing. - * @throws IllegalArgumentException unexpected token found during parsing. + * @param parser The {@link XContentParser} to read from. + * @return A populated {@code Offset} object. + * @throws IOException If an I/O error occurs or the JSON structure is invalid. */ - public static Offset parse(XContentParser parser) throws IOException, IllegalArgumentException { + public static Offset parse(XContentParser parser) throws IOException { String context = null; - long offset = 0; + Long offset = null; String resource = null; - Offset.Type type = null; - long version = 0; + Type type = null; + Long version = null; List operations = new ArrayList<>(); - Map payload = new HashMap<>(); + Map payload = null; while (parser.nextToken() != XContentParser.Token.END_OBJECT) { if (parser.currentToken() == XContentParser.Token.FIELD_NAME) { String fieldName = parser.currentName(); parser.nextToken(); switch (fieldName) { - case CONTEXT: - context = parser.text(); - break; - case OFFSET: - offset = parser.longValue(); - break; - case RESOURCE: - resource = parser.text(); - break; - case TYPE: - String opType = parser.text().trim().toUpperCase(Locale.ROOT); - type = Offset.Type.valueOf(opType); - break; - case VERSION: - version = parser.longValue(); - break; - case OPERATIONS: - XContentParserUtils.ensureExpectedToken( - XContentParser.Token.START_ARRAY, parser.currentToken(), parser); + case CONTEXT -> context = parser.text(); + case OFFSET -> offset = parser.longValue(); + case RESOURCE -> resource = parser.text(); + case TYPE -> type = Type.valueOf(parser.text().trim().toUpperCase(Locale.ROOT)); + case VERSION -> version = parser.longValue(); + case OPERATIONS -> { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); while (parser.nextToken() != XContentParser.Token.END_ARRAY) { operations.add(Operation.parse(parser)); } - break; - case PAYLOAD: - XContentParserUtils.ensureExpectedToken( - XContentParser.Token.START_OBJECT, parser.currentToken(), parser); - payload = Offset.parseObject(parser); - break; - default: - parser.skipChildren(); - break; + } + case PAYLOAD -> { + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + payload = parser.map(); + } + } + default -> parser.skipChildren(); } } } - - return new Offset(context, offset, resource, type, version, operations, payload); + return new Offset(context, offset != null ? offset : 0, resource, type, version != null ? version : 0, operations, payload); } /** - * @param parser - * @return - * @throws IOException - */ - private static Map parseObject(XContentParser parser) throws IOException { - Map result = new HashMap<>(); - - while (parser.nextToken() != XContentParser.Token.END_OBJECT) { - if (parser.currentToken() == XContentParser.Token.FIELD_NAME) { - String fieldName = parser.currentName(); - switch (parser.nextToken()) { - case START_OBJECT: - result.put(fieldName, Offset.parseObject(parser)); - break; - case START_ARRAY: - result.put(fieldName, Offset.parseArray(parser)); - break; - case VALUE_STRING: - result.put(fieldName, parser.text()); - break; - case VALUE_NUMBER: - result.put(fieldName, parser.numberValue()); - break; - case VALUE_BOOLEAN: - result.put(fieldName, parser.booleanValue()); - break; - case VALUE_NULL: - result.put(fieldName, null); - break; - default: - parser.skipChildren(); - break; - } - } - } - - return result; - } - - /** - * A method to parse arrays recursively + * Gets the unique identifier of the resource affected by this change. * - * @param parser an XContentParser containing an array - * @return the parsed list as a List - * @throws IOException rethrown from parseObject + * @return The resource ID string. */ - private static List parseArray(XContentParser parser) throws IOException { - List array = new ArrayList<>(); - - while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - switch (parser.currentToken()) { - case START_OBJECT: - array.add(Offset.parseObject(parser)); - break; - case START_ARRAY: - array.add(Offset.parseArray(parser)); - break; - case VALUE_STRING: - array.add(parser.text()); - break; - case VALUE_NUMBER: - array.add(parser.numberValue()); - break; - case VALUE_BOOLEAN: - array.add(parser.booleanValue()); - break; - case VALUE_NULL: - array.add(null); - break; - default: - parser.skipChildren(); - break; - } - } - - return array; - } + public String getResource() { return resource; } /** - * Returns the resource's name. + * Gets the type of modification performed. * - * @return the resource name. + * @return The {@link Type} enum value (CREATE, UPDATE, DELETE). */ - public String getResource() { - return this.resource; - } + public Type getType() { return type; } /** - * Getter for the type + * Gets the list of patch operations associated with this change. * - * @return the type as a String + * @return A list of {@link Operation} objects, or an empty list if none exist. */ - public Offset.Type getType() { - return this.type; - } + public List getOperations() { return operations; } /** - * Getter for the operations + * Gets the sequential offset ID of this change event. * - * @return the operations as a List of JsonPatch + * @return The offset value as a long. */ - public List getOperations() { - return this.operations; - } + public long getOffset() { return offset; } /** - * {@link Offset#offset} getter. + * Gets the full content payload of the resource. * - * @return the number identifier of the change. + * @return A Map representing the resource JSON, or null if not present. */ - public long getOffset() { - return this.offset; - } + public Map getPayload() { return payload; } /** - * Outputs an XContentBuilder object ready to be printed or manipulated + * Serializes this object into an {@link XContentBuilder}. * - * @param builder the received builder object - * @param params We don't really use this one - * @return an XContentBuilder object ready to be printed - * @throws IOException rethrown from Offset's toXContent + * @param builder The builder to write to. + * @param params Contextual parameters for the serialization. + * @return The builder instance for chaining. + * @throws IOException If an error occurs while writing to the builder. */ @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(CONTEXT, this.context); - builder.field(OFFSET, this.offset); - builder.field(RESOURCE, this.resource); - builder.field(TYPE, this.type); - builder.field(VERSION, this.version); - builder.startArray(OPERATIONS); - if (this.operations != null) { - for (Operation operation : this.operations) { - operation.toXContent(builder, ToXContentObject.EMPTY_PARAMS); - } + if (context != null) builder.field(CONTEXT, context); + builder.field(OFFSET, offset); + if (resource != null) builder.field(RESOURCE, resource); + if (type != null) builder.field(TYPE, type); + builder.field(VERSION, version); + if (operations != null) { + builder.startArray(OPERATIONS); + for (Operation op : operations) op.toXContent(builder, params); + builder.endArray(); } - builder.endArray(); - builder.field(PAYLOAD, this.payload); + if (payload != null) builder.field(PAYLOAD, payload); return builder.endObject(); } } diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/model/Operation.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/model/Operation.java index b4fd3b39..45684de2 100644 --- a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/model/Operation.java +++ b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/model/Operation.java @@ -27,17 +27,14 @@ import java.io.IOException; /** * Class representing a JSON Patch operation. - * - *

This class implements the ToXContentObject interface, allowing it to be serialized to XContent - * format. It is used to define operations that can be applied to a JSON document, such as adding, - * removing, or replacing elements. */ public class Operation implements ToXContentObject { public static final String OP = "op"; public static final String PATH = "path"; public static final String FROM = "from"; public static final String VALUE = "value"; - private final String op; // TODO replace with Operation.Type + + private final String op; private final String path; private final String from; private final Object value; @@ -45,26 +42,12 @@ public class Operation implements ToXContentObject { private static final Logger log = LogManager.getLogger(Operation.class); /** - * This enumeration represents the types of supported operations of the Content Manager plugin - * from the JSON Patch operations set. Check the RFC 6902. - */ - public enum Type { - TEST, - REMOVE, - ADD, - REPLACE, - MOVE, - COPY - } - - /** - * Constructor. + * Constructs a new JSON Patch Operation. * - * @param op Operation type (add, remove, replace). - * @param path Path to the element to be modified. - * @param from Source path for move operations. - * @param value Value to be added or replaced. + * @param op The operation to perform (e.g., "add", "replace", "remove"). + * @param path A JSON Pointer string indicating the location to perform the operation. + * @param from A JSON Pointer string indicating the location to move/copy from (optional, depends on 'op'). + * @param value The value to be added, replaced, or tested (optional, depends on 'op'). */ public Operation(String op, String path, String from, Object value) { this.op = op; @@ -74,72 +57,51 @@ public class Operation implements ToXContentObject { } /** - * Parses a JSON object to create a PatchOperation instance. + * Parses an XContent stream to create an {@code Operation} instance. * - * @param parser The XContentParser to parse the JSON object. - * @return A PatchOperation instance. - * @throws IllegalArgumentException if the JSON object is invalid. - * @throws IOException if an I/O error occurs during parsing. + * @param parser The {@link XContentParser} to read from. + * @return A populated {@code Operation} object. + * @throws IOException If an I/O error occurs or the content structure is invalid. */ - public static Operation parse(XContentParser parser) - throws IllegalArgumentException, IOException { + public static Operation parse(XContentParser parser) throws IOException { String op = null; String path = null; String from = null; Object value = null; - // Make sure we are at the start - XContentParserUtils.ensureExpectedToken( - XContentParser.Token.START_OBJECT, parser.currentToken(), parser); - // Iterate over the object and add each Offset object to changes array + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { - String fieldName = parser.currentName(); // Get key - parser.nextToken(); // Move to value + String fieldName = parser.currentName(); + parser.nextToken(); switch (fieldName) { - case OP: - op = parser.text(); - break; - case PATH: - path = parser.text(); - break; - // "from" is only used for "copy" and "move" operations, - // which are currently un-supported. - case FROM: - from = parser.text(); - break; - case VALUE: - // value can be anything. + case OP -> op = parser.text(); + case PATH -> path = parser.text(); + case FROM -> from = parser.text(); + case VALUE -> { switch (parser.currentToken()) { - case START_OBJECT: - value = parser.map(); - break; - case START_ARRAY: - value = parser.list(); - break; - case VALUE_STRING: - value = parser.text(); - break; - default: - parser.skipChildren(); - break; + case START_OBJECT -> value = parser.map(); + case START_ARRAY -> value = parser.list(); + case VALUE_STRING -> value = parser.text(); + case VALUE_NUMBER -> value = parser.numberValue(); + case VALUE_BOOLEAN -> value = parser.booleanValue(); + case VALUE_NULL -> value = null; + default -> parser.skipChildren(); } - break; - default: - log.error("Unknown field [{}] parsing a JSON Patch operation", fieldName); - parser.skipChildren(); - break; + } + default -> parser.skipChildren(); } } - return new Operation(op, path, from, value); } /** - * Outputs an XContentBuilder object ready to be printed or manipulated + * Serializes this operation into an {@link XContentBuilder}. * - * @param builder the received builder object - * @param params We don't really use this one - * @return an XContentBuilder object ready to be printed - * @throws IOException rethrown from Offset's toXContent + * @param builder The builder to write to. + * @param params Contextual parameters for the serialization. + * @return The builder instance for chaining. + * @throws IOException If an error occurs while writing to the builder. */ @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/service/ConsumerServiceImpl.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/service/ConsumerServiceImpl.java index b5bc8789..59fd8d9a 100644 --- a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/service/ConsumerServiceImpl.java +++ b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/service/ConsumerServiceImpl.java @@ -22,9 +22,6 @@ import java.util.concurrent.TimeoutException; public class ConsumerServiceImpl extends AbstractService implements ConsumerService { private static final Logger log = LogManager.getLogger(ConsumerServiceImpl.class); -// private static final String CONTEXT = "rules_development_0.0.1"; -// private static final String CONSUMER = "rules_consumer"; - private final String context; private final String consumer; private final ConsumersIndex consumerIndex; @@ -92,7 +89,6 @@ public class ConsumerServiceImpl extends AbstractService implements ConsumerServ * @return The initialized {@link LocalConsumer}, or null if persistence fails. */ public LocalConsumer setConsumer() { - // Default consumer. Initialize. LocalConsumer consumer = new LocalConsumer(this.context, this.consumer); try { diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/service/ContentUpdater.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/service/ContentUpdater.java deleted file mode 100644 index 06fc0fd8..00000000 --- a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/service/ContentUpdater.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright (C) 2024, Wazuh Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package com.wazuh.contentmanager.cti.catalog.service; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.wazuh.contentmanager.client.CTIClient; -import com.wazuh.contentmanager.cti.catalog.index.ContentIndex; -import com.wazuh.contentmanager.cti.catalog.index.ConsumersIndex; -import com.wazuh.contentmanager.cti.catalog.model.Changes; -import com.wazuh.contentmanager.cti.catalog.model.ConsumerInfo; -import com.wazuh.contentmanager.settings.PluginSettings; -import com.wazuh.contentmanager.utils.Privileged; -import com.wazuh.contentmanager.utils.VisibleForTesting; - -/** Class responsible for managing content updates by fetching and applying changes in chunks. */ -public class ContentUpdater { - private static final Logger log = LogManager.getLogger(ContentUpdater.class); - private final ConsumersIndex consumersIndex; - private final ContentIndex contentIndex; - private final CTIClient ctiClient; - private final Privileged privileged; - private final PluginSettings pluginSettings; - - /** Exception thrown by the Content Updater in case of errors. */ - public static class ContentUpdateException extends RuntimeException { - /** - * Constructor method - * - * @param message Message to be thrown - * @param cause Cause of the exception - */ - public ContentUpdateException(String message, Throwable cause) { - super(message, cause); - } - } - - /** - * Constructor. Mainly used for testing purposes. Dependency injection. - * - * @param ctiClient the CTIClient to interact with the CTI API. - * @param consumersIndex An object that handles context and consumer information. - * @param contentIndex An object that handles content index interactions. - */ - public ContentUpdater( - CTIClient ctiClient, - ConsumersIndex consumersIndex, - ContentIndex contentIndex, - Privileged privileged) { - this.consumersIndex = consumersIndex; - this.contentIndex = contentIndex; - this.ctiClient = ctiClient; - this.pluginSettings = PluginSettings.getInstance(); - this.privileged = privileged; - } - - /** - * This constructor is only used on tests. - * - * @param ctiClient mocked @CTIClient. - * @param contentIndex mocked @ContentIndex. - * @param pluginSettings mocked @PluginSettings. - */ - @VisibleForTesting - public ContentUpdater( - CTIClient ctiClient, - ConsumersIndex consumersIndex, - ContentIndex contentIndex, - Privileged privileged, - PluginSettings pluginSettings) { - this.consumersIndex = consumersIndex; - this.contentIndex = contentIndex; - this.ctiClient = ctiClient; - this.pluginSettings = pluginSettings; - this.privileged = privileged; - } - - /** - * Starts and orchestrates the process to update the content in the index with the latest changes - * from the CTI API. The content needs an update when the "offset" and the "lastOffset" values are - * different. In that case, the update process tries to bring the content up to date by querying - * the CTI API for a list of changes to apply to the content. These changes are applied - * sequentially. A maximum of {@link PluginSettings#MAX_CHANGES} changes are applied on each - * iteration. When the update is completed, the value of "offset" is updated and equal to - * "lastOffset" {@link ConsumersIndex#index(ConsumerInfo)}. If - * the update fails, the "offset" is set to 0 to force a recovery from a snapshot. - * - * @return true if the updates were successfully applied, false otherwise. - * @throws ContentUpdateException If there was an error fetching the changes. - */ - public boolean update() throws ContentUpdateException { - ConsumerInfo consumerInfo = - this.consumersIndex.get( - this.pluginSettings.getContextId(), this.pluginSettings.getConsumerId()); - long currentOffset = consumerInfo.getOffset(); - long lastOffset = consumerInfo.getLastOffset(); - - if (lastOffset == currentOffset) { - log.info("No updates available. Current offset ({}) is up to date.", currentOffset); - return true; - } - - log.info("Updating [{}]", ContentIndex.INDEX_NAME); - while (currentOffset < lastOffset) { - long nextOffset = - Math.min(currentOffset + this.pluginSettings.getMaximumChanges(), lastOffset); - Changes changes = this.privileged.getChanges(this.ctiClient, currentOffset, nextOffset); - log.debug("Fetched offsets from {} to {}", currentOffset, nextOffset); - - // Update halted. Save current state and exit. - if (changes == null) { - log.error("Updated interrupted on offset [{}]", currentOffset); - consumerInfo.setOffset(currentOffset); - this.consumersIndex.index(consumerInfo); - return false; - } - // Update failed. Force initialization from a snapshot. - if (!this.applyChanges(changes)) { - log.error("Updated finally failed on offset [{}]", currentOffset); - consumerInfo.setOffset(0); - consumerInfo.setLastOffset(0); - this.consumersIndex.index(consumerInfo); - return false; - } - - currentOffset = nextOffset; - log.debug("Update current offset to {}", currentOffset); - } - - // Update consumer info. - consumerInfo.setLastOffset(currentOffset); - this.consumersIndex.index(consumerInfo); - log.info("[{}] updated to offset [{}]", ContentIndex.INDEX_NAME, consumerInfo.getOffset()); - return true; - } - - /** - * Applies the fetched changes to the indexed content. - * - * @param changes Detected content changes. - * @return true if the changes were successfully applied, false otherwise. - */ - @VisibleForTesting - protected boolean applyChanges(Changes changes) { - try { - this.contentIndex.patch(changes); - return true; - } catch (RuntimeException e) { - return false; - } - } -} diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/service/UpdateService.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/service/UpdateService.java new file mode 100644 index 00000000..89123f59 --- /dev/null +++ b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/service/UpdateService.java @@ -0,0 +1,16 @@ +package com.wazuh.contentmanager.cti.catalog.service; + +/** + * Service interface for managing CTI snapshots. + * Defines the contract for initializing consumers from remote snapshots. + */ +public interface UpdateService { + + /** + * Performs a content update within the specified offset range. + * + * @param fromOffset The starting offset (exclusive) to fetch changes from. + * @param toOffset The target offset (inclusive) to reach. + */ + void update(long fromOffset, long toOffset); +} diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/service/UpdateServiceImpl.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/service/UpdateServiceImpl.java new file mode 100644 index 00000000..e3c0e286 --- /dev/null +++ b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/service/UpdateServiceImpl.java @@ -0,0 +1,201 @@ +/* + * Copyright (C) 2024, Wazuh Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package com.wazuh.contentmanager.cti.catalog.service; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.wazuh.contentmanager.cti.catalog.client.ApiClient; +import com.wazuh.contentmanager.cti.catalog.index.ConsumersIndex; +import com.wazuh.contentmanager.cti.catalog.index.ContentIndex; +import com.wazuh.contentmanager.cti.catalog.model.Changes; +import com.wazuh.contentmanager.cti.catalog.model.LocalConsumer; +import com.wazuh.contentmanager.cti.catalog.model.Offset; +import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.ResourceNotFoundException; +import org.opensearch.action.get.GetResponse; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; + +import java.util.Map; + +/** + * Service responsible for keeping the catalog content up-to-date. + */ +public class UpdateServiceImpl extends AbstractService implements UpdateService { + private static final Logger log = LogManager.getLogger(UpdateServiceImpl.class); + + private final ConsumersIndex consumersIndex; + private final Map indices; + private final String context; + private final String consumer; + private final Gson gson; + + /** + * Constructs a new UpdateServiceImpl. + * + * @param context The context string (e.g., catalog ID) for the consumer. + * @param consumer The name of the consumer entity. + * @param client The API client used to fetch changes. + * @param consumersIndex The index responsible for storing consumer state (offsets). + * @param indices A map of content type to {@link ContentIndex} managers. + */ + public UpdateServiceImpl(String context, String consumer, ApiClient client, ConsumersIndex consumersIndex, Map indices) { + if (this.client != null) { + this.client.close(); + } + + this.client = client; + this.consumersIndex = consumersIndex; + this.indices = indices; + this.context = context; + this.consumer = consumer; + this.gson = new Gson(); + } + + /** + * + * Performs a content update within the specified offset range. + * + * Implementation details: + * 1. Fetches the changes JSON from the API for the given range. + * 2. Parses the response into {@link Changes} and {@link Offset} objects. + * 3. Iterates through offsets, skipping specific internal resources ("policy"). + * 4. Delegates specific operations to {@link #applyOffset(Offset)}. + * 5. Updates the {@link LocalConsumer} record in the index with the last successfully applied offset. + * + * If an exception occurs, the consumer state is reset to prevent data corruption or stuck states. + */ + @Override + public void update(long fromOffset, long toOffset) { + log.info("Starting content update for consumer [{}] from [{}] to [{}]", this.consumer, fromOffset, toOffset); + try { + SimpleHttpResponse response = this.client.getChanges(this.context, this.consumer, fromOffset, toOffset); + if (response.getCode() != 200) { + log.error("Failed to fetch changes: {} {}", response.getCode(), response.getBodyText()); + return; + } + + // TODO: Study if it can be changed to Jackson Databind and if so apply the necessary changes + try (XContentParser parser = XContentType.JSON.xContent().createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + response.getBodyBytes())) { + Changes changes = Changes.parse(parser); + long lastAppliedOffset = fromOffset; + + for (Offset offset : changes.get()) { + if ("policy".equals(offset.getResource())) { + lastAppliedOffset = offset.getOffset(); + continue; + } + + this.applyOffset(offset); + lastAppliedOffset = offset.getOffset(); + } + + // Update consumer state + LocalConsumer consumer = new LocalConsumer(this.context, this.consumer); + + // Properly handle the GetResponse to check if the document exists before parsing + GetResponse getResponse = this.consumersIndex.getConsumer(this.context, this.consumer); + LocalConsumer current = (getResponse != null && getResponse.isExists()) ? + this.mapper.readValue(getResponse.getSourceAsString(), LocalConsumer.class) : consumer; + + LocalConsumer updated = new LocalConsumer(this.context, this.consumer, lastAppliedOffset, current.getRemoteOffset(), current.getSnapshotLink()); + this.consumersIndex.setConsumer(updated); + + log.info("Successfully updated consumer [{}] to offset [{}]", consumer, lastAppliedOffset); + } + } catch (Exception e) { + log.error("Error during content update: {}", e.getMessage(), e); + this.resetConsumer(); + } + } + + /** + * Applies a specific change offset to the appropriate content index. + * + * @param offset The {@link Offset} containing the type of change and data. + * @throws Exception If the indexing operation fails. + */ + private void applyOffset(Offset offset) throws Exception { + String id = offset.getResource(); + ContentIndex index; + + switch (offset.getType()) { + case CREATE: + if (offset.getPayload() != null) { + // TODO: Change the Offset logic to use JsonNode and use Jackson Object Mapper to obtain the payload + JsonObject payload = this.gson.toJsonTree(offset.getPayload()).getAsJsonObject(); + if (payload.has("type")) { + String type = payload.get("type").getAsString(); + index = this.indices.get(type); + if (index != null) { + index.create(id, payload); + } else { + log.warn("No index mapped for type [{}]", type); + } + } + } + break; + case UPDATE: + index = this.findIndexForId(id); + index.update(id, offset.getOperations()); + break; + case DELETE: + index = this.findIndexForId(id); + index.delete(id); + break; + default: + log.warn("Unsupported JSON patch operation [{}]", offset.getType()); + break; + } + } + + /** + * Locates the {@link ContentIndex} that contains the document with the specified ID. + * + * @param id The document ID to search for. + * @return The matching {@link ContentIndex}. + * @throws ResourceNotFoundException If no {@link ContentIndex} contains the document with the specified ID. + */ + private ContentIndex findIndexForId(String id) throws ResourceNotFoundException { + for (ContentIndex index : this.indices.values()) { + if (index.exists(id)) { + return index; + } + } + throw new ResourceNotFoundException("Document with ID '" + id + "' could not be found in any ContentIndex."); + } + + /** + * Resets the local consumer offset to 0. + */ + private void resetConsumer() { + log.info("Resetting consumer [{}] offset to 0 due to update failure.", this.consumer); + try { + LocalConsumer reset = new LocalConsumer(this.context, this.consumer, 0, 0, ""); + this.consumersIndex.setConsumer(reset); + } catch (Exception e) { + log.error("Failed to reset consumer: {}", e.getMessage()); + } + } +} diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/utils/JsonPatch.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/utils/JsonPatch.java index 15732aca..5b865c21 100644 --- a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/utils/JsonPatch.java +++ b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/cti/catalog/utils/JsonPatch.java @@ -16,6 +16,7 @@ */ package com.wazuh.contentmanager.cti.catalog.utils; +import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import org.apache.logging.log4j.LogManager; @@ -23,6 +24,9 @@ import org.apache.logging.log4j.Logger; import com.wazuh.contentmanager.cti.catalog.model.Operation; +import java.util.HashSet; +import java.util.Map; + /** * Utility class for applying JSON Patch operations to JSON documents. * @@ -43,9 +47,8 @@ public class JsonPatch { String path = operation.get(Operation.PATH).getAsString(); JsonElement value = operation.has(Operation.VALUE) ? operation.get(Operation.VALUE) : null; String from = - operation.has(Operation.FROM) ? operation.get(Operation.FROM).getAsString() : null; + operation.has(Operation.FROM) ? operation.get(Operation.FROM).getAsString() : null; - // TODO replace with Operation.Type switch (op) { case "add": JsonPatch.addOperation(document, path, value); @@ -79,6 +82,18 @@ public class JsonPatch { * @param value The value to be added. */ private static void addOperation(JsonObject document, String path, JsonElement value) { + if (path.isEmpty()) { + for (String key : new HashSet<>(document.keySet())) { + document.remove(key); + } + if (value != null && value.isJsonObject()) { + for (Map.Entry entry : value.getAsJsonObject().entrySet()) { + document.add(entry.getKey(), entry.getValue()); + } + } + return; + } + JsonElement target = JsonPatch.navigateToParent(document, path); if (target instanceof JsonObject) { String key = extractKeyFromPath(path); @@ -93,6 +108,13 @@ public class JsonPatch { * @param path The JSON path where the value should be removed. */ private static void removeOperation(JsonObject document, String path) { + if (path.isEmpty()) { + for (String key : new HashSet<>(document.keySet())) { + document.remove(key); + } + return; + } + JsonElement target = JsonPatch.navigateToParent(document, path); if (target instanceof JsonObject) { String key = extractKeyFromPath(path); @@ -120,7 +142,14 @@ public class JsonPatch { * @param toPath The JSON path where the value should be moved. */ private static void moveOperation(JsonObject document, String fromPath, String toPath) { - JsonElement value = navigateToParent(document, fromPath); + JsonElement parent = navigateToParent(document, fromPath); + if (parent == null || !parent.isJsonObject()) return; + + String key = extractKeyFromPath(fromPath); + if (!parent.getAsJsonObject().has(key)) return; + + JsonElement value = parent.getAsJsonObject().get(key); + JsonPatch.removeOperation(document, fromPath); JsonPatch.addOperation(document, toPath, value); } @@ -145,11 +174,8 @@ public class JsonPatch { return; } - // Get the actual value to copy JsonElement valueToCopy = parent.getAsJsonObject().get(fromKey); - // Deep copy to avoid reference issues JsonElement copiedValue = valueToCopy.deepCopy(); - // Now add the copied value to the new location JsonPatch.addOperation(document, toPath, copiedValue); } @@ -180,20 +206,28 @@ public class JsonPatch { */ private static JsonElement navigateToParent(JsonObject document, String path) { String[] parts = path.split("/"); - JsonElement current = document; - for (int i = 1; i < parts.length - 1; i++) { // Navigate to parent object - if (current.isJsonObject()) { - JsonObject obj = current.getAsJsonObject(); - if (!obj.has(parts[i])) { - return null; // Path does not exist + return java.util.Arrays.stream(parts, 1, parts.length - 1) + .reduce((JsonElement) document, (current, part) -> { + if (current == null) return null; + + if (current.isJsonObject()) { + JsonObject obj = current.getAsJsonObject(); + return obj.has(part) ? obj.get(part) : null; } - current = obj.get(parts[i]); - } else { - return null; // Trying to navigate inside a non-object - } - } - return current; + + if (current.isJsonArray()) { + try { + int index = Integer.parseInt(part); + JsonArray arr = current.getAsJsonArray(); + return (index >= 0 && index < arr.size()) ? arr.get(index) : null; + } catch (NumberFormatException e) { + return null; + } + } + + return null; + }, (a, b) -> a); } /** diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/jobscheduler/jobs/CatalogSyncJob.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/jobscheduler/jobs/CatalogSyncJob.java index c4adc368..dfdfc372 100644 --- a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/jobscheduler/jobs/CatalogSyncJob.java +++ b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/jobscheduler/jobs/CatalogSyncJob.java @@ -1,5 +1,6 @@ package com.wazuh.contentmanager.jobscheduler.jobs; +import com.wazuh.contentmanager.cti.catalog.client.ApiClient; import com.wazuh.contentmanager.cti.catalog.index.ConsumersIndex; import com.wazuh.contentmanager.cti.catalog.index.ContentIndex; import com.wazuh.contentmanager.cti.catalog.model.LocalConsumer; @@ -7,6 +8,7 @@ import com.wazuh.contentmanager.cti.catalog.model.RemoteConsumer; import com.wazuh.contentmanager.cti.catalog.service.ConsumerService; import com.wazuh.contentmanager.cti.catalog.service.ConsumerServiceImpl; import com.wazuh.contentmanager.cti.catalog.service.SnapshotServiceImpl; +import com.wazuh.contentmanager.cti.catalog.service.UpdateServiceImpl; import com.wazuh.contentmanager.jobscheduler.JobExecutor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -21,6 +23,7 @@ import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.concurrent.Semaphore; /** * Job responsible for executing the synchronization logic for Rules and Decoders consumers. @@ -31,6 +34,9 @@ public class CatalogSyncJob implements JobExecutor { // Identifier used to route this specific job type public static final String JOB_TYPE = "consumer-sync-task"; + // Semaphore to control concurrency + private final Semaphore semaphore = new Semaphore(1); + private final Client client; private final ConsumersIndex consumersIndex; private final Environment environment; @@ -52,24 +58,71 @@ public class CatalogSyncJob implements JobExecutor { } /** - * Triggers the execution of the synchronization job. + * Triggers the execution of the synchronization job via the Job Scheduler. * * @param context The execution context provided by the Job Scheduler, containing metadata like the Job ID. */ @Override public void execute(JobExecutionContext context) { + if (!this.semaphore.tryAcquire()) { + log.warn("CatalogSyncJob (ID: {}) skipped because synchronization is already running.", context.getJobId()); + return; + } + // Offload execution to the generic thread pool to allow blocking operations this.threadPool.generic().execute(() -> { try { log.info("Executing Consumer Sync Job (ID: {})", context.getJobId()); - this.rulesConsumer(); - this.decodersConsumer(); + this.performSynchronization(); } catch (Exception e) { log.error("Error executing Consumer Sync Job (ID: {}): {}", context.getJobId(), e.getMessage(), e); + } finally { + this.semaphore.release(); } }); } + /** + * Checks if the synchronization job is currently running. + * + * @return true if running, false otherwise. + */ + public boolean isRunning() { + return this.semaphore.availablePermits() == 0; + } + + /** + * Attempts to trigger the synchronization process manually. + * + * @return true if the job was successfully started, false if it is already running. + */ + public boolean trigger() { + if (!this.semaphore.tryAcquire()) { + log.warn("Attempted to trigger CatalogSyncJob manually while it is already running."); + return false; + } + this.threadPool.generic().execute(() -> { + try { + log.info("Executing Manually Triggered Consumer Sync Job"); + this.performSynchronization(); + } catch (Exception e) { + log.error("Error executing Manual Consumer Sync Job: {}", e.getMessage(), e); + } finally { + this.semaphore.release(); + } + }); + + return true; + } + + /** + * Centralized synchronization logic used by both execute() and trigger(). + */ + private void performSynchronization() { + this.rulesConsumer(); + this.decodersConsumer(); + } + /** * Orchestrates the synchronization process specifically for the Rules consumer. */ @@ -137,7 +190,7 @@ public class CatalogSyncJob implements JobExecutor { /** * The core logic for synchronizing consumer services. - * + *

* This method performs the following actions: * 1. Retrieve the Local and Remote consumer metadata. * 2. Iterate through the requested mappings to check if indices exist. @@ -157,12 +210,14 @@ public class CatalogSyncJob implements JobExecutor { RemoteConsumer remoteConsumer = consumerService.getRemoteConsumer(); List indices = new ArrayList<>(); + Map indicesMap = new HashMap<>(); for (Map.Entry entry : mappings.entrySet()) { String indexName = this.getIndexName(context, consumer, entry.getKey()); String alias = aliases.get(entry.getKey()); ContentIndex index = new ContentIndex(this.client, indexName, entry.getValue(), alias); indices.add(index); + indicesMap.put(entry.getKey(), index); // Check if index exists to avoid creation exception boolean indexExists = this.client.admin().indices().prepareExists(indexName).get().isExists(); @@ -190,7 +245,16 @@ public class CatalogSyncJob implements JobExecutor { ); snapshotService.initialize(remoteConsumer); } else if (remoteConsumer != null && localConsumer.getLocalOffset() != remoteConsumer.getOffset()) { - // TODO: Implement offset based update process + log.info("Starting offset-based update for consumer [{}]", consumer); + UpdateServiceImpl updateService = new UpdateServiceImpl( + context, + consumer, + new ApiClient(), + this.consumersIndex, + indicesMap + ); + updateService.update(localConsumer.getLocalOffset(), remoteConsumer.getOffset()); + updateService.close(); } } } diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/rest/services/RestDeleteSubscriptionAction.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/rest/services/RestDeleteSubscriptionAction.java index 7e510b56..aa9dd5c8 100644 --- a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/rest/services/RestDeleteSubscriptionAction.java +++ b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/rest/services/RestDeleteSubscriptionAction.java @@ -1,9 +1,9 @@ package com.wazuh.contentmanager.rest.services; -import com.wazuh.contentmanager.ContentManagerPlugin; import com.wazuh.contentmanager.cti.console.CtiConsole; import com.wazuh.contentmanager.cti.console.model.Token; import com.wazuh.contentmanager.rest.model.RestResponse; +import com.wazuh.contentmanager.settings.PluginSettings; import org.opensearch.transport.client.node.NodeClient; import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.BaseRestHandler; @@ -57,11 +57,11 @@ public class RestDeleteSubscriptionAction extends BaseRestHandler { @Override public List routes() { return List.of( - new NamedRoute.Builder() - .path(ContentManagerPlugin.SUBSCRIPTION_URI) - .method(DELETE) - .uniqueName(ENDPOINT_UNIQUE_NAME) - .build() + new NamedRoute.Builder() + .path(PluginSettings.SUBSCRIPTION_URI) + .method(DELETE) + .uniqueName(ENDPOINT_UNIQUE_NAME) + .build() ); } diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/rest/services/RestGetSubscriptionAction.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/rest/services/RestGetSubscriptionAction.java index 25aba754..7620dfae 100644 --- a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/rest/services/RestGetSubscriptionAction.java +++ b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/rest/services/RestGetSubscriptionAction.java @@ -1,9 +1,9 @@ package com.wazuh.contentmanager.rest.services; -import com.wazuh.contentmanager.ContentManagerPlugin; import com.wazuh.contentmanager.cti.console.CtiConsole; import com.wazuh.contentmanager.cti.console.model.Token; import com.wazuh.contentmanager.rest.model.RestResponse; +import com.wazuh.contentmanager.settings.PluginSettings; import org.opensearch.transport.client.node.NodeClient; import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.BaseRestHandler; @@ -57,11 +57,11 @@ public class RestGetSubscriptionAction extends BaseRestHandler { @Override public List routes() { return List.of( - new NamedRoute.Builder() - .path(ContentManagerPlugin.SUBSCRIPTION_URI) - .method(GET) - .uniqueName(ENDPOINT_UNIQUE_NAME) - .build() + new NamedRoute.Builder() + .path(PluginSettings.SUBSCRIPTION_URI) + .method(GET) + .uniqueName(ENDPOINT_UNIQUE_NAME) + .build() ); } diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/rest/services/RestPostSubscriptionAction.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/rest/services/RestPostSubscriptionAction.java index b4319e79..bbf2c062 100644 --- a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/rest/services/RestPostSubscriptionAction.java +++ b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/rest/services/RestPostSubscriptionAction.java @@ -1,9 +1,9 @@ package com.wazuh.contentmanager.rest.services; -import com.wazuh.contentmanager.ContentManagerPlugin; import com.wazuh.contentmanager.cti.console.CtiConsole; import com.wazuh.contentmanager.cti.console.model.Subscription; import com.wazuh.contentmanager.rest.model.RestResponse; +import com.wazuh.contentmanager.settings.PluginSettings; import org.opensearch.transport.client.node.NodeClient; import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.BaseRestHandler; @@ -57,11 +57,11 @@ public class RestPostSubscriptionAction extends BaseRestHandler { @Override public List routes() { return List.of( - new NamedRoute.Builder() - .path(ContentManagerPlugin.SUBSCRIPTION_URI) - .method(POST) - .uniqueName(ENDPOINT_UNIQUE_NAME) - .build() + new NamedRoute.Builder() + .path(PluginSettings.SUBSCRIPTION_URI) + .method(POST) + .uniqueName(ENDPOINT_UNIQUE_NAME) + .build() ); } diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/rest/services/RestPostUpdateAction.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/rest/services/RestPostUpdateAction.java index b1494b58..19392181 100644 --- a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/rest/services/RestPostUpdateAction.java +++ b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/rest/services/RestPostUpdateAction.java @@ -1,14 +1,14 @@ package com.wazuh.contentmanager.rest.services; -import com.wazuh.contentmanager.ContentManagerPlugin; +import com.wazuh.contentmanager.jobscheduler.jobs.CatalogSyncJob; import com.wazuh.contentmanager.cti.console.CtiConsole; -import com.wazuh.contentmanager.cti.console.model.Subscription; import com.wazuh.contentmanager.rest.model.RestResponse; +import com.wazuh.contentmanager.settings.PluginSettings; +import org.opensearch.rest.NamedRoute; import org.opensearch.transport.client.node.NodeClient; import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; -import org.opensearch.rest.NamedRoute; import org.opensearch.rest.RestRequest; import java.io.IOException; @@ -38,14 +38,11 @@ public class RestPostUpdateAction extends BaseRestHandler { private static final String ENDPOINT_NAME = "content_manager_subscription_update"; private static final String ENDPOINT_UNIQUE_NAME = "plugin:content_manager/subscription_update"; private final CtiConsole ctiConsole; + private final CatalogSyncJob catalogSyncJob; - /** - * Construct the update REST handler. - * - * @param console the CTI console used to check subscription state and trigger updates - */ - public RestPostUpdateAction(CtiConsole console) { + public RestPostUpdateAction(CtiConsole console, CatalogSyncJob catalogSyncJob) { this.ctiConsole = console; + this.catalogSyncJob = catalogSyncJob; } /** @@ -64,7 +61,7 @@ public class RestPostUpdateAction extends BaseRestHandler { return List.of( // POST /_plugins/content-manager/update new NamedRoute.Builder() - .path(ContentManagerPlugin.UPDATE_URI) + .path(PluginSettings.UPDATE_URI) .method(POST) .uniqueName(ENDPOINT_UNIQUE_NAME) .build() @@ -75,7 +72,7 @@ public class RestPostUpdateAction extends BaseRestHandler { * Prepare the request by returning a consumer that executes the update operation. * * @param request the incoming REST request - * @param client the node client + * @param client the node client * @return a consumer that executes the update operation */ @Override @@ -103,8 +100,7 @@ public class RestPostUpdateAction extends BaseRestHandler { } // 2. Conflict Check (409 Conflict) - // TODO: Implement actual concurrency control - if (1 == 2) { + if (this.catalogSyncJob.isRunning()) { RestResponse error = new RestResponse( "An update operation is already in progress. Please wait for it to complete.", RestStatus.CONFLICT.getStatus() @@ -119,9 +115,10 @@ public class RestPostUpdateAction extends BaseRestHandler { * - X-RateLimit-Reset: Unix timestamp when the rate limit window resets */ - // TODO: Add actual update logic + // 4. Update Accepted (202 ACCEPTED) + this.catalogSyncJob.trigger(); RestResponse response = new RestResponse("Update accepted", RestStatus.ACCEPTED.getStatus()); - return new BytesRestResponse(RestStatus.ACCEPTED, response.toXContent()); + return new BytesRestResponse(RestStatus.ACCEPTED, response.toXContent()); } catch (Exception e) { RestResponse error = new RestResponse( e.getMessage() != null ? e.getMessage() : "An unexpected error occurred while processing your request.", diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/settings/PluginSettings.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/settings/PluginSettings.java index 73998ca6..6ab56786 100644 --- a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/settings/PluginSettings.java +++ b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/settings/PluginSettings.java @@ -18,203 +18,104 @@ package com.wazuh.contentmanager.settings; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.settings.SecureSetting; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; -import org.opensearch.core.common.settings.SecureString; -import com.wazuh.contentmanager.utils.ClusterInfo; import reactor.util.annotation.NonNull; /** This class encapsulates configuration settings and constants for the Content Manager plugin. */ public class PluginSettings { private static final Logger log = LogManager.getLogger(PluginSettings.class); - /** Settings default values */ - private static final String DEFAULT_CONSUMER_ID = "rules_consumer"; + // Rest API endpoints + public static final String PLUGINS_BASE_URI = "/_plugins/content-manager"; + public static final String SUBSCRIPTION_URI = PLUGINS_BASE_URI + "/subscription"; + public static final String UPDATE_URI = PLUGINS_BASE_URI + "/update"; - private static final String DEFAULT_CONTEXT_ID = "rules_development_0.0.1"; - private static final int DEFAULT_CTI_MAX_ATTEMPTS = 3; - private static final int DEFAULT_CTI_SLEEP_TIME = 60; + /** Settings default values */ private static final int DEFAULT_MAX_ITEMS_PER_BULK = 25; private static final int DEFAULT_MAX_CONCURRENT_BULKS = 5; private static final int DEFAULT_CLIENT_TIMEOUT = 10; - private static final int DEFAULT_MAX_CHANGES = 1000; - private static final int DEFAULT_MAX_DOCS = 1000; - private static final int DEFAULT_JOB_SCHEDULE = 1; + private static final int DEFAULT_CATALOG_SYNC_INTERVAL = 60; /** Singleton instance. */ private static PluginSettings INSTANCE; /** Base Wazuh CTI URL */ - // https://cti-pre.wazuh.com/api/v1/catalog/contexts/rules_development_0.0.1/consumers/rules_consumer public static final String CTI_URL = "https://cti-pre.wazuh.com"; /** The CTI API URL from the configuration file */ - // TODO: Change to the new CTI_API_URL public static final Setting CTI_API_URL = - Setting.simpleString( - "content_manager.cti.api", - CTI_URL + "/api/v1", - Setting.Property.NodeScope, - Setting.Property.Filtered); - - /** Content Manager CTI API consumer id/name */ - public static final Setting CONSUMER_ID = - Setting.simpleString( - "content_manager.cti.consumer", - DEFAULT_CONSUMER_ID, - Setting.Property.NodeScope, - Setting.Property.Filtered); - - /** Content Manager CTI API context id/name */ - public static final Setting CONTEXT_ID = - Setting.simpleString( - "content_manager.cti.context", - DEFAULT_CONTEXT_ID, - Setting.Property.NodeScope, - Setting.Property.Filtered); - - /** The maximum number of retries for a request to the CTI Client */ - public static final Setting CTI_CLIENT_MAX_ATTEMPTS = - Setting.intSetting( - "content_manager.cti.client.max_attempts", - DEFAULT_CTI_MAX_ATTEMPTS, - 2, - 5, - Setting.Property.NodeScope, - Setting.Property.Filtered); - - /** - * Specifies the initial wait time in seconds for the first retry to the CTI API after receiving a - * 'Too Many Requests' response or other retry conditions. This value also serves as the base for - * calculating increased wait times for subsequent retries, helping to manage request rates and - * prevent server overload. - */ - public static final Setting CTI_CLIENT_SLEEP_TIME = - Setting.intSetting( - "content_manager.cti.client.sleep_time", - DEFAULT_CTI_SLEEP_TIME, - 20, - 100, - Setting.Property.NodeScope, - Setting.Property.Filtered); + Setting.simpleString( + "content_manager.cti.api", + CTI_URL + "/api/v1", + Setting.Property.NodeScope, + Setting.Property.Filtered); /** * The maximum number of elements that are included in a bulk request during the initialization * from a snapshot. */ public static final Setting MAX_ITEMS_PER_BULK = - Setting.intSetting( - "content_manager.max_items_per_bulk", - DEFAULT_MAX_ITEMS_PER_BULK, - 10, - 25, - Setting.Property.NodeScope, - Setting.Property.Filtered); + Setting.intSetting( + "content_manager.max_items_per_bulk", + DEFAULT_MAX_ITEMS_PER_BULK, + 10, + 25, + Setting.Property.NodeScope, + Setting.Property.Filtered); /** * The maximum number of co-existing bulk operations during the initialization from a snapshot. */ public static final Setting MAX_CONCURRENT_BULKS = - Setting.intSetting( - "content_manager.max_concurrent_bulks", - DEFAULT_MAX_CONCURRENT_BULKS, - 1, - 5, - Setting.Property.NodeScope, - Setting.Property.Filtered); + Setting.intSetting( + "content_manager.max_concurrent_bulks", + DEFAULT_MAX_CONCURRENT_BULKS, + 1, + 5, + Setting.Property.NodeScope, + Setting.Property.Filtered); /** Timeout of indexing operations */ public static final Setting CLIENT_TIMEOUT = - Setting.longSetting( - "content_manager.client.timeout", - DEFAULT_CLIENT_TIMEOUT, - 10, - 50, - Setting.Property.NodeScope, - Setting.Property.Filtered); + Setting.longSetting( + "content_manager.client.timeout", + DEFAULT_CLIENT_TIMEOUT, + 10, + 50, + Setting.Property.NodeScope, + Setting.Property.Filtered); - /** The maximum number of changes to be fetched and applied during the update of the content. */ - public static final Setting MAX_CHANGES = - Setting.longSetting( - "content_manager.max_changes", - DEFAULT_MAX_CHANGES, - 10, - 1000, - Setting.Property.NodeScope, - Setting.Property.Filtered); - - /** Maximum number of documents processed per indexing job. */ - public static final Setting JOB_MAX_DOCS = - Setting.intSetting( - "content_manager.job.max_docs", - DEFAULT_MAX_DOCS, - 5, - 100000, - Setting.Property.NodeScope, - Setting.Property.Filtered); - - /** Interval in minutes between each scheduled job execution. */ - public static final Setting JOB_SCHEDULE = - Setting.intSetting( - "content_manager.job.schedule", - DEFAULT_JOB_SCHEDULE, - 1, - 10, - Setting.Property.NodeScope, - Setting.Property.Filtered); + /** + * The interval in minutes for the catalog synchronization job. + */ + public static final Setting CATALOG_SYNC_INTERVAL = + Setting.intSetting( + "content_manager.catalog.sync_interval", + DEFAULT_CATALOG_SYNC_INTERVAL, + 1, + 1440, + Setting.Property.NodeScope, + Setting.Property.Filtered); private final String ctiBaseUrl; - private final String consumerId; - private final String contextId; - private final ClusterService clusterService; - - private final int ctiClientMaxAttempts; - private final int ctiClientSleepTime; private final int maximumItemsPerBulk; private final int maximumConcurrentBulks; private final long clientTimeout; - private final long maximumChanges; - private final int jobMaximumDocuments; - private final int jobSchedule; + private final int catalogSyncInterval; /** * Private default constructor * * @param settings as obtained in createComponents. */ - private PluginSettings(@NonNull final Settings settings, ClusterService clusterService) { + private PluginSettings(@NonNull final Settings settings) { this.ctiBaseUrl = CTI_API_URL.get(settings); - - if (validateConsumerId(CONSUMER_ID.get(settings))) { - this.consumerId = CONSUMER_ID.get(settings); - } else { - this.consumerId = DEFAULT_CONSUMER_ID; - log.error( - "Setting [content_manager.cti.consumer] must follow the patter 'vd_{Number}.{Number}.{Number}'. Falling back to the default value [{}]", - DEFAULT_CONSUMER_ID); - } - - if (validateContextId(CONTEXT_ID.get(settings))) { - this.contextId = CONTEXT_ID.get(settings); - } else { - this.contextId = DEFAULT_CONTEXT_ID; - log.error( - "Setting [content_manager.cti.context] must follow the patter 'vd_{Number}.{Number}.{Number}'. Falling back to the default value [{}]", - DEFAULT_CONTEXT_ID); - } - - this.clusterService = clusterService; - this.ctiClientMaxAttempts = CTI_CLIENT_MAX_ATTEMPTS.get(settings); - this.ctiClientSleepTime = CTI_CLIENT_SLEEP_TIME.get(settings); this.maximumItemsPerBulk = MAX_ITEMS_PER_BULK.get(settings); this.maximumConcurrentBulks = MAX_CONCURRENT_BULKS.get(settings); this.clientTimeout = CLIENT_TIMEOUT.get(settings); - this.maximumChanges = MAX_CHANGES.get(settings); - this.jobMaximumDocuments = JOB_MAX_DOCS.get(settings); - this.jobSchedule = JOB_SCHEDULE.get(settings); + this.catalogSyncInterval = CATALOG_SYNC_INTERVAL.get(settings); log.debug("Settings.loaded: {}", this.toString()); } @@ -222,13 +123,12 @@ public class PluginSettings { * Singleton instance accessor. Initializes the settings * * @param settings as obtained in createComponents. - * @param clusterService service to get cluster stats. * @return {@link PluginSettings#INSTANCE} */ public static synchronized PluginSettings getInstance( - @NonNull final Settings settings, ClusterService clusterService) { + @NonNull final Settings settings) { if (INSTANCE == null) { - INSTANCE = new PluginSettings(settings, clusterService); + INSTANCE = new PluginSettings(settings); } return INSTANCE; } @@ -238,7 +138,7 @@ public class PluginSettings { * * @return {@link PluginSettings#INSTANCE} * @throws IllegalStateException if the instance has not been initialized - * @see PluginSettings#getInstance(Settings, ClusterService) + * @see PluginSettings#getInstance(Settings) */ public static synchronized PluginSettings getInstance() { if (PluginSettings.INSTANCE == null) { @@ -256,43 +156,6 @@ public class PluginSettings { return this.ctiBaseUrl; } - /** - * Retrieves the consumer ID. - * - * @return a string representing the consumer ID. - */ - public String getConsumerId() { - return this.consumerId; - } - - /** - * Retrieves the context ID. - * - * @return a String representing the context ID. - */ - public String getContextId() { - return this.contextId; - } - - /* - * Retrieves the maximum number of retry attempts allowed for the CTI client. - * - * @return an Integer representing the maximum number of retry attempts. - */ - public Integer getCtiClientMaxAttempts() { - return this.ctiClientMaxAttempts; - } - - /** - * Retrieves the wait time used by the CTI client after receiving a 'too many requests' response. - * This attribute helps calculate the delay before retrying the request. - * - * @return an Integer representing the duration of the sleep time for the CTI client. - */ - public Integer getCtiClientSleepTime() { - return this.ctiClientSleepTime; - } - /** * Retrieves the maximum number of documents that can be indexed. * @@ -321,97 +184,31 @@ public class PluginSettings { } /** - * Retrieves the maximum number of changes to be fetched and applied during the update of the - * content. + * Retrieves the interval in minutes for the catalog synchronization job. * - * @return a Long representing the maximum number of changes. + * @return an Integer representing the interval in minutes. */ - public Long getMaximumChanges() { - return this.maximumChanges; - } - - /** - * Retrieves the maximum number of documents allowed for a job. - * - * @return an Integer representing the maximum number of documents that can be processed in a - * single job. - */ - public Integer getJobMaximumDocuments() { - return this.jobMaximumDocuments; - } - - /** - * Retrieves the job schedule interval. - * - * @return an Integer representing the job execution interval in minutes - */ - public Integer getJobSchedule() { - return this.jobSchedule; - } - - /** - * Validates the given consumer ID against a predefined format. The consumer ID should match the - * pattern {@code vd_<1-2 digits>.<1-2 digits>.<1-2 digits>}. - * - * @param consumerId the consumer ID to validate - * @return true if the consumer ID matches the expected format, false otherwise - */ - public static Boolean validateConsumerId(String consumerId) { - String regex = "vd_\\d{1,2}\\.\\d{1,2}\\.\\d{1,2}"; - - // Ensure the context id have the correct format - return consumerId.matches(regex); - } - - /** - * Validates the given context ID against a predefined format. The context ID should match the - * pattern {@code vd_<1-2 digits>.<1-2 digits>.<1-2 digits>}. - * - * @param contextId the context ID to validate - * @return true if the context ID matches the expected format, false otherwise - */ - public static Boolean validateContextId(String contextId) { - String regex = "vd_\\d{1,2}\\.\\d{1,2}\\.\\d{1,2}"; - - // Ensure the context id have the correct format - return contextId.matches(regex); + public Integer getCatalogSyncInterval() { + return this.catalogSyncInterval; } @Override public String toString() { return "{" - + "ctiBaseUrl='" - + this.ctiBaseUrl - + "', " - + "consumerId='" - + this.consumerId - + "', " - + "contextId='" - + this.contextId - + "', " - + "ctiClientMaxAttempts=" - + this.ctiClientMaxAttempts - + ", " - + "ctiClientSleepTime=" - + this.ctiClientSleepTime - + ", " - + "maximumItemsPerBulk=" - + this.maximumItemsPerBulk - + ", " - + "maximumConcurrentBulks=" - + this.maximumConcurrentBulks - + ", " - + "clientTimeout=" - + this.clientTimeout - + ", " - + "maximumChanges=" - + this.maximumChanges - + ", " - + "jobMaximumDocuments=" - + this.jobMaximumDocuments - + ", " - + "jobSchedule=" - + this.jobSchedule - + "}"; + + "ctiBaseUrl='" + + this.ctiBaseUrl + + "', " + + "maximumItemsPerBulk=" + + this.maximumItemsPerBulk + + ", " + + "maximumConcurrentBulks=" + + this.maximumConcurrentBulks + + ", " + + "clientTimeout=" + + this.clientTimeout + + ", " + + "catalogSyncInterval=" + + this.catalogSyncInterval + + "}"; } } diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/utils/ClusterInfo.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/utils/ClusterInfo.java index 8795b88e..d40b81db 100644 --- a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/utils/ClusterInfo.java +++ b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/utils/ClusterInfo.java @@ -21,46 +21,12 @@ import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.opensearch.transport.client.Client; import org.opensearch.cluster.health.ClusterHealthStatus; -import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.settings.Settings; - -import java.util.Locale; /** * ClusterInfo provides utility methods for retrieving cluster-related information, such as * security settings and the cluster base URL. */ public class ClusterInfo { - /** - * Checks if the OpenSearch cluster is using HTTPS for communication. - * - * @param clusterService The ClusterService instance providing cluster settings. - * @return true if HTTPS is enabled, false otherwise. - */ - public static boolean isHttpsEnabled(ClusterService clusterService) { - Settings settings = clusterService.getSettings(); - - // Check if security plugins have HTTPS enabled - return settings.getAsBoolean("plugins.security.ssl.http.enabled", false) - || settings.getAsBoolean("xpack.security.http.ssl.enabled", false); - } - - /** - * Retrieves the base URL of the OpenSearch cluster with the appropriate protocol (HTTP/HTTPS). - * - * @param clusterService The ClusterService instance providing cluster state and nodes. - * @return The cluster base URL in the format "http(s)://[IP]:[PORT]". - */ - public static String getClusterBaseUrl(ClusterService clusterService) { - String protocol = ClusterInfo.isHttpsEnabled(clusterService) ? "https" : "http"; - String host = "127.0.0.1"; - String port = "9200"; - if (clusterService.state().nodes().getClusterManagerNode() != null) { - host = clusterService.getSettings().get("network.host", host); - port = clusterService.getSettings().get("http.port", port); - } - return String.format(Locale.ROOT, "%s://%s:%s", protocol, host, port); - } /** * Checks if a given index is ready for operations. diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/utils/Privileged.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/utils/Privileged.java deleted file mode 100644 index 5608475c..00000000 --- a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/utils/Privileged.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (C) 2024, Wazuh Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package com.wazuh.contentmanager.utils; - -import java.security.AccessController; - -import com.wazuh.contentmanager.client.CTIClient; -import com.wazuh.contentmanager.cti.catalog.model.Changes; - -/** Privileged utility class for executing privileged HTTP requests. */ -public class Privileged { - - /** - * Executes an HTTP request with elevated privileges. - * - * @param request The Action to be executed with privileged permissions - * @param A privileged action that performs the HTTP request. - * @return The return value resulting from the request execution. - */ - @SuppressWarnings("removal") - public T doPrivilegedRequest(java.security.PrivilegedAction request) { - return AccessController.doPrivileged(request); - } - - /** - * Fetches the context changes between a given offset range from the CTI API. - * - * @param fromOffset Starting offset (inclusive). - * @param toOffset Ending offset (exclusive). - * @return ContextChanges object containing the changes. - */ - public Changes getChanges(CTIClient client, long fromOffset, long toOffset) { - return this.doPrivilegedRequest(() -> client.getChanges(fromOffset, toOffset, false)); - } -} diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/utils/VisibleForTesting.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/utils/VisibleForTesting.java deleted file mode 100644 index aca781ba..00000000 --- a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/utils/VisibleForTesting.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2024, Wazuh Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package com.wazuh.contentmanager.utils; - -/** - * Annotation to indicate that a method, field, or class is more visible than necessary strictly for - * testing purposes. - * - *

This annotation serves as a documentation aid to highlight that the increased visibility of an - * otherwise private or package-private member is intentional for unit testing. - * - *

Usage example: - * - * 

{@code
- * @VisibleForTesting
- * void someTestableMethod() {
- *     // Implementation
- * }
- * }
- */ -public @interface VisibleForTesting {} diff --git a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/utils/XContentUtils.java b/plugins/content-manager/src/main/java/com/wazuh/contentmanager/utils/XContentUtils.java deleted file mode 100644 index 5fd312d6..00000000 --- a/plugins/content-manager/src/main/java/com/wazuh/contentmanager/utils/XContentUtils.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (C) 2024, Wazuh Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package com.wazuh.contentmanager.utils; - -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.xcontent.DeprecationHandler; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.xcontent.ToXContent; -import org.opensearch.core.xcontent.ToXContentObject; -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.core.xcontent.XContentParser; - -import java.io.IOException; - -/** - * Class wrapping util functions for the {@code XContentParser} and {@code XContentBuilder} class - * methods. - */ -public class XContentUtils { - - /** - * Converts a ToXContentObject to a JsonObject. - * - * @param content the ToXContentObject to convert. - * @return the converted JsonObject. - * @throws IOException if an error occurs during conversion. - */ - public static JsonObject xContentObjectToJson(ToXContentObject content) throws IOException { - XContentBuilder builder = XContentFactory.jsonBuilder(); - content.toXContent(builder, ToXContent.EMPTY_PARAMS); - return JsonParser.parseString(builder.toString()).getAsJsonObject(); - } - - /** - * Creates a new XContentParser for the given JSON object. - * - * @param content the JsonObject to create the parser for. - * @return an XContentParser for the given JSON object. - * @throws IOException if an error occurs during creation of the parser. - */ - public static XContentParser createJSONParser(JsonObject content) throws IOException { - return XContentType.JSON - .xContent() - .createParser( - NamedXContentRegistry.EMPTY, - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, - content.toString()); - } - - /** - * Creates a new XContentParser for the given JSON byte array representation. - * - * @param content the byte array representing the JSON content to create the parser for. - * @return an XContentParser for the given JSON object. - * @throws IOException if an error occurs during creation of the parser. - */ - public static XContentParser createJSONParser(byte[] content) throws IOException { - return XContentType.JSON - .xContent() - .createParser( - NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, content); - } -} diff --git a/plugins/content-manager/src/main/resources/mappings/cti-decoders-integrations-mappings.json b/plugins/content-manager/src/main/resources/mappings/cti-decoders-integrations-mappings.json index f6b34352..b77580bf 100644 --- a/plugins/content-manager/src/main/resources/mappings/cti-decoders-integrations-mappings.json +++ b/plugins/content-manager/src/main/resources/mappings/cti-decoders-integrations-mappings.json @@ -1,5 +1,5 @@ { - "dynamic": "strict", + "dynamic": "true", "properties": { "type": { "type": "keyword" @@ -22,8 +22,7 @@ "type": "keyword" }, "date": { - "type": "date", - "format": "yyyy-MM-dd" + "type": "date" }, "documentation": { "type": "keyword" @@ -43,4 +42,4 @@ } } } -} \ No newline at end of file +} diff --git a/plugins/content-manager/src/main/resources/mappings/cti-decoders-mappings.json b/plugins/content-manager/src/main/resources/mappings/cti-decoders-mappings.json index 12a35059..7815b480 100644 --- a/plugins/content-manager/src/main/resources/mappings/cti-decoders-mappings.json +++ b/plugins/content-manager/src/main/resources/mappings/cti-decoders-mappings.json @@ -1,5 +1,5 @@ { - "dynamic": "strict_allow_templates", + "dynamic": "true", "dynamic_templates": [ { "parse_fields": { diff --git a/plugins/content-manager/src/main/resources/mappings/cti-kvdbs-mappings.json b/plugins/content-manager/src/main/resources/mappings/cti-kvdbs-mappings.json index 33a6f642..7c077ca9 100644 --- a/plugins/content-manager/src/main/resources/mappings/cti-kvdbs-mappings.json +++ b/plugins/content-manager/src/main/resources/mappings/cti-kvdbs-mappings.json @@ -1,5 +1,5 @@ { - "dynamic": "strict", + "dynamic": "true", "properties": { "type": { "type": "keyword" @@ -16,8 +16,7 @@ "type": "keyword" }, "date": { - "type": "date", - "format": "yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ss||epoch_millis" + "type": "date" }, "author": { "type": "keyword" diff --git a/plugins/content-manager/src/main/resources/mappings/cti-rules-integrations-mappings.json b/plugins/content-manager/src/main/resources/mappings/cti-rules-integrations-mappings.json index 42cd9428..82cd83e1 100644 --- a/plugins/content-manager/src/main/resources/mappings/cti-rules-integrations-mappings.json +++ b/plugins/content-manager/src/main/resources/mappings/cti-rules-integrations-mappings.json @@ -1,5 +1,5 @@ { - "dynamic": "strict", + "dynamic": "true", "properties": { "type": { "type": "keyword" @@ -20,8 +20,7 @@ "type": "keyword" }, "date": { - "type": "date", - "format": "yyyy-MM-dd" + "type": "date" }, "documentation": { "type": "keyword" diff --git a/plugins/content-manager/src/main/resources/mappings/cti-rules-mappings.json b/plugins/content-manager/src/main/resources/mappings/cti-rules-mappings.json index fea051ca..741c25df 100644 --- a/plugins/content-manager/src/main/resources/mappings/cti-rules-mappings.json +++ b/plugins/content-manager/src/main/resources/mappings/cti-rules-mappings.json @@ -1,5 +1,5 @@ { - "dynamic": "strict", + "dynamic": "true", "dynamic_templates": [ { "detection_fields": { @@ -38,12 +38,10 @@ "type": "keyword" }, "date": { - "type": "date", - "format": "yyyy-MM-dd" + "type": "date" }, "modified": { - "type": "date", - "format": "yyyy-MM-dd" + "type": "date" }, "status": { "type": "keyword" diff --git a/plugins/content-manager/src/test/java/com/wazuh/contentmanager/client/CTIClientTests.java b/plugins/content-manager/src/test/java/com/wazuh/contentmanager/client/CTIClientTests.java deleted file mode 100644 index 9d4d454a..00000000 --- a/plugins/content-manager/src/test/java/com/wazuh/contentmanager/client/CTIClientTests.java +++ /dev/null @@ -1,379 +0,0 @@ -/* - * Copyright (C) 2024, Wazuh Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package com.wazuh.contentmanager.client; - -import org.apache.hc.client5.http.HttpHostConnectException; -import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; -import org.apache.hc.core5.http.ContentType; -import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; -import org.apache.hc.core5.http.Method; -import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.settings.Settings; -import org.opensearch.env.Environment; -import org.opensearch.test.OpenSearchIntegTestCase; -import org.junit.After; -import org.junit.Before; - -import java.io.IOException; -import java.util.Collections; -import java.util.Map; - -import com.wazuh.contentmanager.cti.catalog.model.Changes; -import com.wazuh.contentmanager.cti.catalog.model.ConsumerInfo; -import com.wazuh.contentmanager.cti.catalog.model.Offset; -import com.wazuh.contentmanager.settings.PluginSettings; -import org.mockito.InjectMocks; -import org.mockito.Mock; - -import static org.mockito.Mockito.*; - -/** Tests the CTIClient */ -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE) -public class CTIClientTests extends OpenSearchIntegTestCase { - - private CTIClient ctiClient; - private CTIClient spyCtiClient; - - @Mock private Environment mockEnvironment; - @Mock private ClusterService mockClusterService; - @InjectMocks private PluginSettings pluginSettings; - - @Before - @Override - public void setUp() throws Exception { - super.setUp(); // Ensure OpenSearch test setup runs - Settings settings = - Settings.builder() - .put("content_manager.cti.client.max_attempts", 3) - .put("content_manager.cti.client.sleep_time", 60) - .put("content_manager.cti.context", "vd_1.0.0") - .put("content_manager.cti.consumer", "https://cti.wazuh.com/TEST/api/v1") - .build(); - - this.mockEnvironment = mock(Environment.class); - when(this.mockEnvironment.settings()).thenReturn(settings); - this.pluginSettings = - PluginSettings.getInstance(this.mockEnvironment.settings(), this.mockClusterService); - this.ctiClient = new CTIClient("www.test.com", this.pluginSettings); - this.spyCtiClient = spy(this.ctiClient); - } - - @After - @Override - public void tearDown() throws Exception { - this.ctiClient = null; - try { - this.spyCtiClient.close(); - } catch (IOException e) { - logger.error( - "Exception trying to close spy of CtiClient {} in test testSendRequest_SuccessfulRequest", - e.getMessage()); - } - super.tearDown(); - } - - /** - * Tests a successful request to the CTI API verifying that: - * - * 
-     *     - The response is not null.
-     *     - The response code is 200 OK.
-     *     - The {@link CTIClient#sendRequest(Method, String, String, Map, Header, int)} is invoked exactly 1 time.
-     *
- */ - public void testSendRequest_SuccessfulRequest() { - // Arrange - SimpleHttpResponse mockResponse = new SimpleHttpResponse(HttpStatus.SC_SUCCESS, "OK"); - - // spotless:off - when(this.spyCtiClient.doHttpClientSendRequest( - any(Method.class), - anyString(), - any(), - anyMap(), - any())) - .thenReturn(mockResponse); - // spotless:on - - // Act - SimpleHttpResponse response; - response = - this.spyCtiClient.sendRequest( - Method.GET, - "/catalog/contexts/vd_1.0.0/consumers/vd_4.8.0/changes", - null, - Collections.emptyMap(), - null, - 3); - - // Assert - assertNotNull("Response should not be null", response); - assertEquals(HttpStatus.SC_SUCCESS, response.getCode()); - verify(this.spyCtiClient, times(1)) - .sendRequest( - any(Method.class), - eq("/catalog/contexts/vd_1.0.0/consumers/vd_4.8.0/changes"), - isNull(), - anyMap(), - isNull(), - eq(3)); - } - - /** - * Tests a bad request to the CTI API verifying that: - * - * 
-     *     - The response is not null.
-     *     - The response code is 400 BAD_REQUEST.
-     *     - The {@link CTIClient#sendRequest(Method, String, String, Map, Header, int)} is invoked exactly 1 time.
-     *
- */ - public void testSendRequest_BadRequest() { - // Arrange - SimpleHttpResponse mockResponse = - new SimpleHttpResponse(HttpStatus.SC_BAD_REQUEST, "Bad Request"); - - // spotless:off - when(this.spyCtiClient.doHttpClientSendRequest( - any(Method.class), - anyString(), - any(), - anyMap(), - any())) - .thenReturn(mockResponse); - // spotless:on - - SimpleHttpResponse response; - response = - this.spyCtiClient.sendRequest( - Method.GET, - "/catalog/contexts/vd_1.0.0/consumers/vd_4.8.0/changes", - null, - Collections.emptyMap(), - null, - 3); - - // Assert - assertNotNull(response); - assertEquals(HttpStatus.SC_BAD_REQUEST, response.getCode()); - verify(this.spyCtiClient, times(1)) - .sendRequest( - any(Method.class), - eq("/catalog/contexts/vd_1.0.0/consumers/vd_4.8.0/changes"), - isNull(), - anyMap(), - isNull(), - eq(3)); - } - - /** - * Tests the rate limiting management for requests the CTI API verifying that: - * - * 
-     *     - The response is not null.
-     *     - The response code is 429 TOO_MANY_REQUESTS.
-     *     - The {@link CTIClient#sendRequest(Method, String, String, Map, Header, int)} is invoked exactly 3 times.
-     *
- */ - public void testSendRequest_TooManyRequests_RetriesThreeTimes() { - // Arrange - // spotless:off - SimpleHttpResponse mockResponse429 = new SimpleHttpResponse( - HttpStatus.SC_TOO_MANY_REQUESTS, - "Too Many Requests" - ); - // spotless:on - // Required by the API. - mockResponse429.setHeader("Retry-After", "1"); - - // Mock that sendRequest returns 429 three times. - // spotless:off - when(this.spyCtiClient.doHttpClientSendRequest( - any(Method.class), - anyString(), - any(), - anyMap(), - any())) - .thenReturn(mockResponse429); - // spotless:on - - // Act - SimpleHttpResponse response = - this.spyCtiClient.sendRequest( - Method.GET, - "/catalog/contexts/vd_1.0.0/consumers/vd_4.8.0/changes", - null, - Collections.emptyMap(), - null, - 3); - - // Assert - assertNotNull(response); - assertEquals(HttpStatus.SC_TOO_MANY_REQUESTS, response.getCode()); - // Verify three calls of doHttpClientSendRequest - verify(this.spyCtiClient, times(3)) - .doHttpClientSendRequest( - any(Method.class), - eq("/catalog/contexts/vd_1.0.0/consumers/vd_4.8.0/changes"), - isNull(), - any(Map.class), - isNull()); - } - - /** - * Tests a successful request to the CTI API to obtain a consumer's information, verifying that: - * - * 
-     *     - The response is not null.
-     *     - The {@link ConsumerInfo} instance matches the response's data.
-     *     - The {@link CTIClient#sendRequest(Method, String, String, Map, Header, int)} is invoked exactly 3 times.
-     *
- * - * @throws IOException {@inheritDoc} - */ - public void testGetConsumerInfo_SuccessfulRequest() throws IOException { - // Arrange - SimpleHttpResponse response = new SimpleHttpResponse(HttpStatus.SC_SUCCESS, "OK"); - response.setBody( - "{\"data\":{\"id\":4,\"name\":\"vd_4.8.0\",\"context\":\"vd_1.0.0\",\"operations\":null,\"inserted_at\":\"2023-11-23T19:34:18.698495Z\",\"updated_at\":\"2025-03-31T15:17:32.839974Z\",\"changes_url\":\"cti.wazuh.com/api/v1/catalog/contexts/vd_1.0.0/consumers/vd_4.8.0/changes\",\"last_offset\":1675416,\"last_snapshot_at\":\"2025-03-31T10:24:21.822354Z\",\"last_snapshot_link\":\"https://cti.wazuh.com/store/contexts/vd_1.0.0/consumers/vd_4.8.0/1672583_1743416661.zip\",\"last_snapshot_offset\":1672583,\"paths_filter\":null}}", - ContentType.APPLICATION_JSON); - - // spotless:off - when(this.spyCtiClient.doHttpClientSendRequest( - any(Method.class), - anyString(), - any(), - any(), - any() - )).thenReturn(response); - // spotless:on - - // Act - ConsumerInfo consumerInfo = this.spyCtiClient.getConsumerInfo(); - - // Assert - assertNotNull(consumerInfo); - verify(this.spyCtiClient, times(1)) - .sendRequest(any(Method.class), anyString(), isNull(), isNull(), isNull(), anyInt()); - assertEquals(1675416, consumerInfo.getLastOffset()); - assertEquals( - "https://cti.wazuh.com/store/contexts/vd_1.0.0/consumers/vd_4.8.0/1672583_1743416661.zip", - consumerInfo.getLastSnapshotLink()); - assertEquals("vd_1.0.0", consumerInfo.getContext()); - assertEquals("vd_4.8.0", consumerInfo.getName()); - } - - /** - * Test that {@link CTIClient#getConsumerInfo()} throws {@link HttpHostConnectException} on no - * response. - */ - public void testGetConsumerInfo_ThrowException() { - // spotless:off - when(this.spyCtiClient.sendRequest( - any(Method.class), - anyString(), - anyString(), - anyMap(), - any(Header.class))) - .thenReturn(null); - // spotless:on - - // Act & Assert - assertThrows(HttpHostConnectException.class, () -> this.spyCtiClient.getConsumerInfo()); - } - - /** - * Tests a successful request to the CTI API to obtain changes in a consumer, verifying that: - * - * 
-     *     - The list of changes is not null.
-     *     - The list of changes is not empty.
-     *     - The {@link Changes} instance matches the response's data.
-     *     - The {@link CTIClient#sendRequest(Method, String, String, Map, Header, int)} is invoked exactly 1 time.
-     *
- */ - public void testGetChanges_SuccessfulRequest() { - // Arrange - SimpleHttpResponse response = new SimpleHttpResponse(HttpStatus.SC_SUCCESS, "OK"); - response.setBody( - "{\"data\":[{\"offset\":1761037,\"type\":\"update\",\"version\":19,\"context\":\"vd_1.0.0\",\"resource\":\"CVE-2019-0605\",\"operations\":[{\"op\":\"replace\",\"path\":\"/containers/cna/x_remediations/windows/0/anyOf/133\",\"value\":\"KB5058922\"},{\"op\":\"replace\",\"path\":\"/containers/cna/x_remediations/windows/5/anyOf/140\",\"value\":\"KB5058921\"},{\"op\":\"add\",\"path\":\"/containers/adp/0/descriptions/1\",\"value\":{\"lang\":\"en\",\"value\":\"OpenStack Ironic fails to restrict paths used for file:// image URLs\"}},{\"op\":\"remove\",\"path\":\"/containers/adp/0/affected/0/platforms\"},{\"op\":\"add\",\"path\":\"/containers/adp/0/affected/0/versions\",\"value\":[{\"status\":\"affected\",\"version\":\"0\",\"lessThan\":\"24.1.3\",\"versionType\":\"custom\"},{\"status\":\"affected\",\"version\":\"25.0.0\",\"lessThan\":\"26.1.1\",\"versionType\":\"custom\"},{\"status\":\"affected\",\"version\":\"0\",\"lessThan\":\"29.0.1\",\"versionType\":\"custom\"},{\"status\":\"affected\",\"version\":\"27.0.0\",\"lessThan\":\"29.0.1\",\"versionType\":\"custom\"}]}]}]}", - ContentType.APPLICATION_JSON); - - // spotless:off - when(this.spyCtiClient.doHttpClientSendRequest( - any(Method.class), - anyString(), - any(), - anyMap(), - any())) - .thenReturn(response); - // spotless:on - - // Act - Changes changes = this.spyCtiClient.getChanges(0, 200, true); - - // Assert - assertNotNull(changes); - assertNotEquals(0, changes.get().size()); - Offset change = changes.getFirst(); - assertEquals(1761037, change.getOffset()); - assertEquals(Offset.Type.UPDATE, change.getType()); - assertEquals("CVE-2019-0605", change.getResource()); - assertEquals(5, change.getOperations().size()); - verify(this.spyCtiClient, times(1)) - .sendRequest(any(Method.class), anyString(), isNull(), anyMap(), isNull(), anyInt()); - } - - /** - * Tests an unsuccessful request to the CTI API to obtain changes in a consumer, verifying that - * even though the response is null: - * - * 
-     *     - The list of changes is not null (properly initialized).
-     *     - The list of changes is empty.
-     *     - The {@link CTIClient#sendRequest(Method, String, String, Map, Header, int)} is invoked exactly 1 time.
-     *
- */ - public void testGetChanges_NullResponse() { - // spotless:off - when(this.spyCtiClient.sendRequest( - any(Method.class), - anyString(), - anyString(), - anyMap(), - any(Header.class))) - .thenReturn(null); - // spotless:on - - Changes changes = this.spyCtiClient.getChanges(0, 100, true); - assertNotNull(changes); - assertEquals(new Changes().get().isEmpty(), changes.get().isEmpty()); - verify(this.spyCtiClient, times(1)) - .sendRequest(any(Method.class), anyString(), isNull(), anyMap(), isNull(), anyInt()); - } - - /** Tests the {@link CTIClient#contextQueryParameters} utility method: */ - public void testContextQueryParameters() { - Map params = CTIClient.contextQueryParameters(0, 10, true); - assertEquals(3, params.size()); - assertEquals(String.valueOf(0), params.get(CTIClient.QueryParameters.FROM_OFFSET.getValue())); - assertEquals(String.valueOf(10), params.get(CTIClient.QueryParameters.TO_OFFSET.getValue())); - assertEquals( - String.valueOf(true), params.get(CTIClient.QueryParameters.WITH_EMPTIES.getValue())); - } -} diff --git a/plugins/content-manager/src/test/java/com/wazuh/contentmanager/client/HttpClientTests.java b/plugins/content-manager/src/test/java/com/wazuh/contentmanager/client/HttpClientTests.java deleted file mode 100644 index 35f1730d..00000000 --- a/plugins/content-manager/src/test/java/com/wazuh/contentmanager/client/HttpClientTests.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (C) 2024, Wazuh Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package com.wazuh.contentmanager.client; - -import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; -import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; -import org.apache.hc.core5.http.Method; -import org.opensearch.test.OpenSearchIntegTestCase; -import org.junit.After; -import org.junit.Before; - -import java.util.Collections; -import java.util.Map; - -import static org.mockito.Mockito.*; - -/** Tests the HttpClient */ -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE) -public class HttpClientTests extends OpenSearchIntegTestCase { - - private HttpClient httpClient; - - @Before - @Override - public void setUp() throws Exception { - super.setUp(); // Ensure OpenSearch test setup runs - httpClient = mock(HttpClient.class); - } - - @SuppressWarnings("EmptyMethod") - @After - @Override - public void tearDown() throws Exception { - super.tearDown(); - } - - /** Test send request success */ - public void testSendRequestSuccess() { - SimpleHttpResponse mockResponse = new SimpleHttpResponse(HttpStatus.SC_SUCCESS, "OK"); - - when(httpClient.sendRequest( - any(Method.class), anyString(), any(), anyMap(), any(Header[].class))) - .thenReturn(mockResponse); - - SimpleHttpResponse response = - httpClient.sendRequest(Method.GET, "/test", null, Collections.emptyMap()); - - assertNotNull("Response should not be null", response); - assertEquals(HttpStatus.SC_SUCCESS, response.getCode()); - } - - /** Test send POST request */ - public void testSendPostRequest() { - SimpleHttpResponse mockResponse = new SimpleHttpResponse(HttpStatus.SC_CREATED, "Created"); - String requestBody = "{\"key\":\"value\"}"; - - when(httpClient.sendRequest( - eq(Method.POST), anyString(), eq(requestBody), anyMap(), any(Header[].class))) - .thenReturn(mockResponse); - - SimpleHttpResponse response = - httpClient.sendRequest(Method.POST, "/create", requestBody, Collections.emptyMap()); - - assertNotNull("Response should not be null", response); - assertEquals(HttpStatus.SC_CREATED, response.getCode()); - } - - /** Test sending request with query parameters */ - public void testSendRequestWithQueryParams() { - SimpleHttpResponse mockResponse = new SimpleHttpResponse(HttpStatus.SC_SUCCESS, "OK"); - Map queryParams = Map.of("param1", "value1", "param2", "value2"); - - when(httpClient.sendRequest( - any(Method.class), anyString(), any(), eq(queryParams), any(Header[].class))) - .thenReturn(mockResponse); - - SimpleHttpResponse response = httpClient.sendRequest(Method.GET, "/test", null, queryParams); - - assertNotNull("Response should not be null", response); - assertEquals(HttpStatus.SC_SUCCESS, response.getCode()); - } - - /** Test send request failure */ - public void testSendRequestFailure() { - SimpleHttpResponse mockResponse = - new SimpleHttpResponse(HttpStatus.SC_SERVER_ERROR, "Internal Server Error"); - - when(httpClient.sendRequest( - any(Method.class), anyString(), any(), anyMap(), any(Header[].class))) - .thenReturn(mockResponse); - - SimpleHttpResponse response = - httpClient.sendRequest(Method.GET, "/error", null, Collections.emptyMap()); - - assertNotNull("Response should not be null", response); - assertEquals(HttpStatus.SC_SERVER_ERROR, response.getCode()); - } - - /** Test sendRequest() timeout */ - public void testSendRequestTimeout() { - when(httpClient.sendRequest( - any(Method.class), anyString(), any(), anyMap(), any(Header[].class))) - .thenThrow(new RuntimeException("Request timeout")); - - try { - httpClient.sendRequest(Method.GET, "/timeout", null, Collections.emptyMap()); - fail("Expected RuntimeException due to timeout"); - } catch (RuntimeException e) { - assertEquals("Request timeout", e.getMessage()); - } - } -} diff --git a/plugins/content-manager/src/test/java/com/wazuh/contentmanager/cti/catalog/index/ConsumersIndexTests.java b/plugins/content-manager/src/test/java/com/wazuh/contentmanager/cti/catalog/index/ConsumersIndexTests.java new file mode 100644 index 00000000..6f85d73b --- /dev/null +++ b/plugins/content-manager/src/test/java/com/wazuh/contentmanager/cti/catalog/index/ConsumersIndexTests.java @@ -0,0 +1,177 @@ +package com.wazuh.contentmanager.cti.catalog.index; + +import com.wazuh.contentmanager.cti.catalog.model.LocalConsumer; +import com.wazuh.contentmanager.settings.PluginSettings; +import org.junit.After; +import org.junit.Before; +import org.mockito.Answers; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; +import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse; +import org.opensearch.action.get.GetRequest; +import org.opensearch.action.get.GetResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.support.PlainActionFuture; +import org.opensearch.cluster.health.ClusterHealthStatus; +import org.opensearch.common.settings.Settings; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.transport.client.Client; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; + +public class ConsumersIndexTests extends OpenSearchTestCase { + + private ConsumersIndex consumersIndex; + private AutoCloseable closeable; + private Client client; + + @Mock private IndexResponse indexResponse; + @Mock private GetResponse getResponse; + @Mock private ClusterHealthResponse clusterHealthResponse; + @Mock private IndicesExistsResponse indicesExistsResponse; + @Mock private CreateIndexResponse createIndexResponse; + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + this.closeable = MockitoAnnotations.openMocks(this); + this.client = mock(Client.class, Answers.RETURNS_DEEP_STUBS); + Settings settings = Settings.builder().build(); + PluginSettings.getInstance(settings); + this.consumersIndex = new ConsumersIndex(this.client); + } + + @After + @Override + public void tearDown() throws Exception { + if (this.closeable != null) { + this.closeable.close(); + } + super.tearDown(); + } + + /** + * Tests that setConsumer constructs the correct ID (context_name) and performs the index operation. + */ + public void testSetConsumer_Success() throws Exception { + // Mock + when(this.client.admin().cluster().prepareHealth().setIndices(anyString()).setWaitForYellowStatus().get()) + .thenReturn(this.clusterHealthResponse); + when(this.clusterHealthResponse.getStatus()).thenReturn(ClusterHealthStatus.GREEN); + + PlainActionFuture future = PlainActionFuture.newFuture(); + future.onResponse(this.indexResponse); + when(this.client.index(any(IndexRequest.class))).thenReturn(future); + + // Act + LocalConsumer consumer = new LocalConsumer("test_context", "test_consumer", 100L, 200L, "http://snapshot"); + this.consumersIndex.setConsumer(consumer); + + // Assert + ArgumentCaptor captor = ArgumentCaptor.forClass(IndexRequest.class); + verify(this.client).index(captor.capture()); + + IndexRequest request = captor.getValue(); + assertEquals(ConsumersIndex.INDEX_NAME, request.index()); + assertEquals("test_context_test_consumer", request.id()); + } + + /** + * Tests that setConsumer throws a RuntimeException if the cluster status is RED. + */ + public void testSetConsumer_IndexNotReady() { + // Mock + when(this.client.admin().cluster().prepareHealth().setIndices(anyString()).setWaitForYellowStatus().get()) + .thenReturn(this.clusterHealthResponse); + when(this.clusterHealthResponse.getStatus()).thenReturn(ClusterHealthStatus.RED); + + LocalConsumer consumer = new LocalConsumer("ctx", "name"); + + // Act and Assert + RuntimeException ex = assertThrows(RuntimeException.class, () -> this.consumersIndex.setConsumer(consumer)); + assertEquals("Index not ready", ex.getMessage()); + } + + /** + * Tests getConsumer retrieves the correct document ID based on context and consumer name. + */ + public void testGetConsumer_Success() throws Exception { + // Mock + when(this.client.admin().cluster().prepareHealth().setIndices(anyString()).setWaitForYellowStatus().get()) + .thenReturn(this.clusterHealthResponse); + when(this.clusterHealthResponse.getStatus()).thenReturn(ClusterHealthStatus.YELLOW); + + PlainActionFuture future = PlainActionFuture.newFuture(); + future.onResponse(this.getResponse); + when(this.client.get(any(GetRequest.class))).thenReturn(future); + + // Act + this.consumersIndex.getConsumer("my_context", "my_consumer"); + + // Assert + ArgumentCaptor captor = ArgumentCaptor.forClass(GetRequest.class); + verify(this.client).get(captor.capture()); + + GetRequest request = captor.getValue(); + assertEquals(ConsumersIndex.INDEX_NAME, request.index()); + assertEquals("my_context_my_consumer", request.id()); + } + + /** + * Tests exists() delegates correctly to the IndicesExistsRequest. + */ + public void testExists() { + // Mock + when(this.client.admin().indices().exists(any(IndicesExistsRequest.class)).actionGet()) + .thenReturn(this.indicesExistsResponse); + when(this.indicesExistsResponse.isExists()).thenReturn(true); + + // Clear invocations to ensure verify only counts the actual method call + clearInvocations(this.client.admin().indices()); + + // Act + boolean result = this.consumersIndex.exists(); + + // Assert + assertTrue(result); + ArgumentCaptor captor = ArgumentCaptor.forClass(IndicesExistsRequest.class); + verify(this.client.admin().indices()).exists(captor.capture()); + assertArrayEquals(new String[]{ConsumersIndex.INDEX_NAME}, captor.getValue().indices()); + } + + /** + * Tests createIndex(). + */ + public void testCreateIndex() throws Exception { + // Mock + PlainActionFuture future = PlainActionFuture.newFuture(); + future.onResponse(this.createIndexResponse); + when(this.client.admin().indices().create(any(CreateIndexRequest.class))).thenReturn(future); + + ConsumersIndex spyIndex = spy(this.consumersIndex); + doReturn("{}").when(spyIndex).loadMappingFromResources(); + + // Act + spyIndex.createIndex(); + + // Assert + ArgumentCaptor captor = ArgumentCaptor.forClass(CreateIndexRequest.class); + verify(this.client.admin().indices()).create(captor.capture()); + + CreateIndexRequest request = captor.getValue(); + assertEquals(ConsumersIndex.INDEX_NAME, request.index()); + + // Validate Settings (Hidden = true, Replicas = 0) + assertEquals("true", request.settings().get("hidden")); + assertEquals("0", request.settings().get("index.number_of_replicas")); + } +} diff --git a/plugins/content-manager/src/test/java/com/wazuh/contentmanager/cti/catalog/index/ContentIndexTests.java b/plugins/content-manager/src/test/java/com/wazuh/contentmanager/cti/catalog/index/ContentIndexTests.java index 09cbcb4f..b6f156ae 100644 --- a/plugins/content-manager/src/test/java/com/wazuh/contentmanager/cti/catalog/index/ContentIndexTests.java +++ b/plugins/content-manager/src/test/java/com/wazuh/contentmanager/cti/catalog/index/ContentIndexTests.java @@ -1,133 +1,215 @@ -/* - * Copyright (C) 2024, Wazuh Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ package com.wazuh.contentmanager.cti.catalog.index; import com.google.gson.JsonObject; -import org.opensearch.action.get.GetResponse; -import org.opensearch.transport.client.Client; -import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.settings.Settings; -import org.opensearch.env.Environment; -import org.opensearch.test.OpenSearchIntegTestCase; -import org.junit.Before; - -import java.util.List; - -import com.wazuh.contentmanager.cti.catalog.model.Changes; -import com.wazuh.contentmanager.cti.catalog.model.Offset; +import com.google.gson.JsonParser; import com.wazuh.contentmanager.cti.catalog.model.Operation; import com.wazuh.contentmanager.settings.PluginSettings; -import org.mockito.InjectMocks; +import org.junit.After; +import org.junit.Before; +import org.mockito.Answers; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; -import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.opensearch.action.delete.DeleteRequest; +import org.opensearch.action.get.GetRequest; +import org.opensearch.action.get.GetResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.support.PlainActionFuture; +import org.opensearch.common.settings.Settings; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.transport.client.Client; + +import java.util.ArrayList; +import java.util.List; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE) -public class ContentIndexTests extends OpenSearchIntegTestCase { - private ContentIndex contentUpdaterSpy; +public class ContentIndexTests extends OpenSearchTestCase { - @Mock private Environment mockEnvironment; - @Mock private ClusterService mockClusterService; - @InjectMocks private PluginSettings pluginSettings; + private ContentIndex contentIndex; + private AutoCloseable closeable; + private Client client; + + @Mock private IndexResponse indexResponse; + @Mock private GetResponse getResponse; + + private static final String INDEX_NAME = ".test-index"; + private static final String MAPPINGS_PATH = "/mappings/test-mapping.json"; + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + this.closeable = MockitoAnnotations.openMocks(this); + this.client = mock(Client.class, Answers.RETURNS_DEEP_STUBS); + + Settings settings = Settings.builder().build(); + PluginSettings.getInstance(settings); + + this.contentIndex = new ContentIndex(this.client, INDEX_NAME, MAPPINGS_PATH); + } + + @After + @Override + public void tearDown() throws Exception { + if (this.closeable != null) { + this.closeable.close(); + } + super.tearDown(); + } /** - * Set up the tests - * - * @throws Exception rethrown from parent method + * Test creating an Integration. + * Validates that fields are removed during preprocessing. */ - @Before - public void setup() throws Exception { - super.setUp(); - - Settings settings = - Settings.builder() - .put("content_manager.max_concurrent_bulks", 5) - .put("content_manager.max_items_per_bulk", 25) - .put("content_manager.client.timeout", "10") - .build(); - - this.mockEnvironment = mock(Environment.class); - when(this.mockEnvironment.settings()).thenReturn(settings); - this.pluginSettings = - PluginSettings.getInstance(this.mockEnvironment.settings(), this.mockClusterService); - - Client client = mock(Client.class); - ContentIndex contentIndex = new ContentIndex(client, this.pluginSettings); - this.contentUpdaterSpy = Mockito.spy(contentIndex); - } - - /** Test the {@link ContentIndex#patch} method with an Offset with Create content type. */ - public void testPatchCreate() throws Exception { + public void testCreate_Integration_Processing() { // Mock - doNothing().when(this.contentUpdaterSpy).index((Offset) any()); - // Arrange - Offset offset = new Offset("test", 1L, "test", Offset.Type.CREATE, 1L, null, null); + PlainActionFuture future = PlainActionFuture.newFuture(); + future.onResponse(this.indexResponse); + when(this.client.index(any(IndexRequest.class))).thenReturn(future); + + String jsonPayload = "{" + + "\"type\": \"integration\"," + + "\"document\": {" + + " \"id\": \"f0c91fac-d749-4ef0-bdfa-0b3632adf32d\"," + + " \"date\": \"2025-11-26\"," + + " \"kvdbs\": []," + + " \"title\": \"wazuh-fim\"," + + " \"author\": \"Wazuh Inc.\"," + + " \"category\": \"System Activity\"," + + " \"enable_decoders\": true" + + "}" + + "}"; + JsonObject payload = JsonParser.parseString(jsonPayload).getAsJsonObject(); + String id = "f0c91fac-d749-4ef0-bdfa-0b3632adf32d"; + // Act - this.contentUpdaterSpy.patch(new Changes(List.of(offset))); + try { + this.contentIndex.create(id, payload); + } catch (Exception e) { + fail("Create should not throw exception: " + e.getMessage()); + } + // Assert - verify(this.contentUpdaterSpy, times(1)).patch(any()); + ArgumentCaptor captor = ArgumentCaptor.forClass(IndexRequest.class); + verify(this.client).index(captor.capture()); + + IndexRequest request = captor.getValue(); + assertEquals(INDEX_NAME, request.index()); + assertEquals(id, request.id()); + + JsonObject source = JsonParser.parseString(request.source().utf8ToString()).getAsJsonObject(); + JsonObject doc = source.getAsJsonObject("document"); + assertTrue("Title should exist", doc.has("title")); } - /** Test the {@link ContentIndex#patch} method with an Offset with Update content type. */ - public void testPatchUpdate() throws Exception { - // Mock a GetResponse that returns a valid existing document - GetResponse mockResponse = mock(GetResponse.class); - when(mockResponse.isExists()).thenReturn(true); - // Mock JsonObject - JsonObject json = new JsonObject(); - json.addProperty("field", "value"); - doReturn(json).when(this.contentUpdaterSpy).getById(any()); - // Mock index() to avoid actual client call - doNothing().when(this.contentUpdaterSpy).index((Offset) any()); - // Arrange - Offset offset = - new Offset( - "test", - 1L, - "test", - Offset.Type.UPDATE, - 1L, - List.of(new Operation("replace", "/field", null, "new_value")), - null); + /** + * Test creating a Decoder. + * Validates that the YAML enrichment is generated. + */ + public void testCreate_Decoder_YamlEnrichment() { + // Mock + PlainActionFuture future = PlainActionFuture.newFuture(); + future.onResponse(this.indexResponse); + when(this.client.index(any(IndexRequest.class))).thenReturn(future); + + String jsonPayload = "{" + + "\"type\": \"decoder\"," + + "\"document\": {" + + " \"id\": \"2ebb3a6b-c4a3-47fb-aae5-a0d9bd8cbfed\"," + + " \"name\": \"decoder/wazuh-fim/0\"," + + " \"check\": \"starts_with($event.original, \\\"8:syscheck:\\\")\"," + + " \"enabled\": true," + + " \"parents\": [\"decoder/integrations/0\"]" + + "}" + + "}"; + JsonObject payload = JsonParser.parseString(jsonPayload).getAsJsonObject(); + String id = "2ebb3a6b-c4a3-47fb-aae5-a0d9bd8cbfed"; + // Act - this.contentUpdaterSpy.patch(new Changes(List.of(offset))); + try { + this.contentIndex.create(id, payload); + } catch (Exception e) { + fail("Create should not throw exception: " + e.getMessage()); + } + // Assert - verify(this.contentUpdaterSpy, times(1)).index((Offset) any()); + ArgumentCaptor captor = ArgumentCaptor.forClass(IndexRequest.class); + verify(this.client).index(captor.capture()); + + JsonObject source = JsonParser.parseString(captor.getValue().source().utf8ToString()).getAsJsonObject(); + + assertTrue("Should contain 'decoder' field", source.has("decoder")); + String yaml = source.get("decoder").getAsString(); + assertTrue(yaml.contains("name: \"decoder/wazuh-fim/0\"")); + assertTrue(yaml.contains("check: \"starts_with($event.original, \\\"8:syscheck:\\\")\"")); } - /** Test the {@link ContentIndex#patch} method with an Offset with Delete content type. */ - public void testPatchDelete() { - // Mock a GetResponse that returns a valid existing document - GetResponse mockResponse = mock(GetResponse.class); - when(mockResponse.isExists()).thenReturn(true); - // Mock this.delete() to avoid actual client call - doNothing().when(this.contentUpdaterSpy).delete(any()); - // Arrange - Offset offset = new Offset("test", 1L, "test", Offset.Type.DELETE, 1L, null, null); + /** + * Test updating a document. + * Simulates fetching an existing document, applying operations, and re-indexing. + */ + public void testUpdate_Operations() throws Exception { + String id = "58dc8e10-0b69-4b81-a851-7a767e831fff"; + + // Mock + String originalDocJson = "{" + + "\"type\": \"decoder\"," + + "\"document\": {" + + " \"normalize\": [{" + + " \"map\": [" + + " { \"springboot.gc.last_info.time.start\": \"old_value\" }" + + " ]" + + " }]" + + "}" + + "}"; + + PlainActionFuture getFuture = PlainActionFuture.newFuture(); + getFuture.onResponse(this.getResponse); + when(this.client.get(any(GetRequest.class))).thenReturn(getFuture); + when(this.getResponse.isExists()).thenReturn(true); + when(this.getResponse.getSourceAsString()).thenReturn(originalDocJson); + + PlainActionFuture indexFuture = PlainActionFuture.newFuture(); + indexFuture.onResponse(this.indexResponse); + when(this.client.index(any(IndexRequest.class))).thenReturn(indexFuture); + + List operations = new ArrayList<>(); + operations.add(new Operation("add", "/document/normalize/0/map/0/springboot.gc.last_info.time.duration", null, "new_duration")); + // Act - this.contentUpdaterSpy.patch(new Changes(List.of(offset))); + this.contentIndex.update(id, operations); + // Assert - verify(this.contentUpdaterSpy, times(1)).delete(any()); + ArgumentCaptor captor = ArgumentCaptor.forClass(IndexRequest.class); + verify(this.client).index(captor.capture()); + + JsonObject updatedDoc = JsonParser.parseString(captor.getValue().source().utf8ToString()).getAsJsonObject(); + + JsonObject mapItem = updatedDoc.getAsJsonObject("document") + .getAsJsonArray("normalize").get(0).getAsJsonObject() + .getAsJsonArray("map").get(0).getAsJsonObject(); + + assertTrue("New field should be added", mapItem.has("springboot.gc.last_info.time.duration")); + assertEquals("new_duration", mapItem.get("springboot.gc.last_info.time.duration").getAsString()); + } + + /** + * Test delete operation. + */ + public void testDelete() { + String id = "test-id"; + + // Act + this.contentIndex.delete(id); + + // Assert + ArgumentCaptor captor = ArgumentCaptor.forClass(DeleteRequest.class); + verify(this.client).delete(captor.capture(), any()); + + assertEquals(INDEX_NAME, captor.getValue().index()); + assertEquals(id, captor.getValue().id()); } } diff --git a/plugins/content-manager/src/test/java/com/wazuh/contentmanager/cti/catalog/service/ContentUpdaterIT.java b/plugins/content-manager/src/test/java/com/wazuh/contentmanager/cti/catalog/service/ContentUpdaterIT.java deleted file mode 100644 index 948de291..00000000 --- a/plugins/content-manager/src/test/java/com/wazuh/contentmanager/cti/catalog/service/ContentUpdaterIT.java +++ /dev/null @@ -1,316 +0,0 @@ -/* - * Copyright (C) 2024, Wazuh Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package com.wazuh.contentmanager.cti.catalog.service; - -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; - -import com.wazuh.contentmanager.cti.catalog.model.Changes; -import com.wazuh.contentmanager.cti.catalog.model.ConsumerInfo; -import com.wazuh.contentmanager.cti.catalog.model.Offset; -import com.wazuh.contentmanager.cti.catalog.model.Operation; -import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix; -import org.opensearch.action.admin.indices.refresh.RefreshRequest; -import org.opensearch.action.support.WriteRequest; -import org.opensearch.transport.client.Client; -import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.core.xcontent.ToXContent; -import org.opensearch.env.Environment; -import org.opensearch.plugins.Plugin; -import org.opensearch.test.OpenSearchIntegTestCase; -import org.junit.Before; - -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import com.wazuh.contentmanager.ContentManagerPlugin; -import com.wazuh.contentmanager.client.CTIClient; -import com.wazuh.contentmanager.cti.catalog.index.ContentIndex; -import com.wazuh.contentmanager.cti.catalog.index.ConsumersIndex; -import com.wazuh.contentmanager.settings.PluginSettings; -import com.wazuh.contentmanager.utils.Privileged; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Mockito; - -import static org.mockito.Mockito.*; - -@AwaitsFix(bugUrl = "https://github.com/wazuh/wazuh-indexer/issues/1250") -@ThreadLeakScope(ThreadLeakScope.Scope.NONE) -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE) -public class ContentUpdaterIT extends OpenSearchIntegTestCase { - private final String resourceId = "CVE-0000-0000"; - private Client client; - private ContentUpdater updater; - private ConsumersIndex consumersIndex; - private ContentIndex contentIndex; - private CTIClient ctiClient; - private Privileged privilegedSpy; - - @Mock private Environment mockEnvironment; - @Mock private ClusterService mockClusterService; - @InjectMocks private PluginSettings pluginSettings; - - @Before - public void setup() throws Exception { - this.client = client(); - this.ctiClient = mock(CTIClient.class); - this.consumersIndex = spy(new ConsumersIndex(client)); - this.contentIndex = new ContentIndex(client); - Settings settings = - Settings.builder() - .put("content_manager.max_changes", 1000) - .put("content_manager.cti.client.sleep_time", 60) - .put("content_manager.cti.context", "vd_1.0.0") - .put("content_manager.cti.consumer", "https://cti.wazuh.com/TEST/api/v1") - .build(); - this.mockEnvironment = mock(Environment.class); - when(this.mockEnvironment.settings()).thenReturn(settings); - this.pluginSettings = - PluginSettings.getInstance(this.mockEnvironment.settings(), this.mockClusterService); - - this.privilegedSpy = Mockito.spy(Privileged.class); - this.updater = - Mockito.spy( - new ContentUpdater( - this.ctiClient, - this.consumersIndex, - this.contentIndex, - this.privilegedSpy, - this.pluginSettings)); - - this.prepareInitialCVEInfo(0); - this.prepareInitialConsumerInfo(); - } - - @Override - protected Collection> nodePlugins() { - return Collections.singletonList(ContentManagerPlugin.class); - } - - /** - * Tests whether a create-type patch is correctly applied to the {@link ContentIndex#INDEX_NAME} - * index. - * - *

The test tries to add new content to the {@link ContentIndex#INDEX_NAME} index, which is - * initially empty (offset 0). The new content consists on a single element, with offset 1. - * - * @throws InterruptedException thrown by {@link ContentIndex#getById(String)} - * @throws ExecutionException thrown by {@link ContentIndex#getById(String)} - * @throws TimeoutException thrown by {@link ContentIndex#getById(String)} - */ - public void testUpdate_ContentChangesTypeCreate() - throws ExecutionException, InterruptedException, TimeoutException { - // Fixtures - // List of changes to apply (offset 1 == create) - Changes changes = new Changes(List.of(this.buildOffset(1, Offset.Type.CREATE))); - ConsumerInfo testConsumer = this.buildTestConsumer(1); - // Mock - when(this.ctiClient.getChanges(0, 1, false)).thenReturn(changes); - when(this.consumersIndex.get( - this.pluginSettings.getContextId(), this.pluginSettings.getConsumerId())) - .thenReturn(testConsumer); - // Act - boolean updated = this.updater.update(); - - // Ensure the index is refreshed. - RefreshRequest request = new RefreshRequest(ContentIndex.INDEX_NAME); - this.client - .admin() - .indices() - .refresh(request) - .get(this.pluginSettings.getClientTimeout(), TimeUnit.SECONDS); - - ConsumerInfo updatedConsumer = - this.consumersIndex.get( - this.pluginSettings.getContextId(), this.pluginSettings.getConsumerId()); - // Assert - assertTrue(updated); - assertNotNull(updatedConsumer); - assertEquals(1, updatedConsumer.getLastOffset()); - } - - /** - * Tests whether an update-type patch is correctly applied to the {@link ContentIndex#INDEX_NAME} - * index. - * - *

The test tries to update the content, which initially is on offset 0, to the latest offset - * on the CTI API, which is 2 (mocked response). The list of changes is [offset 1: create, offset - * 2: update]. The updated element is first created and then updated. - * - * @throws InterruptedException thrown by {@link ContentIndex#getById(String)} - * @throws ExecutionException thrown by {@link ContentIndex#getById(String)} - * @throws TimeoutException thrown by {@link ContentIndex#getById(String)} - */ - public void testUpdate_ContentChangesTypeUpdate() - throws ExecutionException, InterruptedException, TimeoutException { - // Fixtures - // List of changes to apply (offset 1 == create, offset 2 == update) - Changes changes = - new Changes( - List.of( - this.buildOffset(1, Offset.Type.CREATE), this.buildOffset(2, Offset.Type.UPDATE))); - ConsumerInfo testConsumer = this.buildTestConsumer(2); - // Mock - when(this.ctiClient.getChanges(0, 2, false)).thenReturn(changes); - when(this.consumersIndex.get( - this.pluginSettings.getContextId(), this.pluginSettings.getConsumerId())) - .thenReturn(testConsumer); - // Act - boolean updated = this.updater.update(); - - // Ensure the index is refreshed. - RefreshRequest request = new RefreshRequest(ContentIndex.INDEX_NAME); - this.client - .admin() - .indices() - .refresh(request) - .get(this.pluginSettings.getClientTimeout(), TimeUnit.SECONDS); - - ConsumerInfo updatedConsumer = - this.consumersIndex.get( - this.pluginSettings.getContextId(), this.pluginSettings.getConsumerId()); - // Assert - assertTrue(updated); - assertNotNull(updatedConsumer); - assertEquals(2, updatedConsumer.getLastOffset()); - } - - /** - * Tests whether a delete-type patch is correctly applied to the {@link ContentIndex#INDEX_NAME} - * index. - * - *

The test tries to delete an element from the {@link ContentIndex#INDEX_NAME} index, which is - * initially empty (offset 0). The test first created the element and the removes it, so the list - * of changes is [offset 1: create, offset 2: delete]. The test finally ensures the element was - * deleted. - * - * @throws InterruptedException thrown by {@link ContentIndex#getById(String)} - * @throws ExecutionException thrown by {@link ContentIndex#getById(String)} - * @throws TimeoutException thrown by {@link ContentIndex#getById(String)} - */ - public void testUpdate_ContentChangesTypeDelete() - throws InterruptedException, ExecutionException, TimeoutException { - // Fixtures - Changes changes = - new Changes( - List.of( - this.buildOffset(1, Offset.Type.CREATE), this.buildOffset(2, Offset.Type.DELETE))); - ConsumerInfo testConsumer = this.buildTestConsumer(2); - // Mock - when(this.ctiClient.getChanges(0, 2, false)).thenReturn(changes); - when(this.consumersIndex.get( - this.pluginSettings.getContextId(), this.pluginSettings.getConsumerId())) - .thenReturn(testConsumer); - // Act - boolean updated = this.updater.update(); - - // Ensure the index is refreshed. - RefreshRequest request = new RefreshRequest(ContentIndex.INDEX_NAME); - this.client - .admin() - .indices() - .refresh(request) - .get(this.pluginSettings.getClientTimeout(), TimeUnit.SECONDS); - - ConsumerInfo updatedConsumer = - this.consumersIndex.get( - this.pluginSettings.getContextId(), this.pluginSettings.getConsumerId()); - // Assert - assertTrue(updated); - assertNotNull(updatedConsumer); - assertEquals(2, updatedConsumer.getLastOffset()); - assertThrows(IllegalArgumentException.class, () -> this.contentIndex.getById(this.resourceId)); - } - - /** - * Creates an Offset object with the specified ID and content type. - * - * @param id The ID of the offset. - * @param type The content type (CREATE, UPDATE, DELETE). - * @return An Offset object with the specified ID and content type. - */ - private Offset buildOffset(long id, Offset.Type type) { - List operations = null; - Map payload = null; - if (type == Offset.Type.UPDATE) { - operations = List.of(new Operation("add", "/newField", null, "test")); - } else if (type == Offset.Type.CREATE) { - payload = new HashMap<>(); - payload.put("name", "Dummy Threat"); - payload.put("indicators", List.of("192.168.1.1", "example.com")); - } - return new Offset( - this.pluginSettings.getContextId(), id, this.resourceId, type, 1L, operations, payload); - } - - /** - * Build an instance of {@link ConsumerInfo}. - * - * @param lastOffset The initial lastOffset to set. - * @return an instance of {@link ConsumerInfo}. - */ - private ConsumerInfo buildTestConsumer(long lastOffset) { - return new ConsumerInfo( - this.pluginSettings.getConsumerId(), - this.pluginSettings.getContextId(), - 0, - lastOffset, - null); - } - - /** - * Prepares the initial ConsumerInfo document in the test index. - * - * @throws IOException If an error occurs while preparing the document. - */ - public void prepareInitialConsumerInfo() throws IOException { - // Create a ConsumerInfo document manually in the test index - ConsumerInfo info = this.buildTestConsumer(0); - this.client - .prepareIndex(consumersIndex.INDEX_NAME) - .setId(this.pluginSettings.getContextId()) - .setSource(info.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get(); - } - - /** - * Prepares the initial CVE information in the test index. - * - * @param offset The initial offset to set. - * @throws IOException If an error occurs while preparing the document. - */ - public void prepareInitialCVEInfo(long offset) throws IOException { - // Create a ConsumerInfo document manually in the test index - Offset mOffset = this.buildOffset(offset, Offset.Type.CREATE); - this.client - .prepareIndex(ContentIndex.INDEX_NAME) - .setId(this.resourceId) - .setSource(mOffset.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get(); - } -} diff --git a/plugins/content-manager/src/test/java/com/wazuh/contentmanager/cti/catalog/service/ContentUpdaterTests.java b/plugins/content-manager/src/test/java/com/wazuh/contentmanager/cti/catalog/service/ContentUpdaterTests.java deleted file mode 100644 index 5783e24e..00000000 --- a/plugins/content-manager/src/test/java/com/wazuh/contentmanager/cti/catalog/service/ContentUpdaterTests.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Copyright (C) 2024, Wazuh Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package com.wazuh.contentmanager.cti.catalog.service; - -import com.wazuh.contentmanager.cti.catalog.model.Changes; -import com.wazuh.contentmanager.cti.catalog.model.ConsumerInfo; -import com.wazuh.contentmanager.cti.catalog.model.Offset; -import com.wazuh.contentmanager.cti.catalog.model.Operation; -import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.settings.Settings; -import org.opensearch.env.Environment; -import org.opensearch.test.OpenSearchIntegTestCase; -import org.junit.Before; - -import java.util.ArrayList; -import java.util.List; - -import com.wazuh.contentmanager.client.CTIClient; -import com.wazuh.contentmanager.cti.catalog.index.ContentIndex; -import com.wazuh.contentmanager.cti.catalog.index.ConsumersIndex; -import com.wazuh.contentmanager.settings.PluginSettings; -import com.wazuh.contentmanager.utils.Privileged; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Mockito; - -import static org.mockito.Mockito.*; - -/** Tests of the Content Manager's updater */ -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE) -public class ContentUpdaterTests extends OpenSearchIntegTestCase { - private ConsumersIndex consumersIndex; - private ContentIndex contentIndex; - private CTIClient ctiClient; - private Privileged privilegedSpy; - private ConsumerInfo consumerInfo; - private ContentUpdater updater; - - @Mock private Environment mockEnvironment; - @Mock private ClusterService mockClusterService; - @InjectMocks private PluginSettings pluginSettings; - - /** - * Set up the tests - * - * @throws Exception rethrown from parent method - */ - @Before - public void setup() throws Exception { - super.setUp(); - this.ctiClient = mock(CTIClient.class); - - Settings settings = Settings.builder().put("content_manager.max_changes", 1000).build(); - this.mockEnvironment = mock(Environment.class); - when(this.mockEnvironment.settings()).thenReturn(settings); - this.pluginSettings = - PluginSettings.getInstance(this.mockEnvironment.settings(), this.mockClusterService); - - this.consumersIndex = mock(ConsumersIndex.class); - this.contentIndex = mock(ContentIndex.class); - this.privilegedSpy = Mockito.spy(Privileged.class); - this.updater = - Mockito.spy( - new ContentUpdater( - this.ctiClient, - this.consumersIndex, - this.contentIndex, - this.privilegedSpy, - this.pluginSettings)); - this.consumerInfo = mock(ConsumerInfo.class); - } - - /** Test Fetch and apply no new updates */ - public void testUpdateNoChanges() { - // Mock current and latest offset. - doReturn(this.consumerInfo).when(this.consumersIndex).get(anyString(), anyString()); - // Act - this.updater.update(); - // Assert applyChangesToContextIndex is not called. - verify(this.updater, never()).applyChanges(any()); - } - - /** Test fetch and apply new updates */ - public void testUpdateNewChanges() { - long offsetsAmount = 3999L; - // Mock current and latest offset. - doReturn(0L).when(this.consumerInfo).getOffset(); - doReturn(offsetsAmount).when(this.consumerInfo).getLastOffset(); - // Mock getContextChanges method. - doReturn(generateContextChanges((int) offsetsAmount)) - .when(this.privilegedSpy) - .getChanges(any(CTIClient.class), anyLong(), anyLong()); - // Mock ContentIndex.patch - doReturn(true).when(this.updater).applyChanges(any()); - doReturn(this.consumerInfo).when(this.consumersIndex).get(anyString(), anyString()); - // Act - doNothing().when(this.consumerInfo).setOffset(anyLong()); - doNothing().when(this.consumerInfo).setLastOffset(anyLong()); - this.updater.update(); - // Assert applyChangesToContextIndex is called 4 times (one each 1000 starting from 0). - verify(this.updater, times(4)).applyChanges(any()); - } - - /** Test error fetching changes */ - public void testUpdateErrorFetchingChanges() { - long offsetsAmount = 3999L; - // Mock current and latest offset. - doReturn(0L).when(this.consumerInfo).getOffset(); - doReturn(offsetsAmount).when(this.consumerInfo).getLastOffset(); - // Mock getContextChanges method. - doReturn(null).when(this.privilegedSpy).getChanges(any(CTIClient.class), anyLong(), anyLong()); - doNothing().when(this.consumerInfo).setOffset(anyLong()); - doNothing().when(this.consumerInfo).setLastOffset(anyLong()); - doReturn(this.consumerInfo).when(this.consumersIndex).get(anyString(), anyString()); - // Act - boolean updated = this.updater.update(); - // Assert - assertFalse(updated); - } - - /** Test error on applyChangesToContextIndex method (method return false) */ - public void testUpdateErrorOnPatchContextIndex() { - long offsetsAmount = 3999L; - // Mock current and latest offset. - doReturn(0L).when(this.consumerInfo).getOffset(); - doReturn(offsetsAmount).when(this.consumerInfo).getLastOffset(); - // Mock getContextChanges method. - doReturn(generateContextChanges((int) offsetsAmount)) - .when(this.privilegedSpy) - .getChanges(any(CTIClient.class), anyLong(), anyLong()); - // Mock applyChangesToContextIndex method. - doReturn(false).when(this.updater).applyChanges(any()); - doNothing().when(this.consumerInfo).setOffset(anyLong()); - doNothing().when(this.consumerInfo).setLastOffset(anyLong()); - doReturn(this.consumerInfo).when(this.consumersIndex).get(anyString(), anyString()); - // Act - boolean updated = this.updater.update(); - // Assert - assertFalse(updated); - verify(this.consumerInfo, times(1)).setOffset(0L); - verify(this.consumerInfo, times(1)).setLastOffset(0L); - } - - /** - * Generate context changes - * - * @param size of the generated changes list - * @return A ContextChanges object - */ - public Changes generateContextChanges(int size) { - List offsets = new ArrayList<>(); - for (int i = 0; i < size; i++) { - offsets.add( - new Offset( - "context", - (long) i, - "resource", - Offset.Type.UPDATE, - 0L, - List.of(new Operation(Operation.OP, Operation.PATH, Operation.FROM, Operation.VALUE)), - null)); - } - return new Changes(offsets); - } -} diff --git a/plugins/content-manager/src/test/java/com/wazuh/contentmanager/cti/catalog/service/SnapshotServiceImplTests.java b/plugins/content-manager/src/test/java/com/wazuh/contentmanager/cti/catalog/service/SnapshotServiceImplTests.java index 3bf51d6f..bb606eeb 100644 --- a/plugins/content-manager/src/test/java/com/wazuh/contentmanager/cti/catalog/service/SnapshotServiceImplTests.java +++ b/plugins/content-manager/src/test/java/com/wazuh/contentmanager/cti/catalog/service/SnapshotServiceImplTests.java @@ -16,7 +16,6 @@ */ package com.wazuh.contentmanager.cti.catalog.service; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.wazuh.contentmanager.cti.catalog.client.SnapshotClient; import com.wazuh.contentmanager.cti.catalog.index.ConsumersIndex; @@ -26,7 +25,6 @@ import com.wazuh.contentmanager.cti.catalog.model.RemoteConsumer; import com.wazuh.contentmanager.settings.PluginSettings; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.index.IndexRequest; -import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.env.Environment; import org.opensearch.test.OpenSearchTestCase; @@ -61,7 +59,6 @@ public class SnapshotServiceImplTests extends OpenSearchTestCase { @Mock private ConsumersIndex consumersIndex; @Mock private ContentIndex contentIndexMock; @Mock private Environment environment; - @Mock private ClusterService clusterService; @Mock private RemoteConsumer remoteConsumer; private AutoCloseable closeable; @@ -80,11 +77,11 @@ public class SnapshotServiceImplTests extends OpenSearchTestCase { when(this.environment.tmpDir()).thenReturn(this.tempDir); when(this.environment.settings()).thenReturn(settings); - PluginSettings.getInstance(settings, this.clusterService); + PluginSettings.getInstance(settings); List contentIndices = Collections.singletonList(this.contentIndexMock); String context = "test-context"; String consumer = "test-consumer"; - this.snapshotService = new SnapshotServiceImpl(context, consumer, contentIndices, consumersIndex, environment); + this.snapshotService = new SnapshotServiceImpl(context, consumer, contentIndices, this.consumersIndex, this.environment); this.snapshotService.setSnapshotClient(this.snapshotClient); } @@ -101,12 +98,12 @@ public class SnapshotServiceImplTests extends OpenSearchTestCase { * Tests that the initialization aborts gracefully if the snapshot URL is missing. */ public void testInitialize_EmptyUrl() throws IOException, URISyntaxException { - when(remoteConsumer.getSnapshotLink()).thenReturn(""); + when(this.remoteConsumer.getSnapshotLink()).thenReturn(""); - this.snapshotService.initialize(remoteConsumer); + this.snapshotService.initialize(this.remoteConsumer); - verify(snapshotClient, never()).downloadFile(anyString()); - verify(contentIndexMock, never()).clear(); + verify(this.snapshotClient, never()).downloadFile(anyString()); + verify(this.contentIndexMock, never()).clear(); } /** @@ -114,13 +111,13 @@ public class SnapshotServiceImplTests extends OpenSearchTestCase { */ public void testInitialize_DownloadFails() throws IOException, URISyntaxException { String url = "http://example.com/snapshot.zip"; - when(remoteConsumer.getSnapshotLink()).thenReturn(url); - when(snapshotClient.downloadFile(url)).thenReturn(null); + when(this.remoteConsumer.getSnapshotLink()).thenReturn(url); + when(this.snapshotClient.downloadFile(url)).thenReturn(null); - this.snapshotService.initialize(remoteConsumer); + this.snapshotService.initialize(this.remoteConsumer); - verify(snapshotClient).downloadFile(url); - verify(contentIndexMock, never()).clear(); + verify(this.snapshotClient).downloadFile(url); + verify(this.contentIndexMock, never()).clear(); } /** @@ -134,20 +131,20 @@ public class SnapshotServiceImplTests extends OpenSearchTestCase { // Mock String url = "http://example.com/snapshot.zip"; long offset = 100L; - when(remoteConsumer.getSnapshotLink()).thenReturn(url); - when(remoteConsumer.getOffset()).thenReturn(offset); + when(this.remoteConsumer.getSnapshotLink()).thenReturn(url); + when(this.remoteConsumer.getOffset()).thenReturn(offset); Path zipPath = createZipFileWithContent("data.json", "{\"payload\": {\"type\": \"kvdb\", \"document\": {\"id\": \"12345678\", \"title\": \"Test Kvdb\"}}}" ); - when(snapshotClient.downloadFile(url)).thenReturn(zipPath); + when(this.snapshotClient.downloadFile(url)).thenReturn(zipPath); // Act - this.snapshotService.initialize(remoteConsumer); + this.snapshotService.initialize(this.remoteConsumer); // Assert - verify(contentIndexMock).clear(); + verify(this.contentIndexMock).clear(); ArgumentCaptor bulkCaptor = ArgumentCaptor.forClass(BulkRequest.class); - verify(contentIndexMock, atLeastOnce()).executeBulk(bulkCaptor.capture()); + verify(this.contentIndexMock, atLeastOnce()).executeBulk(bulkCaptor.capture()); BulkRequest request = bulkCaptor.getValue(); assertEquals(1, request.numberOfActions()); @@ -157,7 +154,7 @@ public class SnapshotServiceImplTests extends OpenSearchTestCase { assertEquals("12345678", indexRequest.id()); ArgumentCaptor consumerCaptor = ArgumentCaptor.forClass(LocalConsumer.class); - verify(consumersIndex).setConsumer(consumerCaptor.capture()); + verify(this.consumersIndex).setConsumer(consumerCaptor.capture()); assertEquals(offset, consumerCaptor.getValue().getLocalOffset()); } @@ -167,18 +164,18 @@ public class SnapshotServiceImplTests extends OpenSearchTestCase { public void testInitialize_SkipPolicyType() throws IOException, URISyntaxException { // Mock String url = "http://example.com/policy.zip"; - when(remoteConsumer.getSnapshotLink()).thenReturn(url); + when(this.remoteConsumer.getSnapshotLink()).thenReturn(url); Path zipPath = createZipFileWithContent("policy.json", "{\"payload\": {\"type\": \"policy\", \"document\": {\"id\": \"p1\"}}}" ); - when(snapshotClient.downloadFile(url)).thenReturn(zipPath); + when(this.snapshotClient.downloadFile(url)).thenReturn(zipPath); // Act - this.snapshotService.initialize(remoteConsumer); + this.snapshotService.initialize(this.remoteConsumer); // Assert - verify(contentIndexMock, never()).executeBulk(any(BulkRequest.class)); + verify(this.contentIndexMock, never()).executeBulk(any(BulkRequest.class)); } /** @@ -187,18 +184,18 @@ public class SnapshotServiceImplTests extends OpenSearchTestCase { public void testInitialize_EnrichDecoderWithYaml() throws IOException, URISyntaxException { // Mock String url = "http://example.com/decoder.zip"; - when(remoteConsumer.getSnapshotLink()).thenReturn(url); + when(this.remoteConsumer.getSnapshotLink()).thenReturn(url); String jsonContent = "{\"payload\": {\"type\": \"decoder\", \"document\": {\"name\": \"syslog\", \"parent\": \"root\"}}}"; Path zipPath = createZipFileWithContent("decoder.json", jsonContent); - when(snapshotClient.downloadFile(url)).thenReturn(zipPath); + when(this.snapshotClient.downloadFile(url)).thenReturn(zipPath); // Act - this.snapshotService.initialize(remoteConsumer); + this.snapshotService.initialize(this.remoteConsumer); // Assert ArgumentCaptor bulkCaptor = ArgumentCaptor.forClass(BulkRequest.class); - verify(contentIndexMock).executeBulk(bulkCaptor.capture()); + verify(this.contentIndexMock).executeBulk(bulkCaptor.capture()); IndexRequest request = (IndexRequest) bulkCaptor.getValue().requests().getFirst(); String source = request.source().utf8ToString(); @@ -212,20 +209,20 @@ public class SnapshotServiceImplTests extends OpenSearchTestCase { public void testInitialize_PreprocessSigmaId() throws IOException, URISyntaxException { // Mock String url = "http://example.com/sigma.zip"; - when(remoteConsumer.getSnapshotLink()).thenReturn(url); + when(this.remoteConsumer.getSnapshotLink()).thenReturn(url); String jsonContent = "{\"payload\": {\"type\": \"rule\", \"document\": {\"id\": \"R1\", \"related\": {\"sigma_id\": \"S-123\", \"type\": \"test-value\"}}}}"; Path zipPath = createZipFileWithContent("sigma.json", jsonContent); - when(snapshotClient.downloadFile(url)).thenReturn(zipPath); + when(this.snapshotClient.downloadFile(url)).thenReturn(zipPath); // Act - this.snapshotService.initialize(remoteConsumer); + this.snapshotService.initialize(this.remoteConsumer); // Assert ArgumentCaptor bulkCaptor = ArgumentCaptor.forClass(BulkRequest.class); - verify(contentIndexMock).executeBulk(bulkCaptor.capture()); + verify(this.contentIndexMock).executeBulk(bulkCaptor.capture()); - IndexRequest request = (IndexRequest) bulkCaptor.getValue().requests().get(0); + IndexRequest request = (IndexRequest) bulkCaptor.getValue().requests().getFirst(); String source = request.source().utf8ToString(); assertFalse("Should not contain sigma_id", source.contains("\"sigma_id\"")); @@ -238,7 +235,7 @@ public class SnapshotServiceImplTests extends OpenSearchTestCase { public void testInitialize_InvalidJsonStructure() throws IOException, URISyntaxException { // Mock String url = "http://example.com/invalid.zip"; - when(remoteConsumer.getSnapshotLink()).thenReturn(url); + when(this.remoteConsumer.getSnapshotLink()).thenReturn(url); String jsonContent = "{}\n" + // Missing payload @@ -246,13 +243,13 @@ public class SnapshotServiceImplTests extends OpenSearchTestCase { "{\"payload\": {\"type\": \"valid\", \"no_doc\": {}}}"; // Missing document Path zipPath = createZipFileWithContent("invalid.json", jsonContent); - when(snapshotClient.downloadFile(url)).thenReturn(zipPath); + when(this.snapshotClient.downloadFile(url)).thenReturn(zipPath); // Act - this.snapshotService.initialize(remoteConsumer); + this.snapshotService.initialize(this.remoteConsumer); // Assert - verify(contentIndexMock, never()).executeBulk(any(BulkRequest.class)); + verify(this.contentIndexMock, never()).executeBulk(any(BulkRequest.class)); } /** @@ -261,18 +258,18 @@ public class SnapshotServiceImplTests extends OpenSearchTestCase { public void testInitialize_PreprocessSigmaIdInArray() throws IOException, URISyntaxException { // Mock String url = "http://example.com/sigma_array.zip"; - when(remoteConsumer.getSnapshotLink()).thenReturn(url); + when(this.remoteConsumer.getSnapshotLink()).thenReturn(url); String jsonContent = "{\"payload\": {\"type\": \"rule\", \"document\": {\"id\": \"R2\", \"related\": [{\"sigma_id\": \"999\"}]}}}"; Path zipPath = createZipFileWithContent("sigma_array.json", jsonContent); - when(snapshotClient.downloadFile(url)).thenReturn(zipPath); + when(this.snapshotClient.downloadFile(url)).thenReturn(zipPath); // Act - this.snapshotService.initialize(remoteConsumer); + this.snapshotService.initialize(this.remoteConsumer); // Assert ArgumentCaptor bulkCaptor = ArgumentCaptor.forClass(BulkRequest.class); - verify(contentIndexMock).executeBulk(bulkCaptor.capture()); + verify(this.contentIndexMock).executeBulk(bulkCaptor.capture()); String source = ((IndexRequest) bulkCaptor.getValue().requests().getFirst()).source().utf8ToString(); @@ -287,7 +284,7 @@ public class SnapshotServiceImplTests extends OpenSearchTestCase { public void testInitialize_SkipInvalidJson() throws IOException, URISyntaxException { // Mock String url = "http://example.com/corrupt.zip"; - when(remoteConsumer.getSnapshotLink()).thenReturn(url); + when(this.remoteConsumer.getSnapshotLink()).thenReturn(url); String jsonContent = "{\"payload\": {\"type\": \"reputation\", \"document\": {\"id\": \"1\", \"ip\": \"1.1.1.1\"}}}\n" + @@ -295,14 +292,14 @@ public class SnapshotServiceImplTests extends OpenSearchTestCase { "{\"payload\": {\"type\": \"reputation\", \"document\": {\"id\": \"2\", \"ip\": \"2.2.2.2\"}}}"; Path zipPath = createZipFileWithContent("mixed.json", jsonContent); - when(snapshotClient.downloadFile(url)).thenReturn(zipPath); + when(this.snapshotClient.downloadFile(url)).thenReturn(zipPath); // Act - this.snapshotService.initialize(remoteConsumer); + this.snapshotService.initialize(this.remoteConsumer); // Assert ArgumentCaptor bulkCaptor = ArgumentCaptor.forClass(BulkRequest.class); - verify(contentIndexMock, atLeastOnce()).executeBulk(bulkCaptor.capture()); + verify(this.contentIndexMock, atLeastOnce()).executeBulk(bulkCaptor.capture()); // We expect exactly 2 valid actions (Line 1 and Line 3), skipping Line 2 int totalActions = bulkCaptor.getAllValues().stream() @@ -318,20 +315,20 @@ public class SnapshotServiceImplTests extends OpenSearchTestCase { public void testInitialize_DecoderYamlKeyOrdering() throws IOException, URISyntaxException { // Mock String url = "http://example.com/decoder_order.zip"; - when(remoteConsumer.getSnapshotLink()).thenReturn(url); + when(this.remoteConsumer.getSnapshotLink()).thenReturn(url); String jsonContent = "{\"payload\": {\"type\": \"decoder\", \"document\": " + "{\"check\": \"some_regex\", \"name\": \"ssh-decoder\", \"parents\": [\"root\"]}}}"; Path zipPath = createZipFileWithContent("decoder_order.json", jsonContent); - when(snapshotClient.downloadFile(url)).thenReturn(zipPath); + when(this.snapshotClient.downloadFile(url)).thenReturn(zipPath); // Act - this.snapshotService.initialize(remoteConsumer); + this.snapshotService.initialize(this.remoteConsumer); // Assert ArgumentCaptor bulkCaptor = ArgumentCaptor.forClass(BulkRequest.class); - verify(contentIndexMock).executeBulk(bulkCaptor.capture()); + verify(this.contentIndexMock).executeBulk(bulkCaptor.capture()); IndexRequest request = (IndexRequest) bulkCaptor.getValue().requests().getFirst(); String source = request.source().utf8ToString(); diff --git a/plugins/content-manager/src/test/java/com/wazuh/contentmanager/cti/catalog/service/UpdateServiceImplTests.java b/plugins/content-manager/src/test/java/com/wazuh/contentmanager/cti/catalog/service/UpdateServiceImplTests.java new file mode 100644 index 00000000..46f86688 --- /dev/null +++ b/plugins/content-manager/src/test/java/com/wazuh/contentmanager/cti/catalog/service/UpdateServiceImplTests.java @@ -0,0 +1,289 @@ +package com.wazuh.contentmanager.cti.catalog.service; + +import com.google.gson.JsonObject; +import com.wazuh.contentmanager.cti.catalog.client.ApiClient; +import com.wazuh.contentmanager.cti.catalog.index.ConsumersIndex; +import com.wazuh.contentmanager.cti.catalog.index.ContentIndex; +import com.wazuh.contentmanager.cti.catalog.model.LocalConsumer; +import com.wazuh.contentmanager.settings.PluginSettings; +import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; +import org.apache.hc.core5.http.ContentType; +import org.junit.After; +import org.junit.Before; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.opensearch.action.get.GetResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.test.OpenSearchTestCase; + +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class UpdateServiceImplTests extends OpenSearchTestCase { + + private UpdateServiceImpl updateService; + private AutoCloseable closeable; + + @Mock private ApiClient apiClient; + @Mock private ConsumersIndex consumersIndex; + @Mock private ContentIndex ruleIndex; + @Mock private ContentIndex decoderIndex; + @Mock private GetResponse getResponse; + + private Map indices; + private static final String CONTEXT = "rules_dev"; + private static final String CONSUMER = "test_consumer"; + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + this.closeable = MockitoAnnotations.openMocks(this); + + PluginSettings.getInstance(Settings.EMPTY); + + this.indices = new HashMap<>(); + this.indices.put("rule", this.ruleIndex); + this.indices.put("decoder", this.decoderIndex); + + this.updateService = new UpdateServiceImpl(CONTEXT, CONSUMER, apiClient, consumersIndex, indices); + } + + @After + @Override + public void tearDown() throws Exception { + if (this.closeable != null) { + this.closeable.close(); + } + super.tearDown(); + } + + /** + * Tests a successful update flow containing CREATE, UPDATE, and DELETE operations. + */ + public void testUpdate_Success() throws Exception { + // Response + String changesJson = "{\n" + + " \"data\": [\n" + + " {\n" + + " \"offset\": 10,\n" + + " \"resource\": \"rule-1\",\n" + + " \"type\": \"CREATE\",\n" + + " \"payload\": { \"type\": \"rule\", \"id\": \"rule-1\", \"name\": \"Rule One\" }\n" + + " },\n" + + " {\n" + + " \"offset\": 11,\n" + + " \"resource\": \"rule-2\",\n" + + " \"type\": \"UPDATE\",\n" + + " \"operations\": [ { \"op\": \"replace\", \"path\": \"/name\", \"value\": \"Updated Rule\" } ]\n" + + " },\n" + + " {\n" + + " \"offset\": 12,\n" + + " \"resource\": \"decoder-1\",\n" + + " \"type\": \"DELETE\"\n" + + " }\n" + + " ]\n" + + "}"; + + // Mock + when(this.apiClient.getChanges(anyString(), anyString(), anyLong(), anyLong())) + .thenReturn(SimpleHttpResponse.create(200, changesJson.getBytes(StandardCharsets.UTF_8), ContentType.APPLICATION_JSON)); + + when(this.ruleIndex.exists("rule-2")).thenReturn(true); + when(this.decoderIndex.exists("decoder-1")).thenReturn(true); + + when(this.consumersIndex.getConsumer(CONTEXT, CONSUMER)).thenReturn(this.getResponse); + when(this.getResponse.isExists()).thenReturn(true); + when(this.getResponse.getSourceAsString()).thenReturn("{\"local_offset\": 9, \"remote_offset\": 100, \"snapshot_link\": \"http://snap\"}"); + + // Act + this.updateService.update(9, 12); + + // Assert + // Verify CREATE + verify(this.ruleIndex).create(eq("rule-1"), any(JsonObject.class)); + + // Verify UPDATE + verify(this.ruleIndex).update(eq("rule-2"), any(List.class)); + + // Verify DELETE + verify(this.decoderIndex).delete("decoder-1"); + + // Verify Consumer State Update + ArgumentCaptor consumerCaptor = ArgumentCaptor.forClass(LocalConsumer.class); + verify(this.consumersIndex).setConsumer(consumerCaptor.capture()); + + LocalConsumer updated = consumerCaptor.getValue(); + assertEquals(12, updated.getLocalOffset()); + assertEquals(CONSUMER, updated.getName()); + } + + /** + * Tests that "policy" resources are skipped but the offset is still tracked. + */ + public void testUpdate_SkipPolicy() throws Exception { + // Response + String changesJson = "{\n" + + " \"data\": [\n" + + " {\n" + + " \"offset\": 20,\n" + + " \"resource\": \"policy-1\",\n" + + " \"type\": \"CREATE\",\n" + + " \"payload\": { \"type\": \"policy\", \"content\": \"...\" }\n" + + " }\n" + + " ]\n" + + "}"; + + when(this.apiClient.getChanges(anyString(), anyString(), anyLong(), anyLong())) + .thenReturn(SimpleHttpResponse.create(200, changesJson.getBytes(StandardCharsets.UTF_8), ContentType.APPLICATION_JSON)); + + // Mock + when(this.consumersIndex.getConsumer(CONTEXT, CONSUMER)).thenReturn(this.getResponse); + when(this.getResponse.isExists()).thenReturn(false); + + // Act + this.updateService.update(19, 20); + + // Assert + verify(this.ruleIndex, never()).create(anyString(), any()); + verify(this.decoderIndex, never()).create(anyString(), any()); + + ArgumentCaptor consumerCaptor = ArgumentCaptor.forClass(LocalConsumer.class); + verify(this.consumersIndex).setConsumer(consumerCaptor.capture()); + assertEquals(20, consumerCaptor.getValue().getLocalOffset()); + } + + /** + * Tests handling of API failures. + */ + public void testUpdate_ApiFailure() throws Exception { + // Mock + when(apiClient.getChanges(anyString(), anyString(), anyLong(), anyLong())) + .thenReturn(SimpleHttpResponse.create(500, "Internal Error", ContentType.TEXT_PLAIN)); + + // Act + updateService.update(1, 5); + + // Assert + verify(ruleIndex, never()).create(anyString(), any()); + verify(consumersIndex, never()).setConsumer(any()); + } + + /** + * Tests that the consumer state is reset to 0 if an exception occurs during processing. + */ + public void testUpdate_ExceptionResetsConsumer() throws Exception { + // Response + String changesJson = "{\n" + + " \"data\": [\n" + + " {\n" + + " \"offset\": 30,\n" + + " \"resource\": \"rule-bad\",\n" + + " \"type\": \"CREATE\",\n" + + " \"payload\": { \"type\": \"rule\" }\n" + + " }\n" + + " ]\n" + + "}"; + + // Mock + when(this.apiClient.getChanges(anyString(), anyString(), anyLong(), anyLong())) + .thenReturn(SimpleHttpResponse.create(200, changesJson.getBytes(StandardCharsets.UTF_8), ContentType.APPLICATION_JSON)); + + doThrow(new RuntimeException("Simulated Indexing Failure")) + .when(this.ruleIndex).create(anyString(), any()); + + // Act + this.updateService.update(29, 30); + + ArgumentCaptor consumerCaptor = ArgumentCaptor.forClass(LocalConsumer.class); + verify(this.consumersIndex).setConsumer(consumerCaptor.capture()); + + LocalConsumer resetConsumer = consumerCaptor.getValue(); + assertEquals(0, resetConsumer.getLocalOffset()); + assertEquals(CONSUMER, resetConsumer.getName()); + } + + /** + * Tests CREATE operation when the 'type' in payload doesn't map to any known index. + */ + public void testUpdate_UnknownType_Create() throws Exception { + // Response + String changesJson = "{\n" + + " \"data\": [\n" + + " {\n" + + " \"offset\": 40,\n" + + " \"resource\": \"unknown-1\",\n" + + " \"type\": \"CREATE\",\n" + + " \"payload\": { \"type\": \"unknown_thing\", \"data\": \"...\" }\n" + + " }\n" + + " ]\n" + + "}"; + + // Mock + when(this.apiClient.getChanges(anyString(), anyString(), anyLong(), anyLong())) + .thenReturn(SimpleHttpResponse.create(200, changesJson.getBytes(StandardCharsets.UTF_8), ContentType.APPLICATION_JSON)); + + when(this.consumersIndex.getConsumer(CONTEXT, CONSUMER)).thenReturn(this.getResponse); + when(this.getResponse.isExists()).thenReturn(true); + when(this.getResponse.getSourceAsString()).thenReturn("{}"); + + // Act + this.updateService.update(39, 40); + + // Assert + verify(this.ruleIndex, never()).create(anyString(), any()); + verify(this.decoderIndex, never()).create(anyString(), any()); + + ArgumentCaptor captor = ArgumentCaptor.forClass(LocalConsumer.class); + verify(this.consumersIndex).setConsumer(captor.capture()); + assertEquals(40, captor.getValue().getLocalOffset()); + } + + /** + * Tests UPDATE/DELETE operation when the resource ID is not found in any index. + */ + public void testUpdate_ResourceNotFound() throws Exception { + // Response + String changesJson = "{\n" + + " \"data\": [\n" + + " {\n" + + " \"offset\": 50,\n" + + " \"resource\": \"fake-id\",\n" + + " \"type\": \"DELETE\"\n" + + " }\n" + + " ]\n" + + "}"; + + // Mock + when(this.apiClient.getChanges(anyString(), anyString(), anyLong(), anyLong())) + .thenReturn(SimpleHttpResponse.create(200, changesJson.getBytes(StandardCharsets.UTF_8), ContentType.APPLICATION_JSON)); + + when(this.ruleIndex.exists("fake-id")).thenReturn(false); + when(this.decoderIndex.exists("fake-id")).thenReturn(false); + + when(this.consumersIndex.getConsumer(CONTEXT, CONSUMER)).thenReturn(this.getResponse); + when(this.getResponse.isExists()).thenReturn(true); + when(this.getResponse.getSourceAsString()).thenReturn("{}"); + + // Act + this.updateService.update(49, 50); + + // Assert + verify(this.ruleIndex, never()).delete(anyString()); + verify(this.decoderIndex, never()).delete(anyString()); + + verify(this.consumersIndex).setConsumer(any(LocalConsumer.class)); + } +} diff --git a/plugins/content-manager/src/test/java/com/wazuh/contentmanager/rest/RestPostUpdateActionTests.java b/plugins/content-manager/src/test/java/com/wazuh/contentmanager/rest/RestPostUpdateActionTests.java index ed1e47ae..86fa3f36 100644 --- a/plugins/content-manager/src/test/java/com/wazuh/contentmanager/rest/RestPostUpdateActionTests.java +++ b/plugins/content-manager/src/test/java/com/wazuh/contentmanager/rest/RestPostUpdateActionTests.java @@ -2,20 +2,21 @@ package com.wazuh.contentmanager.rest; import com.wazuh.contentmanager.cti.console.CtiConsole; import com.wazuh.contentmanager.cti.console.model.Token; +import com.wazuh.contentmanager.jobscheduler.jobs.CatalogSyncJob; import com.wazuh.contentmanager.rest.model.RestResponse; import com.wazuh.contentmanager.rest.services.RestPostUpdateAction; import org.junit.Before; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.BytesRestResponse; import org.opensearch.test.OpenSearchTestCase; -import org.opensearch.core.rest.RestStatus; import java.io.IOException; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; public class RestPostUpdateActionTests extends OpenSearchTestCase { private CtiConsole console; + private CatalogSyncJob catalogSyncJob; private RestPostUpdateAction action; /** @@ -28,16 +29,18 @@ public class RestPostUpdateActionTests extends OpenSearchTestCase { public void setUp() throws Exception { super.setUp(); this.console = mock(CtiConsole.class); - this.action = new RestPostUpdateAction(this.console); + this.catalogSyncJob = mock(CatalogSyncJob.class); + this.action = new RestPostUpdateAction(this.console, this.catalogSyncJob); } /** Test the {@link RestPostUpdateAction#handleRequest()} method when the token is created (mock). * The expected response is: {200, Token} */ - public void testGetToken202() throws IOException { + public void testHandleRequest_Accepted() throws IOException { // Mock Token token = new Token("test_token", "test_type"); when(this.console.getToken()).thenReturn(token); + when(this.catalogSyncJob.isRunning()).thenReturn(false); // Act BytesRestResponse bytesRestResponse = this.action.handleRequest(); @@ -46,15 +49,18 @@ public class RestPostUpdateActionTests extends OpenSearchTestCase { RestResponse expectedResponse = new RestResponse("Update accepted", RestStatus.ACCEPTED.getStatus()); // Assert - assertTrue(bytesRestResponse.content().utf8ToString().contains(expectedResponse.getMessage())); - assertTrue(bytesRestResponse.content().utf8ToString().contains(String.valueOf(expectedResponse.getStatus()))); assertEquals(RestStatus.ACCEPTED, bytesRestResponse.status()); + String content = bytesRestResponse.content().utf8ToString(); + assertTrue(content.contains(expectedResponse.getMessage())); + + // Verify trigger was called + verify(this.catalogSyncJob, times(1)).trigger(); } /** Test the {@link RestPostUpdateAction#handleRequest()} method when the token has not been created (mock). * The expected response is: {404, RestResponse} */ - public void testGetToken404() throws IOException { + public void testHandleRequest_NoToken() throws IOException { // Mock when(this.console.getToken()).thenReturn(null); @@ -65,16 +71,36 @@ public class RestPostUpdateActionTests extends OpenSearchTestCase { RestResponse expectedResponse = new RestResponse("Token not found", RestStatus.NOT_FOUND.getStatus()); // Assert - assertTrue(bytesRestResponse.content().utf8ToString().contains(expectedResponse.getMessage())); - assertTrue(bytesRestResponse.content().utf8ToString().contains(String.valueOf(expectedResponse.getStatus()))); assertEquals(RestStatus.NOT_FOUND, bytesRestResponse.status()); + String content = bytesRestResponse.content().utf8ToString(); + assertTrue(content.contains(expectedResponse.getMessage())); + + // Verify trigger was NOT called + verify(this.catalogSyncJob, never()).trigger(); } /** Test the {@link RestPostUpdateAction#handleRequest()} method when there is already a request being performed. * The expected response is: {409, RestResponse} */ - public void testGetToken409() throws IOException { - // TODO + public void testHandleRequest_Conflict() throws IOException { + // Mock + Token token = new Token("test_token", "test_type"); + when(this.console.getToken()).thenReturn(token); + when(this.catalogSyncJob.isRunning()).thenReturn(true); + + // Act + BytesRestResponse bytesRestResponse = this.action.handleRequest(); + + // Expected response + RestResponse expectedResponse = new RestResponse("An update operation is already in progress", RestStatus.CONFLICT.getStatus()); + + // Assert + assertEquals(RestStatus.CONFLICT, bytesRestResponse.status()); + String content = bytesRestResponse.content().utf8ToString(); + assertTrue(content.contains("An update operation is already in progress")); + + // Verify trigger was NOT called + verify(this.catalogSyncJob, never()).trigger(); } /** Test the {@link RestPostUpdateAction#handleRequest()} method when the rate limit is exceeded. @@ -83,5 +109,4 @@ public class RestPostUpdateActionTests extends OpenSearchTestCase { public void testGetToken429() throws IOException { // TODO } - }