From 0d86adf4251512b602e55d1fc6a39f15774260d6 Mon Sep 17 00:00:00 2001 From: Swati Panchal Date: Fri, 20 Jun 2025 12:46:14 +0530 Subject: [PATCH 1/5] chore: ADDON-80802 PSA implementation with uuid flag --- pytest_splunk_addon/app_test_generator.py | 3 ++- .../event_ingestors/hec_event_ingestor.py | 2 ++ .../event_ingestors/ingestor_helper.py | 2 +- .../fields_tests/test_generator.py | 9 ++++++--- .../fields_tests/test_templates.py | 11 ++++++++++- .../pytest_splunk_addon_data_parser.py | 5 +++-- .../sample_generation/sample_event.py | 3 +++ .../sample_generation/sample_generator.py | 5 +++-- .../sample_generation/sample_stanza.py | 3 ++- .../sample_generation/sample_xdist_generator.py | 17 ++++++++++++----- pytest_splunk_addon/splunk.py | 11 +++++++++++ 11 files changed, 55 insertions(+), 16 deletions(-) diff --git a/pytest_splunk_addon/app_test_generator.py b/pytest_splunk_addon/app_test_generator.py index a4f9c8e5..e4396006 100644 --- a/pytest_splunk_addon/app_test_generator.py +++ b/pytest_splunk_addon/app_test_generator.py @@ -48,8 +48,9 @@ def __init__(self, pytest_config): store_events = self.pytest_config.getoption("store_events") config_path = self.pytest_config.getoption("splunk_data_generator") + ingest_with_uuid = self.pytest_config.getoption("ingest_with_uuid") sample_generator = SampleXdistGenerator( - self.pytest_config.getoption("splunk_app"), config_path + self.pytest_config.getoption("splunk_app"), ingest_with_uuid, config_path ) store_sample = sample_generator.get_samples(store_events) self.tokenized_events = store_sample.get("tokenized_events") diff --git a/pytest_splunk_addon/event_ingestors/hec_event_ingestor.py b/pytest_splunk_addon/event_ingestors/hec_event_ingestor.py index bc806822..754d7266 100644 --- a/pytest_splunk_addon/event_ingestors/hec_event_ingestor.py +++ b/pytest_splunk_addon/event_ingestors/hec_event_ingestor.py @@ -93,6 +93,8 @@ def ingest(self, events, thread_count): "event": event.event, "index": event.metadata.get("index", "main"), } + if event.metadata["ingest_with_uuid"] == "true": + event_dict["fields"] = {"unique_identifier": event.unique_identifier} if event.metadata.get("host_type") in ("plugin", None): host = event.metadata.get("host") diff --git a/pytest_splunk_addon/event_ingestors/ingestor_helper.py b/pytest_splunk_addon/event_ingestors/ingestor_helper.py index c73e13fc..30dc8c50 100644 --- a/pytest_splunk_addon/event_ingestors/ingestor_helper.py +++ b/pytest_splunk_addon/event_ingestors/ingestor_helper.py @@ -95,7 +95,7 @@ def ingest_events( thread_count (int): number of threads to use for ingestion store_events (bool): Boolean param for generating json files with tokenised events """ - sample_generator = SampleXdistGenerator(addon_path, config_path) + sample_generator = SampleXdistGenerator(addon_path, ingest_meta_data["ingest_with_uuid"], config_path) store_sample = sample_generator.get_samples(store_events) tokenized_events = store_sample.get("tokenized_events") ingestor_dict = cls.get_consolidated_events(tokenized_events) diff --git a/pytest_splunk_addon/fields_tests/test_generator.py b/pytest_splunk_addon/fields_tests/test_generator.py index 273ae192..00f9f66e 100644 --- a/pytest_splunk_addon/fields_tests/test_generator.py +++ b/pytest_splunk_addon/fields_tests/test_generator.py @@ -261,12 +261,15 @@ def generate_requirements_tests(self): for field, value in requirement_fields.items() if field not in exceptions } - yield pytest.param( - { + sample_event = { "escaped_event": escaped_event, "fields": requirement_fields, "modinput_params": modinput_params, - }, + } + if metadata["ingest_with_uuid"] == "true": + sample_event["unique_identifier"] = event.unique_identifier + yield pytest.param( + sample_event, id=f"sample_name::{event.sample_name}::host::{event.metadata.get('host')}", ) diff --git a/pytest_splunk_addon/fields_tests/test_templates.py b/pytest_splunk_addon/fields_tests/test_templates.py index fde16489..a388b40a 100644 --- a/pytest_splunk_addon/fields_tests/test_templates.py +++ b/pytest_splunk_addon/fields_tests/test_templates.py @@ -185,7 +185,15 @@ def test_requirements_fields( if param_value is not None: basic_search += f" {param}={param_value}" - search = f"search {index_list} {basic_search} {escaped_event} | fields *" + if splunk_searchtime_fields_requirements.get("unique_identifier"): + record_property( + "stanza_name", splunk_searchtime_fields_requirements["unique_identifier"] + ) + unique_identifier = splunk_searchtime_fields_requirements["unique_identifier"] + + search = f"search {index_list} {basic_search} unique_identifier=\"{unique_identifier}\" | fields *" + else: + search = f"search {index_list} {basic_search} {escaped_event} | fields *" self.logger.info(f"Executing the search query: {search}") @@ -225,6 +233,7 @@ def test_requirements_fields( assert wrong_value_fields == {}, ( f"\nNot all required fields have correct values or some fields are missing in Splunk. Wrong field values:\n{wrong_values_table}" f"{format_search_query_log(search)}" + f"Test failed for event: {escaped_event}\n" ) @pytest.mark.splunk_searchtime_fields diff --git a/pytest_splunk_addon/sample_generation/pytest_splunk_addon_data_parser.py b/pytest_splunk_addon/sample_generation/pytest_splunk_addon_data_parser.py index 381b234f..71b6a4cc 100644 --- a/pytest_splunk_addon/sample_generation/pytest_splunk_addon_data_parser.py +++ b/pytest_splunk_addon/sample_generation/pytest_splunk_addon_data_parser.py @@ -40,12 +40,13 @@ class PytestSplunkAddonDataParser: conf_name = " " - def __init__(self, addon_path: str, config_path: str): + def __init__(self, addon_path: str, config_path: str, ingest_with_uuid: str): self._conf_parser = conf_parser.TABConfigParser() self.config_path = config_path self._psa_data = None self.addon_path = addon_path self.match_stanzas = set() + self.ingest_with_uuid = ingest_with_uuid self._path_to_samples = self._get_path_to_samples() def _get_path_to_samples(self): @@ -106,7 +107,7 @@ def get_sample_stanzas(self): results = [] for sample_name, stanza_params in sorted(_psa_data.items()): sample_path = os.path.join(self._path_to_samples, sample_name) - results.append(SampleStanza(sample_path, stanza_params)) + results.append(SampleStanza(sample_path, stanza_params, self.ingest_with_uuid)) return results def _get_psa_data_stanzas(self): diff --git a/pytest_splunk_addon/sample_generation/sample_event.py b/pytest_splunk_addon/sample_generation/sample_event.py index 4eadb50e..eedcde35 100644 --- a/pytest_splunk_addon/sample_generation/sample_event.py +++ b/pytest_splunk_addon/sample_generation/sample_event.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import uuid import re import logging from ..index_tests import key_fields @@ -67,6 +68,8 @@ def __init__(self, event_string, metadata, sample_name, requirement_test_data=No self.time_values = list() self.metadata = metadata self.sample_name = sample_name + if metadata["ingest_with_uuid"] == "true": + self.unique_identifier = str(uuid.uuid4()) self.host_count = 0 self.requirement_test_data = requirement_test_data diff --git a/pytest_splunk_addon/sample_generation/sample_generator.py b/pytest_splunk_addon/sample_generation/sample_generator.py index bb511395..59b0b8b2 100644 --- a/pytest_splunk_addon/sample_generation/sample_generator.py +++ b/pytest_splunk_addon/sample_generation/sample_generator.py @@ -33,10 +33,11 @@ class SampleGenerator(object): sample_stanzas = [] conf_name = " " - def __init__(self, addon_path, config_path=None, process_count=4): + def __init__(self, addon_path, ingest_with_uuid, config_path=None, process_count=4): self.addon_path = addon_path self.process_count = process_count self.config_path = config_path + self.ingest_with_uuid = ingest_with_uuid def get_samples(self): """ @@ -44,7 +45,7 @@ def get_samples(self): """ if not SampleGenerator.sample_stanzas: psa_data_parser = PytestSplunkAddonDataParser( - self.addon_path, config_path=self.config_path + self.addon_path, config_path=self.config_path, ingest_with_uuid=self.ingest_with_uuid ) sample_stanzas = psa_data_parser.get_sample_stanzas() SampleGenerator.conf_name = psa_data_parser.conf_name diff --git a/pytest_splunk_addon/sample_generation/sample_stanza.py b/pytest_splunk_addon/sample_generation/sample_stanza.py index 0d75aab4..f1dfea8e 100644 --- a/pytest_splunk_addon/sample_generation/sample_stanza.py +++ b/pytest_splunk_addon/sample_generation/sample_stanza.py @@ -47,12 +47,13 @@ class SampleStanza(object): psa_data_params (dict): Dictionary representing pytest-splunk-addon-data.conf """ - def __init__(self, sample_path, psa_data_params): + def __init__(self, sample_path, psa_data_params, ingest_with_uuid): self.sample_path = sample_path self.sample_name = os.path.basename(sample_path) self.metadata = self._parse_meta(psa_data_params) self.sample_rules = list(self._parse_rules(psa_data_params, self.sample_path)) self.input_type = self.metadata.get("input_type", "default") + self.metadata["ingest_with_uuid"] = ingest_with_uuid self.host_count = 0 def get_raw_events(self): diff --git a/pytest_splunk_addon/sample_generation/sample_xdist_generator.py b/pytest_splunk_addon/sample_generation/sample_xdist_generator.py index 6d1dd1e7..2238296f 100644 --- a/pytest_splunk_addon/sample_generation/sample_xdist_generator.py +++ b/pytest_splunk_addon/sample_generation/sample_xdist_generator.py @@ -33,10 +33,11 @@ class SampleXdistGenerator: process_count (num): generate {no} process for execution """ - def __init__(self, addon_path, config_path=None, process_count=4): + def __init__(self, addon_path, ingest_with_uuid, config_path=None, process_count=4): self.addon_path = addon_path self.process_count = process_count self.config_path = config_path + self.ingest_with_uuid = ingest_with_uuid def get_samples(self, store_events): """ @@ -67,7 +68,7 @@ def get_samples(self, store_events): store_sample = pickle.load(file_obj) else: sample_generator = SampleGenerator( - self.addon_path, self.config_path + self.addon_path, self.ingest_with_uuid, self.config_path ) tokenized_events = list(sample_generator.get_samples()) store_sample = { @@ -79,7 +80,7 @@ def get_samples(self, store_events): with open(file_path, "wb") as file_obj: pickle.dump(store_sample, file_obj) else: - sample_generator = SampleGenerator(self.addon_path, self.config_path) + sample_generator = SampleGenerator(self.addon_path, self.ingest_with_uuid, self.config_path) tokenized_events = list(sample_generator.get_samples()) store_sample = { "conf_name": SampleGenerator.conf_name, @@ -125,6 +126,7 @@ def store_events(self, tokenized_events): "sourcetype": each_event.metadata.get("sourcetype"), "timestamp_type": each_event.metadata.get("timestamp_type"), "input_type": each_event.metadata.get("input_type"), + "ingest_with_uuid": self.ingest_with_uuid, "expected_event_count": expected_count, "index": each_event.metadata.get("index", "main"), }, @@ -137,14 +139,19 @@ def store_events(self, tokenized_events): } ], } + if self.ingest_with_uuid == "true": + tokenized_samples_dict[each_event.sample_name]["events"][0]["unique_identifier"] = each_event.unique_identifier else: - tokenized_samples_dict[each_event.sample_name]["events"].append( - { + sample_event = { "event": each_event.event, "key_fields": each_event.key_fields, "time_values": each_event.time_values, "requirement_test_data": each_event.requirement_test_data, } + if self.ingest_with_uuid == "true": + sample_event["unique_identifier"] = each_event.unique_identifier + tokenized_samples_dict[each_event.sample_name]["events"].append( + sample_event ) for sample_name, tokenized_sample in tokenized_samples_dict.items(): diff --git a/pytest_splunk_addon/splunk.py b/pytest_splunk_addon/splunk.py index 143c8bf3..435f03ee 100644 --- a/pytest_splunk_addon/splunk.py +++ b/pytest_splunk_addon/splunk.py @@ -48,6 +48,16 @@ def pytest_addoption(parser): by another process such as a ci/cd pipeline """ group = parser.getgroup("splunk-addon") + group.addoption( + "--ingest-with-uuid", + action="store", + dest="ingest_with_uuid", + default="False", + help=( + "Type of ingesting and searching the events into Splunk " + "with uuid or without uuid." + ), + ) group.addoption( "--splunk-app", @@ -747,6 +757,7 @@ def splunk_ingest_data(request, splunk_hec_uri, sc4s, uf, splunk_events_cleanup) "splunk_hec_uri": splunk_hec_uri[1], "sc4s_host": sc4s[0], # for sc4s "sc4s_port": sc4s[1][514], # for sc4s + "ingest_with_uuid": request.config.getoption("ingest_with_uuid"), } thread_count = int(request.config.getoption("thread_count")) store_events = request.config.getoption("store_events") From 461bf0332b934e3cb7da5fb7c9f473427d831ced Mon Sep 17 00:00:00 2001 From: Swati Panchal Date: Tue, 24 Jun 2025 17:22:44 +0530 Subject: [PATCH 2/5] chore: ADDON-80802 Updated the working of test_datamodels test using uuid --- .../event_ingestors/hec_event_ingestor.py | 1 - .../fields_tests/test_generator.py | 9 ++++--- .../fields_tests/test_templates.py | 25 ++++++++++++++----- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/pytest_splunk_addon/event_ingestors/hec_event_ingestor.py b/pytest_splunk_addon/event_ingestors/hec_event_ingestor.py index 754d7266..c9850b49 100644 --- a/pytest_splunk_addon/event_ingestors/hec_event_ingestor.py +++ b/pytest_splunk_addon/event_ingestors/hec_event_ingestor.py @@ -14,7 +14,6 @@ # limitations under the License. # import json - from .base_event_ingestor import EventIngestor import requests from time import time, mktime diff --git a/pytest_splunk_addon/fields_tests/test_generator.py b/pytest_splunk_addon/fields_tests/test_generator.py index 00f9f66e..455dec0c 100644 --- a/pytest_splunk_addon/fields_tests/test_generator.py +++ b/pytest_splunk_addon/fields_tests/test_generator.py @@ -190,11 +190,14 @@ def generate_requirements_datamodels_tests(self): datamodel.replace(" ", "_").replace(":", "_") for datamodel in datamodels ] - yield pytest.param( - { + sample_event = { "datamodels": datamodels, "stanza": escaped_event, - }, + } + if event.metadata["ingest_with_uuid"] == "true": + sample_event["unique_identifier"] = event.unique_identifier + yield pytest.param( + sample_event, id=f"{'-'.join(datamodels)}::sample_name::{event.sample_name}::host::{event.metadata.get('host')}", ) diff --git a/pytest_splunk_addon/fields_tests/test_templates.py b/pytest_splunk_addon/fields_tests/test_templates.py index a388b40a..37c91d84 100644 --- a/pytest_splunk_addon/fields_tests/test_templates.py +++ b/pytest_splunk_addon/fields_tests/test_templates.py @@ -401,13 +401,10 @@ def test_datamodels( record_property (fixture): Document facts of test cases to provide more info in the test failure reports. caplog (fixture): fixture to capture logs. """ - esacaped_event = splunk_searchtime_fields_datamodels["stanza"] + escaped_event = splunk_searchtime_fields_datamodels["stanza"] datamodels = splunk_searchtime_fields_datamodels["datamodels"] - self.logger.info( - f"Testing for tag {datamodels} with tag_query {esacaped_event}" - ) - record_property("Event_with", esacaped_event) + record_property("Event_with", escaped_event) record_property("datamodels", datamodels) index_list = ( @@ -415,7 +412,23 @@ def test_datamodels( + " OR index=".join(splunk_search_util.search_index.split(",")) + ")" ) - search = f"search {index_list} {esacaped_event} | fields *" + + if splunk_searchtime_fields_datamodels.get("unique_identifier"): + record_property( + "stanza_name", splunk_searchtime_fields_datamodels["unique_identifier"] + ) + unique_identifier = splunk_searchtime_fields_datamodels["unique_identifier"] + + self.logger.info( + f"Testing for tag {datamodels} with unique_identifier=\"{unique_identifier}\"" + ) + + search = f"search {index_list} unique_identifier=\"{unique_identifier}\" | fields *" + else: + self.logger.info( + f"Testing for tag {datamodels} with tag_query {escaped_event}" + ) + search = f"search {index_list} {escaped_event} | fields *" self.logger.info(f"Search: {search}") From ca90d603b058126f894cd7e097859b0330d5b845 Mon Sep 17 00:00:00 2001 From: Swati Panchal Date: Tue, 8 Jul 2025 11:53:09 +0530 Subject: [PATCH 3/5] chore: ADDON-80802 fixing linting failures --- .../event_ingestors/hec_event_ingestor.py | 3 +- .../event_ingestors/ingestor_helper.py | 4 +- .../fields_tests/test_generator.py | 20 ++++---- .../fields_tests/test_templates.py | 47 +++++++++++-------- .../pytest_splunk_addon_data_parser.py | 4 +- .../sample_generation/sample_event.py | 2 +- .../sample_generation/sample_generator.py | 4 +- .../sample_xdist_generator.py | 18 ++++--- pytest_splunk_addon/splunk.py | 5 +- 9 files changed, 62 insertions(+), 45 deletions(-) diff --git a/pytest_splunk_addon/event_ingestors/hec_event_ingestor.py b/pytest_splunk_addon/event_ingestors/hec_event_ingestor.py index c9850b49..e99c7d0a 100644 --- a/pytest_splunk_addon/event_ingestors/hec_event_ingestor.py +++ b/pytest_splunk_addon/event_ingestors/hec_event_ingestor.py @@ -85,14 +85,13 @@ def ingest(self, events, thread_count): """ data = list() for event in events: - event_dict = { "sourcetype": event.metadata.get("sourcetype", "pytest_splunk_addon"), "source": event.metadata.get("source", "pytest_splunk_addon:hec:event"), "event": event.event, "index": event.metadata.get("index", "main"), } - if event.metadata["ingest_with_uuid"] == "true": + if event.metadata.get("ingest_with_uuid") == "true": event_dict["fields"] = {"unique_identifier": event.unique_identifier} if event.metadata.get("host_type") in ("plugin", None): diff --git a/pytest_splunk_addon/event_ingestors/ingestor_helper.py b/pytest_splunk_addon/event_ingestors/ingestor_helper.py index 30dc8c50..56a7f7d0 100644 --- a/pytest_splunk_addon/event_ingestors/ingestor_helper.py +++ b/pytest_splunk_addon/event_ingestors/ingestor_helper.py @@ -95,7 +95,9 @@ def ingest_events( thread_count (int): number of threads to use for ingestion store_events (bool): Boolean param for generating json files with tokenised events """ - sample_generator = SampleXdistGenerator(addon_path, ingest_meta_data["ingest_with_uuid"], config_path) + sample_generator = SampleXdistGenerator( + addon_path, ingest_meta_data["ingest_with_uuid"], config_path + ) store_sample = sample_generator.get_samples(store_events) tokenized_events = store_sample.get("tokenized_events") ingestor_dict = cls.get_consolidated_events(tokenized_events) diff --git a/pytest_splunk_addon/fields_tests/test_generator.py b/pytest_splunk_addon/fields_tests/test_generator.py index 455dec0c..af1c533d 100644 --- a/pytest_splunk_addon/fields_tests/test_generator.py +++ b/pytest_splunk_addon/fields_tests/test_generator.py @@ -191,11 +191,11 @@ def generate_requirements_datamodels_tests(self): for datamodel in datamodels ] sample_event = { - "datamodels": datamodels, - "stanza": escaped_event, - } - if event.metadata["ingest_with_uuid"] == "true": - sample_event["unique_identifier"] = event.unique_identifier + "datamodels": datamodels, + "stanza": escaped_event, + } + if event.metadata.get("ingest_with_uuid") == "true": + sample_event["unique_identifier"] = event.unique_identifier yield pytest.param( sample_event, id=f"{'-'.join(datamodels)}::sample_name::{event.sample_name}::host::{event.metadata.get('host')}", @@ -265,11 +265,11 @@ def generate_requirements_tests(self): if field not in exceptions } sample_event = { - "escaped_event": escaped_event, - "fields": requirement_fields, - "modinput_params": modinput_params, - } - if metadata["ingest_with_uuid"] == "true": + "escaped_event": escaped_event, + "fields": requirement_fields, + "modinput_params": modinput_params, + } + if metadata.get("ingest_with_uuid") == "true": sample_event["unique_identifier"] = event.unique_identifier yield pytest.param( sample_event, diff --git a/pytest_splunk_addon/fields_tests/test_templates.py b/pytest_splunk_addon/fields_tests/test_templates.py index 37c91d84..d97199d7 100644 --- a/pytest_splunk_addon/fields_tests/test_templates.py +++ b/pytest_splunk_addon/fields_tests/test_templates.py @@ -163,7 +163,7 @@ def test_requirements_fields( # Search Query record_property( - "stanza_name", splunk_searchtime_fields_requirements["escaped_event"] + "Event_with", splunk_searchtime_fields_requirements["escaped_event"] ) record_property("fields", splunk_searchtime_fields_requirements["fields"]) record_property( @@ -187,11 +187,13 @@ def test_requirements_fields( if splunk_searchtime_fields_requirements.get("unique_identifier"): record_property( - "stanza_name", splunk_searchtime_fields_requirements["unique_identifier"] - ) - unique_identifier = splunk_searchtime_fields_requirements["unique_identifier"] - - search = f"search {index_list} {basic_search} unique_identifier=\"{unique_identifier}\" | fields *" + "Event_with", splunk_searchtime_fields_requirements["unique_identifier"] + ) + unique_identifier = splunk_searchtime_fields_requirements[ + "unique_identifier" + ] + + search = f'search {index_list} {basic_search} unique_identifier="{unique_identifier}" | fields *' else: search = f"search {index_list} {basic_search} {escaped_event} | fields *" @@ -230,12 +232,16 @@ def test_requirements_fields( if not wrong_value_fields == {}: self.logger.error("Wrong field values:\n" + wrong_values_table) - assert wrong_value_fields == {}, ( + error_message = ( f"\nNot all required fields have correct values or some fields are missing in Splunk. Wrong field values:\n{wrong_values_table}" f"{format_search_query_log(search)}" - f"Test failed for event: {escaped_event}\n" ) + if splunk_searchtime_fields_requirements.get("unique_identifier"): + error_message += f"Test failed for event: {escaped_event}\n" + + assert wrong_value_fields == {}, error_message + @pytest.mark.splunk_searchtime_fields @pytest.mark.splunk_searchtime_fields_negative def test_props_fields_no_dash_not_empty( @@ -415,19 +421,19 @@ def test_datamodels( if splunk_searchtime_fields_datamodels.get("unique_identifier"): record_property( - "stanza_name", splunk_searchtime_fields_datamodels["unique_identifier"] - ) + "Event_with", splunk_searchtime_fields_datamodels["unique_identifier"] + ) unique_identifier = splunk_searchtime_fields_datamodels["unique_identifier"] self.logger.info( - f"Testing for tag {datamodels} with unique_identifier=\"{unique_identifier}\"" - ) - - search = f"search {index_list} unique_identifier=\"{unique_identifier}\" | fields *" + f'Testing for tag {datamodels} with unique_identifier="{unique_identifier}"' + ) + + search = f'search {index_list} unique_identifier="{unique_identifier}" | fields *' else: self.logger.info( - f"Testing for tag {datamodels} with tag_query {escaped_event}" - ) + f"Testing for tag {datamodels} with tag_query {escaped_event}" + ) search = f"search {index_list} {escaped_event} | fields *" self.logger.info(f"Search: {search}") @@ -487,9 +493,12 @@ def test_datamodels( ], ) - assert ( - missing_datamodels == [] and wrong_datamodels == [] - ), f"Incorrect datamodels found:\n{exc_message}" + error_message = f"Incorrect datamodels found:\n{exc_message}" + + if splunk_searchtime_fields_datamodels.get("unique_identifier"): + error_message += f"\nTest failed for event: {escaped_event}\n" + + assert missing_datamodels == [] and wrong_datamodels == [], error_message @pytest.mark.splunk_searchtime_fields @pytest.mark.splunk_searchtime_fields_eventtypes diff --git a/pytest_splunk_addon/sample_generation/pytest_splunk_addon_data_parser.py b/pytest_splunk_addon/sample_generation/pytest_splunk_addon_data_parser.py index 71b6a4cc..a05658b1 100644 --- a/pytest_splunk_addon/sample_generation/pytest_splunk_addon_data_parser.py +++ b/pytest_splunk_addon/sample_generation/pytest_splunk_addon_data_parser.py @@ -107,7 +107,9 @@ def get_sample_stanzas(self): results = [] for sample_name, stanza_params in sorted(_psa_data.items()): sample_path = os.path.join(self._path_to_samples, sample_name) - results.append(SampleStanza(sample_path, stanza_params, self.ingest_with_uuid)) + results.append( + SampleStanza(sample_path, stanza_params, self.ingest_with_uuid) + ) return results def _get_psa_data_stanzas(self): diff --git a/pytest_splunk_addon/sample_generation/sample_event.py b/pytest_splunk_addon/sample_generation/sample_event.py index eedcde35..7b78391e 100644 --- a/pytest_splunk_addon/sample_generation/sample_event.py +++ b/pytest_splunk_addon/sample_generation/sample_event.py @@ -68,7 +68,7 @@ def __init__(self, event_string, metadata, sample_name, requirement_test_data=No self.time_values = list() self.metadata = metadata self.sample_name = sample_name - if metadata["ingest_with_uuid"] == "true": + if metadata.get("ingest_with_uuid") == "true": self.unique_identifier = str(uuid.uuid4()) self.host_count = 0 self.requirement_test_data = requirement_test_data diff --git a/pytest_splunk_addon/sample_generation/sample_generator.py b/pytest_splunk_addon/sample_generation/sample_generator.py index 59b0b8b2..45e8e87a 100644 --- a/pytest_splunk_addon/sample_generation/sample_generator.py +++ b/pytest_splunk_addon/sample_generation/sample_generator.py @@ -45,7 +45,9 @@ def get_samples(self): """ if not SampleGenerator.sample_stanzas: psa_data_parser = PytestSplunkAddonDataParser( - self.addon_path, config_path=self.config_path, ingest_with_uuid=self.ingest_with_uuid + self.addon_path, + config_path=self.config_path, + ingest_with_uuid=self.ingest_with_uuid, ) sample_stanzas = psa_data_parser.get_sample_stanzas() SampleGenerator.conf_name = psa_data_parser.conf_name diff --git a/pytest_splunk_addon/sample_generation/sample_xdist_generator.py b/pytest_splunk_addon/sample_generation/sample_xdist_generator.py index 2238296f..3b717d46 100644 --- a/pytest_splunk_addon/sample_generation/sample_xdist_generator.py +++ b/pytest_splunk_addon/sample_generation/sample_xdist_generator.py @@ -80,7 +80,9 @@ def get_samples(self, store_events): with open(file_path, "wb") as file_obj: pickle.dump(store_sample, file_obj) else: - sample_generator = SampleGenerator(self.addon_path, self.ingest_with_uuid, self.config_path) + sample_generator = SampleGenerator( + self.addon_path, self.ingest_with_uuid, self.config_path + ) tokenized_events = list(sample_generator.get_samples()) store_sample = { "conf_name": SampleGenerator.conf_name, @@ -140,14 +142,16 @@ def store_events(self, tokenized_events): ], } if self.ingest_with_uuid == "true": - tokenized_samples_dict[each_event.sample_name]["events"][0]["unique_identifier"] = each_event.unique_identifier + tokenized_samples_dict[each_event.sample_name]["events"][0][ + "unique_identifier" + ] = each_event.unique_identifier else: sample_event = { - "event": each_event.event, - "key_fields": each_event.key_fields, - "time_values": each_event.time_values, - "requirement_test_data": each_event.requirement_test_data, - } + "event": each_event.event, + "key_fields": each_event.key_fields, + "time_values": each_event.time_values, + "requirement_test_data": each_event.requirement_test_data, + } if self.ingest_with_uuid == "true": sample_event["unique_identifier"] = each_event.unique_identifier tokenized_samples_dict[each_event.sample_name]["events"].append( diff --git a/pytest_splunk_addon/splunk.py b/pytest_splunk_addon/splunk.py index 435f03ee..b12bcb8a 100644 --- a/pytest_splunk_addon/splunk.py +++ b/pytest_splunk_addon/splunk.py @@ -52,10 +52,9 @@ def pytest_addoption(parser): "--ingest-with-uuid", action="store", dest="ingest_with_uuid", - default="False", + default="false", help=( - "Type of ingesting and searching the events into Splunk " - "with uuid or without uuid." + 'Use generated UUID for ingesting and searching events. Setting this parameter to "true" will lead to matching events in search by the ID and not by escaped _raw. Default is "false".' ), ) From cebf4ed1cde794b638ac6d15b3ef1090d61f22be Mon Sep 17 00:00:00 2001 From: Swati Panchal Date: Thu, 10 Jul 2025 12:40:26 +0530 Subject: [PATCH 4/5] chore: ADDON-81952 Added/Updated unit test cases --- .../test_app_test_generator.py | 1 + .../test_event_ingestors/conftest.py | 13 ++ .../test_ingestor_helper.py | 9 +- .../test_fields_tests/test_test_generator.py | 113 ++++++++++++++++++ .../test_pytest_splunk_addon_data_parser.py | 5 +- .../test_sample_event.py | 19 ++- .../test_sample_generator.py | 6 +- .../test_sample_stanza.py | 18 ++- .../test_sample_xdist_generator.py | 91 ++++++++++++-- 9 files changed, 256 insertions(+), 19 deletions(-) diff --git a/tests/unit/tests_standard_lib/test_app_test_generator.py b/tests/unit/tests_standard_lib/test_app_test_generator.py index a22821e9..3499af0d 100644 --- a/tests/unit/tests_standard_lib/test_app_test_generator.py +++ b/tests/unit/tests_standard_lib/test_app_test_generator.py @@ -11,6 +11,7 @@ "store_events": True, "splunk_data_generator": "psa.conf", "requirement_test": "fake_requirement_path", + "ingest_with_uuid": "false", } pytest_config = namedtuple("Config", ["getoption"]) test_config = pytest_config(getoption=lambda x, *y: config[x]) diff --git a/tests/unit/tests_standard_lib/test_event_ingestors/conftest.py b/tests/unit/tests_standard_lib/test_event_ingestors/conftest.py index 492b6e45..ce77bc50 100644 --- a/tests/unit/tests_standard_lib/test_event_ingestors/conftest.py +++ b/tests/unit/tests_standard_lib/test_event_ingestors/conftest.py @@ -11,6 +11,10 @@ class SampleEvent: key_fields: dict = None time_values: list = None + def __post_init__(self): + if self.metadata.get("ingest_with_uuid") == "true": + self.unique_identifier = "uuid" + @pytest.fixture() def modinput_events(): @@ -28,6 +32,7 @@ def modinput_events(): "sample_count": "2", "host": "modinput_host_event_time_plugin.samples_1", "expected_event_count": 2, + "ingest_with_uuid": "true", }, sample_name="modinput_host_event_time_plugin.samples", ), @@ -44,6 +49,7 @@ def modinput_events(): "sample_count": "2", "host": "modinput_host_event_time_plugin.samples_2", "expected_event_count": 2, + "ingest_with_uuid": "false", }, sample_name="modinput_host_event_time_plugin.samples", ), @@ -56,6 +62,7 @@ def modinput_events(): "index": "fake_index", "timestamp_type": "event", "host": "fake host", + "ingest_with_uuid": "false", }, sample_name="fake.samples", time_values=[1234.5678, 1234.5679], @@ -112,6 +119,7 @@ def file_monitor_events(): "sample_count": "2", "host": "file_monitor_host_prefix.sample", "expected_event_count": 1, + "ingest_with_uuid": "false", }, sample_name="file_monitor_host_prefix.sample", ), @@ -130,6 +138,7 @@ def file_monitor_events(): "host": "failing-samples-1", "id": "failing.samples_1", "expected_event_count": 2, + "ingest_with_uuid": "false", }, sample_name="failing.samples", ), @@ -138,6 +147,7 @@ def file_monitor_events(): metadata={ "input_type": "file_monitor", "index": "fake_index", + "ingest_with_uuid": "false", }, sample_name="fake.samples", ), @@ -234,6 +244,7 @@ def requirement_events(): "sample_count": "2", "host": "requirement_host_prefix.sample", "expected_event_count": 1, + "ingest_with_uuid": "false", }, sample_name="requirement_test", ), @@ -256,6 +267,7 @@ def sc4s_events(): "host": "sc4s-host-plugin-time-sample-31", "id": "sc4s_host_plugin_time.sample_31", "expected_event_count": 2, + "ingest_with_uuid": "false", }, sample_name="sc4s_host_plugin_time.sample", ), @@ -272,6 +284,7 @@ def sc4s_events(): "host": "sc4s-host-plugin-time-sample-32", "id": "sc4s-host-plugin-time-sample-32", "expected_event_count": 2, + "ingest_with_uuid": "false", }, sample_name="sc4s_host_plugin_time.sample", ), diff --git a/tests/unit/tests_standard_lib/test_event_ingestors/test_ingestor_helper.py b/tests/unit/tests_standard_lib/test_event_ingestors/test_ingestor_helper.py index 3d9a37dc..1dcdbd2e 100644 --- a/tests/unit/tests_standard_lib/test_event_ingestors/test_ingestor_helper.py +++ b/tests/unit/tests_standard_lib/test_event_ingestors/test_ingestor_helper.py @@ -22,7 +22,6 @@ def ingestors_mocks(): ) as hec_metric_event_mock, patch( f"{EVENT_INGESTOR_PATH}.sc4s_event_ingestor.SC4SEventIngestor" ) as sc4s_event_mock: - hec_event_mock.return_value = HEC_EVENT_INGESTOR_RETURN_VALUE hec_raw_event_mock.return_value = HEC_RAW_EVENT_INGESTOR_RETURN_VALUE hec_metric_event_mock.return_value = HEC_METRIC_EVENT_INGESTOR_RETURN_VALUE @@ -125,7 +124,7 @@ def test_events_can_be_ingested( get_ingestor_mock, sample_mock, file_monitor_events, modinput_events ): event_ingestors.ingestor_helper.IngestorHelper.ingest_events( - ingest_meta_data={}, + ingest_meta_data={"ingest_with_uuid": "false"}, addon_path="fake_path", config_path="tests/unit/event_ingestors", thread_count=20, @@ -133,7 +132,11 @@ def test_events_can_be_ingested( ) assert get_ingestor_mock.call_count == 2 get_ingestor_mock.assert_has_calls( - [call("file_monitor", {}), call("modinput", {})], any_order=True + [ + call("file_monitor", {"ingest_with_uuid": "false"}), + call("modinput", {"ingest_with_uuid": "false"}), + ], + any_order=True, ) assert get_ingestor_mock.ingest.call_count == 2 get_ingestor_mock.ingest.assert_has_calls( diff --git a/tests/unit/tests_standard_lib/test_fields_tests/test_test_generator.py b/tests/unit/tests_standard_lib/test_fields_tests/test_test_generator.py index e57daefa..0544fa42 100644 --- a/tests/unit/tests_standard_lib/test_fields_tests/test_test_generator.py +++ b/tests/unit/tests_standard_lib/test_fields_tests/test_test_generator.py @@ -24,6 +24,15 @@ def field_3(): field_3.__dict__.update({"name": "field_3"}) +@pytest.fixture +def mock_uuid4(): + with patch( + "pytest_splunk_addon.sample_generation.sample_event.uuid.uuid4", + return_value="uuid", + ) as mock_uuid: + yield mock_uuid + + @pytest.fixture() def addon_parser_mock(monkeypatch): ap = MagicMock() @@ -422,6 +431,7 @@ def test_generate_field_tests( "input_type": "modinput", "sourcetype_to_search": "dummy_sourcetype", "host": "dummy_host", + "ingest_with_uuid": "false", }, sample_name="file1.xml", requirement_test_data={ @@ -445,6 +455,7 @@ def test_generate_field_tests( "input_type": "syslog_tcp", "sourcetype_to_search": "dummy_sourcetype", "host": "dummy_host_syslog", + "ingest_with_uuid": "false", }, sample_name="file1.xml", requirement_test_data={}, @@ -455,6 +466,7 @@ def test_generate_field_tests( "input_type": "syslog_tcp", "sourcetype_to_search": "dummy_sourcetype", "host": "dummy_host_syslog", + "ingest_with_uuid": "false", }, sample_name="file1.xml", requirement_test_data={ @@ -522,6 +534,63 @@ def test_generate_requirement_tests(tokenised_events, expected_output): assert param_mock.call_count == len(expected_output) +def test_generate_requirement_tests_with_uuid(mock_uuid4): + tokenised_events = [ + SampleEvent( + event_string="escaped_event", + metadata={ + "input_type": "modinput", + "sourcetype_to_search": "dummy_sourcetype", + "host": "dummy_host", + "ingest_with_uuid": "true", + }, + sample_name="file1.xml", + requirement_test_data={ + "cim_fields": { + "severity": "low", + "signature_id": "405001", + "src": "192.168.0.1", + "type": "event", + }, + }, + ) + ] + + expected_output = [ + ( + { + "escaped_event": "escaped_event", + "unique_identifier": "uuid", + "fields": { + "severity": "low", + "signature_id": "405001", + "src": "192.168.0.1", + "type": "event", + }, + "modinput_params": {"sourcetype": "dummy_sourcetype"}, + }, + "sample_name::file1.xml::host::dummy_host", + ) + ] + + with patch.object( + xml_event_parser, "strip_syslog_header", return_value="escaped_event" + ), patch.object( + xml_event_parser, "escape_char_event", return_value="escaped_event" + ), patch.object( + pytest, "param", side_effect=lambda x, id: (x, id) + ) as param_mock: + out = list( + FieldTestGenerator( + "app_path", + tokenised_events, + "field_bank", + ).generate_requirements_tests() + ) + assert out == expected_output + assert param_mock.call_count == len(expected_output) + + @pytest.mark.parametrize( "tokenised_events, expected_output", [ @@ -596,3 +665,47 @@ def test_generate_requirement_datamodel_tests(tokenised_events, expected_output) ) assert out == expected_output assert param_mock.call_count == len(expected_output) + + +def test_generate_requirement_datamodel_tests_with_uuid(mock_uuid4): + tokenised_events = [ + SampleEvent( + event_string="escaped_event", + metadata={ + "input_type": "modinput", + "sourcetype_to_search": "dummy_sourcetype", + "host": "dummy_host", + "ingest_with_uuid": "true", + }, + sample_name="file1.xml", + requirement_test_data={"datamodels": {"model": "Alerts"}}, + ) + ] + + expected_output = [ + ( + { + "datamodels": ["Alerts"], + "stanza": "escaped_event", + "unique_identifier": "uuid", + }, + "Alerts::sample_name::file1.xml::host::dummy_host", + ) + ] + + with patch.object( + xml_event_parser, "strip_syslog_header", return_value="escaped_event" + ), patch.object( + xml_event_parser, "escape_char_event", return_value="escaped_event" + ), patch.object( + pytest, "param", side_effect=lambda x, id: (x, id) + ) as param_mock: + out = list( + FieldTestGenerator( + "app_path", + tokenised_events, + "field_bank", + ).generate_requirements_datamodels_tests() + ) + assert out == expected_output + assert param_mock.call_count == len(expected_output) diff --git a/tests/unit/tests_standard_lib/tests_sample_generation/test_pytest_splunk_addon_data_parser.py b/tests/unit/tests_standard_lib/tests_sample_generation/test_pytest_splunk_addon_data_parser.py index ce9130c7..23f5e7f1 100644 --- a/tests/unit/tests_standard_lib/tests_sample_generation/test_pytest_splunk_addon_data_parser.py +++ b/tests/unit/tests_standard_lib/tests_sample_generation/test_pytest_splunk_addon_data_parser.py @@ -11,7 +11,7 @@ def test_psa_data_when_no_config(): with tempfile.TemporaryDirectory() as tempdir: with pytest.raises(FileNotFoundError): - psa_data_parser = PytestSplunkAddonDataParser(tempdir, tempdir) + psa_data_parser = PytestSplunkAddonDataParser(tempdir, tempdir, "false") _ = psa_data_parser.psa_data @@ -20,6 +20,7 @@ def test_path_to_samples(): psa_data_parser = PytestSplunkAddonDataParser( path, path, + "false", ) assert os.path.join(path, "samples") == psa_data_parser._path_to_samples @@ -29,6 +30,7 @@ def test_get_psa_data_stanzas_with_samples(): psa_data_parser = PytestSplunkAddonDataParser( path, path, + "false", ) expected_result = { "test1.samples": { @@ -101,6 +103,7 @@ def test_get_sample_stanzas_without_samples(caplog): parser = PytestSplunkAddonDataParser( tempdir, config_path, + "false", ) parser.get_sample_stanzas() assert "No sample file found for stanza : test1.samples" in caplog.messages diff --git a/tests/unit/tests_standard_lib/tests_sample_generation/test_sample_event.py b/tests/unit/tests_standard_lib/tests_sample_generation/test_sample_event.py index c2fa5156..e64faa37 100644 --- a/tests/unit/tests_standard_lib/tests_sample_generation/test_sample_event.py +++ b/tests/unit/tests_standard_lib/tests_sample_generation/test_sample_event.py @@ -8,7 +8,7 @@ EVENT_STRING = "Event_string dad ad dfd ddas Value_5." UPDATED_STRING = "Updated_string" SAMPLE_NAME = "Sample_name" -METADATA = {"Metadata": "metadata"} +METADATA = {"Metadata": "metadata", "ingest_with_uuid": "false"} RULE = "Rule" SAMPLE_HOST = "sample_host" FAKE_IPV4 = "222.222.222.222" @@ -33,6 +33,23 @@ def samp_eve(): ) +def test_sample_event_generates_uuid(): + METADATA["ingest_with_uuid"] = "true" + with patch( + "pytest_splunk_addon.sample_generation.sample_event.uuid.uuid4", + return_value="uuid", + ) as mock_uuid: + event = pytest_splunk_addon.sample_generation.sample_event.SampleEvent( + event_string=EVENT_STRING, + metadata=METADATA, + sample_name=SAMPLE_NAME, + ) + + mock_uuid.assert_called_once() # Ensures uuid4 was called + assert hasattr(event, "unique_identifier") # The field was set + assert event.unique_identifier == "uuid" + + def check_host_count(value): assert pytest_splunk_addon.sample_generation.sample_event.host_count == value diff --git a/tests/unit/tests_standard_lib/tests_sample_generation/test_sample_generator.py b/tests/unit/tests_standard_lib/tests_sample_generation/test_sample_generator.py index 87c87b5b..72100e55 100644 --- a/tests/unit/tests_standard_lib/tests_sample_generation/test_sample_generator.py +++ b/tests/unit/tests_standard_lib/tests_sample_generation/test_sample_generator.py @@ -11,11 +11,11 @@ class TestSampleGenerator: def test_init(self): - sg = SampleGenerator(ADDON_PATH, CONFIG_PATH) + sg = SampleGenerator(ADDON_PATH, "false", CONFIG_PATH) assert sg.addon_path == ADDON_PATH assert sg.config_path == CONFIG_PATH assert sg.process_count == 4 - sg = SampleGenerator(ADDON_PATH, CONFIG_PATH, 2) + sg = SampleGenerator(ADDON_PATH, "false", CONFIG_PATH, 2) assert sg.addon_path == ADDON_PATH assert sg.config_path == CONFIG_PATH assert sg.process_count == 2 @@ -36,7 +36,7 @@ def test_get_samples(self): sample_stanza_mock.get_raw_events = ["event_1", "event_2"] sample_stanza_mock.tokenize = lambda x, y: (x, y) psa_data_mock.conf_name = CONFIG_PATH - sg = SampleGenerator(ADDON_PATH) + sg = SampleGenerator(ADDON_PATH, "false") assert list(sg.get_samples()) == [tks_1, tks_2, tks_1, tks_2] def test_clean_samples(self): diff --git a/tests/unit/tests_standard_lib/tests_sample_generation/test_sample_stanza.py b/tests/unit/tests_standard_lib/tests_sample_generation/test_sample_stanza.py index 92da36e3..d1befda2 100644 --- a/tests/unit/tests_standard_lib/tests_sample_generation/test_sample_stanza.py +++ b/tests/unit/tests_standard_lib/tests_sample_generation/test_sample_stanza.py @@ -27,7 +27,11 @@ def get_params_for_get_raw_sample(): }, [ "sample_raw", - {"input_type": input_type, "host": "path_to.file_1"}, + { + "input_type": input_type, + "host": "path_to.file_1", + "ingest_with_uuid": "false", + }, "path_to.file", ], ) @@ -47,7 +51,11 @@ def get_params_for_get_raw_sample(): }, [ "sample_raw", - {"input_type": input_type, "host": "path_to.file"}, + { + "input_type": input_type, + "host": "path_to.file", + "ingest_with_uuid": "false", + }, "path_to.file", ], ) @@ -66,7 +74,7 @@ def func( "pytest_splunk_addon.sample_generation.sample_stanza.Rule", MagicMock(return_value=rule_mock_value), ): - ss = SampleStanza(SAMPLE_PATH, psa_data_params) + ss = SampleStanza(SAMPLE_PATH, psa_data_params, "false") return ss return func @@ -186,6 +194,7 @@ def test_parse_meta( "expected_event_count": "hh", "count": "ll", "index": 1, + "ingest_with_uuid": "false", "input_type": input_type, "tokens": { "token_1": {"replacementType": "all"}, @@ -199,6 +208,7 @@ def test_parse_meta( "host": host, "host_type": "plugin", "index": 1, + "ingest_with_uuid": "false", "input_type": input_type_expected, "sample_count": "1", "timestamp_type": "plugin", @@ -213,6 +223,7 @@ def test_get_eventmetadata(self, sample_stanza): assert ss.get_eventmetadata() == { "host": "path_to.file_1", "input_type": "default", + "ingest_with_uuid": "false", } assert ss.host_count == 1 @@ -246,6 +257,7 @@ def test_break_events(self, sample_stanza, sample_raw, expected): "breaker": "aa", "host": "path_to.file_1", "input_type": "default", + "ingest_with_uuid": "false", }, "path_to.file", ], diff --git a/tests/unit/tests_standard_lib/tests_sample_generation/test_sample_xdist_generator.py b/tests/unit/tests_standard_lib/tests_sample_generation/test_sample_xdist_generator.py index 0b7f5236..5b3a4d5a 100644 --- a/tests/unit/tests_standard_lib/tests_sample_generation/test_sample_xdist_generator.py +++ b/tests/unit/tests_standard_lib/tests_sample_generation/test_sample_xdist_generator.py @@ -71,7 +71,7 @@ class TestSampleXdistGenerator: def test_init(self): - sample_xdist_generator = SampleXdistGenerator("path", "config_path", 5) + sample_xdist_generator = SampleXdistGenerator("path", "false", "config_path", 5) assert sample_xdist_generator.addon_path == "path" assert sample_xdist_generator.config_path == "config_path" assert sample_xdist_generator.process_count == 5 @@ -104,7 +104,7 @@ def test_init(self): ) def test_get_samples(self, pickle_mock, exists_value, environ, expected): pickle_mock.load.return_value = "pickle_loaded" - sample_xdist_generator = SampleXdistGenerator("path") + sample_xdist_generator = SampleXdistGenerator("path", "false") sample_xdist_generator.store_events = MagicMock() with patch("os.path.exists", MagicMock(return_value=exists_value)), patch( "os.environ", @@ -118,16 +118,20 @@ def test_get_samples(self, pickle_mock, exists_value, environ, expected): assert sample_xdist_generator.get_samples(True) == expected @pytest.mark.parametrize( - "exists_value, makedirs_calls", - [(True, []), (False, [call("/path/to/cwd/.tokenized_events")])], + "exists_value, makedirs_calls, ingest_with_uuid", + [ + (True, [], "false"), + (False, [call("/path/to/cwd/.tokenized_events")], "false"), + (False, [call("/path/to/cwd/.tokenized_events")], "false"), + ], ) - def test_store_events(self, exists_value, makedirs_calls): + def test_store_events(self, exists_value, makedirs_calls, ingest_with_uuid): with patch("os.path.exists", MagicMock(return_value=exists_value)), patch( "os.getcwd", MagicMock(return_value="/path/to/cwd") ), patch("os.makedirs", MagicMock()) as mock_makedirs, patch( "builtins.open", mock_open() ) as open_mock: - sample_xdist_generator = SampleXdistGenerator("path") + sample_xdist_generator = SampleXdistGenerator("path", ingest_with_uuid) sample_xdist_generator.store_events(tokenized_events) mock_makedirs.assert_has_calls(makedirs_calls) open_mock.assert_has_calls( @@ -140,10 +144,81 @@ def test_store_events(self, exists_value, makedirs_calls): open_mock().write.assert_has_calls( [ call( - '{\n\t"sample_name_1": {\n\t\t"metadata": {\n\t\t\t"host": "host_1",\n\t\t\t"source": "source_1",\n\t\t\t"sourcetype": "sourcetype_1",\n\t\t\t"timestamp_type": "timestamp_type_1",\n\t\t\t"input_type": "modinput",\n\t\t\t"expected_event_count": 1,\n\t\t\t"index": "main"\n\t\t},\n\t\t"events": [\n\t\t\t{\n\t\t\t\t"event": "event_field",\n\t\t\t\t"key_fields": "key_fields_field",\n\t\t\t\t"time_values": "time_values_field",\n\t\t\t\t"requirement_test_data": "requirement_test_data"\n\t\t\t},\n\t\t\t{\n\t\t\t\t"event": "event_field",\n\t\t\t\t"key_fields": "key_fields_field_3",\n\t\t\t\t"time_values": "time_values_field_3",\n\t\t\t\t"requirement_test_data": "requirement_test_data"\n\t\t\t}\n\t\t]\n\t}\n}' + '{\n\t"sample_name_1": {\n\t\t"metadata": {\n\t\t\t"host": "host_1",\n\t\t\t"source": "source_1",\n\t\t\t"sourcetype": "sourcetype_1",\n\t\t\t"timestamp_type": "timestamp_type_1",\n\t\t\t"input_type": "modinput",\n\t\t\t"ingest_with_uuid": "false",\n\t\t\t"expected_event_count": 1,\n\t\t\t"index": "main"\n\t\t},\n\t\t"events": [\n\t\t\t{\n\t\t\t\t"event": "event_field",\n\t\t\t\t"key_fields": "key_fields_field",\n\t\t\t\t"time_values": "time_values_field",\n\t\t\t\t"requirement_test_data": "requirement_test_data"\n\t\t\t},\n\t\t\t{\n\t\t\t\t"event": "event_field",\n\t\t\t\t"key_fields": "key_fields_field_3",\n\t\t\t\t"time_values": "time_values_field_3",\n\t\t\t\t"requirement_test_data": "requirement_test_data"\n\t\t\t}\n\t\t]\n\t}\n}' + ), + call( + '{\n\t"sample_name_2": {\n\t\t"metadata": {\n\t\t\t"host": "host_2",\n\t\t\t"source": "source_2",\n\t\t\t"sourcetype": "sourcetype_2",\n\t\t\t"timestamp_type": "timestamp_type_2",\n\t\t\t"input_type": "input_else",\n\t\t\t"ingest_with_uuid": "false",\n\t\t\t"expected_event_count": 8,\n\t\t\t"index": "main"\n\t\t},\n\t\t"events": [\n\t\t\t{\n\t\t\t\t"event": "event_field",\n\t\t\t\t"key_fields": "key_fields_field",\n\t\t\t\t"time_values": "time_values_field",\n\t\t\t\t"requirement_test_data": "requirement_test_data"\n\t\t\t}\n\t\t]\n\t}\n}' ), + ] + ) + + def test_store_events_with_uuid(self): + tokenized_event = namedtuple( + "tokenized_event", + [ + "sample_name", + "metadata", + "event", + "unique_identifier", + "key_fields", + "time_values", + "requirement_test_data", + ], + ) + + tokenized_events = [ + tokenized_event( + "sample_with_uuid", + { + "expected_event_count": 1, + "host": "host_1", + "source": "source_1", + "sourcetype": "sourcetype_1", + "timestamp_type": "timestamp_type_1", + "input_type": "modinput", + }, + "event_field", + "uuid", + "key_fields_field", + "time_values_field", + "requirement_test_data", + ), + tokenized_event( + "sample_with_uuid", + { + "expected_event_count": 2, + "host": "host_2", + "source": "source_2", + "sourcetype": "sourcetype_2", + "timestamp_type": "timestamp_type_2", + "input_type": "input_else", + "sample_count": 4, + }, + "event_field", + "uuid", + "key_fields_field_3", + "time_values_field_3", + "requirement_test_data", + ), + ] + with patch("os.path.exists", MagicMock(return_value=False)), patch( + "os.getcwd", MagicMock(return_value="/path/to/cwd") + ), patch("os.makedirs", MagicMock()) as mock_makedirs, patch( + "builtins.open", mock_open() + ) as open_mock: + sample_xdist_generator = SampleXdistGenerator("path", "true") + sample_xdist_generator.store_events(tokenized_events) + mock_makedirs.assert_has_calls([call("/path/to/cwd/.tokenized_events")]) + open_mock.assert_has_calls( + [ + call("/path/to/cwd/.tokenized_events/sample_with_uuid.json", "w"), + ], + any_order=True, + ) + open_mock().write.assert_has_calls( + [ call( - '{\n\t"sample_name_2": {\n\t\t"metadata": {\n\t\t\t"host": "host_2",\n\t\t\t"source": "source_2",\n\t\t\t"sourcetype": "sourcetype_2",\n\t\t\t"timestamp_type": "timestamp_type_2",\n\t\t\t"input_type": "input_else",\n\t\t\t"expected_event_count": 8,\n\t\t\t"index": "main"\n\t\t},\n\t\t"events": [\n\t\t\t{\n\t\t\t\t"event": "event_field",\n\t\t\t\t"key_fields": "key_fields_field",\n\t\t\t\t"time_values": "time_values_field",\n\t\t\t\t"requirement_test_data": "requirement_test_data"\n\t\t\t}\n\t\t]\n\t}\n}' + '{\n\t"sample_with_uuid": {\n\t\t"metadata": {\n\t\t\t"host": "host_1",\n\t\t\t"source": "source_1",\n\t\t\t"sourcetype": "sourcetype_1",\n\t\t\t"timestamp_type": "timestamp_type_1",\n\t\t\t"input_type": "modinput",\n\t\t\t"ingest_with_uuid": "true",\n\t\t\t"expected_event_count": 1,\n\t\t\t"index": "main"\n\t\t},\n\t\t"events": [\n\t\t\t{\n\t\t\t\t"event": "event_field",\n\t\t\t\t"key_fields": "key_fields_field",\n\t\t\t\t"time_values": "time_values_field",\n\t\t\t\t"requirement_test_data": "requirement_test_data",\n\t\t\t\t"unique_identifier": "uuid"\n\t\t\t},\n\t\t\t{\n\t\t\t\t"event": "event_field",\n\t\t\t\t"key_fields": "key_fields_field_3",\n\t\t\t\t"time_values": "time_values_field_3",\n\t\t\t\t"requirement_test_data": "requirement_test_data",\n\t\t\t\t"unique_identifier": "uuid"\n\t\t\t}\n\t\t]\n\t}\n}' ), ] ) From bc1e289388bb43ec6f6a60291c9728bc7f8f55bc Mon Sep 17 00:00:00 2001 From: Swati Panchal Date: Fri, 1 Aug 2025 12:30:44 +0530 Subject: [PATCH 5/5] chore: ADDON-80802 Resolve minor review comments --- pytest_splunk_addon/fields_tests/test_templates.py | 8 ++------ .../test_event_ingestors/conftest.py | 1 + .../test_fields_tests/test_test_generator.py | 12 ++---------- 3 files changed, 5 insertions(+), 16 deletions(-) diff --git a/pytest_splunk_addon/fields_tests/test_templates.py b/pytest_splunk_addon/fields_tests/test_templates.py index d97199d7..170f24ea 100644 --- a/pytest_splunk_addon/fields_tests/test_templates.py +++ b/pytest_splunk_addon/fields_tests/test_templates.py @@ -186,12 +186,10 @@ def test_requirements_fields( basic_search += f" {param}={param_value}" if splunk_searchtime_fields_requirements.get("unique_identifier"): - record_property( - "Event_with", splunk_searchtime_fields_requirements["unique_identifier"] - ) unique_identifier = splunk_searchtime_fields_requirements[ "unique_identifier" ] + record_property("Event_with", unique_identifier) search = f'search {index_list} {basic_search} unique_identifier="{unique_identifier}" | fields *' else: @@ -420,10 +418,8 @@ def test_datamodels( ) if splunk_searchtime_fields_datamodels.get("unique_identifier"): - record_property( - "Event_with", splunk_searchtime_fields_datamodels["unique_identifier"] - ) unique_identifier = splunk_searchtime_fields_datamodels["unique_identifier"] + record_property("Event_with", unique_identifier) self.logger.info( f'Testing for tag {datamodels} with unique_identifier="{unique_identifier}"' diff --git a/tests/unit/tests_standard_lib/test_event_ingestors/conftest.py b/tests/unit/tests_standard_lib/test_event_ingestors/conftest.py index ce77bc50..1462ae7b 100644 --- a/tests/unit/tests_standard_lib/test_event_ingestors/conftest.py +++ b/tests/unit/tests_standard_lib/test_event_ingestors/conftest.py @@ -80,6 +80,7 @@ def modinput_posts_sent(): '"source": "pytest-splunk-addon:modinput", ' '"event": "test_modinput_1 host=modinput_host_event_time_plugin.samples_1", ' '"index": "main", ' + '"fields": {"unique_identifier": "uuid"}, ' '"host": "modinput_host_event_time_plugin.samples_1"' "}\n{" '"sourcetype": "test:indextime:sourcetype:modinput_host_event_time_plugin", ' diff --git a/tests/unit/tests_standard_lib/test_fields_tests/test_test_generator.py b/tests/unit/tests_standard_lib/test_fields_tests/test_test_generator.py index 0544fa42..36b54983 100644 --- a/tests/unit/tests_standard_lib/test_fields_tests/test_test_generator.py +++ b/tests/unit/tests_standard_lib/test_fields_tests/test_test_generator.py @@ -574,12 +574,8 @@ def test_generate_requirement_tests_with_uuid(mock_uuid4): ] with patch.object( - xml_event_parser, "strip_syslog_header", return_value="escaped_event" - ), patch.object( xml_event_parser, "escape_char_event", return_value="escaped_event" - ), patch.object( - pytest, "param", side_effect=lambda x, id: (x, id) - ) as param_mock: + ), patch.object(pytest, "param", side_effect=lambda x, id: (x, id)) as param_mock: out = list( FieldTestGenerator( "app_path", @@ -694,12 +690,8 @@ def test_generate_requirement_datamodel_tests_with_uuid(mock_uuid4): ] with patch.object( - xml_event_parser, "strip_syslog_header", return_value="escaped_event" - ), patch.object( xml_event_parser, "escape_char_event", return_value="escaped_event" - ), patch.object( - pytest, "param", side_effect=lambda x, id: (x, id) - ) as param_mock: + ), patch.object(pytest, "param", side_effect=lambda x, id: (x, id)) as param_mock: out = list( FieldTestGenerator( "app_path",