From 09c4d9057255f500bc06f57cdf164d247ab5be86 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Wed, 12 Nov 2025 14:37:49 -0500 Subject: [PATCH 1/6] Add synthetic properties check --- detection_rules/index_mappings.py | 14 +++++++++++--- pyproject.toml | 2 +- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/detection_rules/index_mappings.py b/detection_rules/index_mappings.py index bfcbd13316f..3b5916b28b9 100644 --- a/detection_rules/index_mappings.py +++ b/detection_rules/index_mappings.py @@ -173,12 +173,16 @@ def prune_mappings_of_unsupported_types( delete_nested_key_from_dict(stream_mappings, field_name) nested_flattened_fields = find_flattened_fields_with_subfields(stream_mappings) for field in nested_flattened_fields: + # Remove both .fields and .properties entries for flattened fields + # properties entries can occur when being merged with non-ecs or custom schemas field_name = str(field).split(".fields.")[0].replace(".", ".properties.") + ".fields" + property_name = str(field).split(".fields.")[0].replace(".", ".properties.") + ".properties" log( f"Warning: flattened field `{field}` found in `{integration}-{stream}` with sub fields. " f"Removing parent field from schema for ES|QL validation." ) delete_nested_key_from_dict(stream_mappings, field_name) + delete_nested_key_from_dict(stream_mappings, property_name) return stream_mappings @@ -246,12 +250,13 @@ def get_index_to_package_lookup(indices: list[str], index_lookup: dict[str, Any] return index_lookup_indices -def get_filtered_index_schema( +def get_filtered_index_schema( # noqa: PLR0913 indices: list[str], index_lookup: dict[str, Any], ecs_schema: dict[str, Any], non_ecs_mapping: dict[str, Any], custom_mapping: dict[str, Any], + log: Callable[[str], None], ) -> tuple[dict[str, Any], dict[str, Any]]: """Check if the provided indices are known based on the integration format. Returns the combined schema.""" @@ -304,7 +309,7 @@ def get_filtered_index_schema( # Need to use a merge here to not overwrite existing fields utils.combine_dicts(base, deepcopy(non_ecs_mapping.get(match, {}))) utils.combine_dicts(base, deepcopy(custom_mapping.get(match, {}))) - filtered_index_lookup[match] = base + filtered_index_lookup[match] = prune_mappings_of_unsupported_types("index", match, base, log) utils.combine_dicts(combined_mappings, deepcopy(base)) # Reduce the index lookup to only the matched indices (remote/Kibana schema validation source of truth) @@ -413,6 +418,9 @@ def find_flattened_fields_with_subfields(mapping: dict[str, Any], path: str = "" # Check if the field is of type 'flattened' and has a 'fields' key if properties.get("type") == "flattened" and "fields" in properties: # type: ignore[reportUnknownVariableType] flattened_fields_with_subfields.append(current_path) # type: ignore[reportUnknownVariableType] + # Check if the field is of type 'flattened' and has a 'properties' key + if properties.get("type") == "flattened" and "properties" in properties: # type: ignore[reportUnknownVariableType] + flattened_fields_with_subfields.append(current_path) # type: ignore[reportUnknownVariableType] # Recurse into subfields if "properties" in properties: @@ -506,7 +514,7 @@ def prepare_mappings( # noqa: PLR0913 # Filter combined mappings based on the provided indices combined_mappings, index_lookup = get_filtered_index_schema( - indices, index_lookup, ecs_schema, non_ecs_mapping, custom_mapping + indices, index_lookup, ecs_schema, non_ecs_mapping, custom_mapping, log ) index_lookup.update({"rule-ecs-index": ecs_schema}) diff --git a/pyproject.toml b/pyproject.toml index 53cecd544e6..d8be95f048c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "detection_rules" -version = "1.5.8" +version = "1.5.9" description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine." readme = "README.md" requires-python = ">=3.12" From 11bdf92834986b8d296568a47ea03ba8a8285fc4 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Wed, 12 Nov 2025 14:48:18 -0500 Subject: [PATCH 2/6] Adjust comment --- detection_rules/index_mappings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detection_rules/index_mappings.py b/detection_rules/index_mappings.py index 3b5916b28b9..ef6e574a7af 100644 --- a/detection_rules/index_mappings.py +++ b/detection_rules/index_mappings.py @@ -174,7 +174,7 @@ def prune_mappings_of_unsupported_types( nested_flattened_fields = find_flattened_fields_with_subfields(stream_mappings) for field in nested_flattened_fields: # Remove both .fields and .properties entries for flattened fields - # properties entries can occur when being merged with non-ecs or custom schemas + # .properties entries can occur when being merged with non-ecs or custom schemas field_name = str(field).split(".fields.")[0].replace(".", ".properties.") + ".fields" property_name = str(field).split(".fields.")[0].replace(".", ".properties.") + ".properties" log( From 799836aa82300ad323aeaafc280053fe96eda3cc Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Wed, 12 Nov 2025 16:34:38 -0500 Subject: [PATCH 3/6] Update comment --- detection_rules/index_mappings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detection_rules/index_mappings.py b/detection_rules/index_mappings.py index ef6e574a7af..de095c6ea65 100644 --- a/detection_rules/index_mappings.py +++ b/detection_rules/index_mappings.py @@ -408,7 +408,7 @@ def find_nested_multifields(mapping: dict[str, Any], path: str = "") -> list[Any def find_flattened_fields_with_subfields(mapping: dict[str, Any], path: str = "") -> list[str]: - """Recursively search for fields of type 'flattened' that have a 'fields' key in Elasticsearch mappings.""" + """Recursively search for type 'flattened' that have a 'fields' or 'properties' key in Elasticsearch mappings.""" flattened_fields_with_subfields: list[str] = [] for field, properties in mapping.items(): From 5e35f883c1174ef285532edf365b144720a8efd9 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Wed, 12 Nov 2025 18:56:46 -0500 Subject: [PATCH 4/6] Use list comprehension --- detection_rules/index_mappings.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/detection_rules/index_mappings.py b/detection_rules/index_mappings.py index de095c6ea65..d6bae16db51 100644 --- a/detection_rules/index_mappings.py +++ b/detection_rules/index_mappings.py @@ -160,14 +160,17 @@ def get_simulated_index_template_mappings(elastic_client: Elasticsearch, name: s def prune_mappings_of_unsupported_types( - integration: str, stream: str, stream_mappings: dict[str, Any], log: Callable[[str], None] + integration: str | None, index: str | None, stream_mappings: dict[str, Any], log: Callable[[str], None] ) -> dict[str, Any]: """Prune fields with unsupported types (ES|QL) from the provided mappings.""" + debug_str = integration if integration is not None else index nested_multifields = find_nested_multifields(stream_mappings) for field in nested_multifields: - field_name = str(field).split(".fields.")[0].replace(".", ".properties.") + ".fields" + parts = str(field).split(".fields.")[0].split(".") + base_name = ".properties.".join(parts) + field_name = f"{base_name}.fields" log( - f"Warning: Nested multi-field `{field}` found in `{integration}-{stream}`. " + f"Warning: Nested multi-field `{field}` found in `{debug_str}`. " f"Removing parent field from schema for ES|QL validation." ) delete_nested_key_from_dict(stream_mappings, field_name) @@ -175,10 +178,12 @@ def prune_mappings_of_unsupported_types( for field in nested_flattened_fields: # Remove both .fields and .properties entries for flattened fields # .properties entries can occur when being merged with non-ecs or custom schemas - field_name = str(field).split(".fields.")[0].replace(".", ".properties.") + ".fields" - property_name = str(field).split(".fields.")[0].replace(".", ".properties.") + ".properties" + parts = str(field).split(".fields.")[0].split(".") + base_name = ".properties.".join(parts) + field_name = f"{base_name}.fields" + property_name = f"{base_name}.properties" log( - f"Warning: flattened field `{field}` found in `{integration}-{stream}` with sub fields. " + f"Warning: flattened field `{field}` found in `{debug_str}` with sub fields. " f"Removing parent field from schema for ES|QL validation." ) delete_nested_key_from_dict(stream_mappings, field_name) @@ -226,7 +231,7 @@ def prepare_integration_mappings( # noqa: PLR0913 for stream in package_schema: flat_schema = package_schema[stream] stream_mappings = flat_schema_to_index_mapping(flat_schema) - stream_mappings = prune_mappings_of_unsupported_types(integration, stream, stream_mappings, log) + stream_mappings = prune_mappings_of_unsupported_types(f"{integration}-{stream}", None, stream_mappings, log) utils.combine_dicts(integration_mappings, deepcopy(stream_mappings)) index_lookup[f"{integration}-{stream}"] = stream_mappings @@ -309,7 +314,7 @@ def get_filtered_index_schema( # noqa: PLR0913 # Need to use a merge here to not overwrite existing fields utils.combine_dicts(base, deepcopy(non_ecs_mapping.get(match, {}))) utils.combine_dicts(base, deepcopy(custom_mapping.get(match, {}))) - filtered_index_lookup[match] = prune_mappings_of_unsupported_types("index", match, base, log) + filtered_index_lookup[match] = prune_mappings_of_unsupported_types(None, match, base, log) utils.combine_dicts(combined_mappings, deepcopy(base)) # Reduce the index lookup to only the matched indices (remote/Kibana schema validation source of truth) @@ -495,8 +500,7 @@ def prepare_mappings( # noqa: PLR0913 # and also at a per index level as custom schemas can override non-ecs fields and/or indices non_ecs_schema = ecs.flatten(non_ecs_schema) non_ecs_schema = utils.convert_to_nested_schema(non_ecs_schema) - non_ecs_schema = prune_mappings_of_unsupported_types("non-ecs", "non-ecs", non_ecs_schema, log) - non_ecs_mapping = prune_mappings_of_unsupported_types("non-ecs", "non-ecs", non_ecs_mapping, log) + non_ecs_schema = prune_mappings_of_unsupported_types(None, "non-ecs", non_ecs_schema, log) # Load custom schema and convert to index mapping format (nested schema) custom_mapping: dict[str, Any] = {} @@ -506,7 +510,6 @@ def prepare_mappings( # noqa: PLR0913 index_mapping = ecs.flatten(index_mapping) index_mapping = utils.convert_to_nested_schema(index_mapping) custom_mapping.update({index: index_mapping}) - custom_mapping = prune_mappings_of_unsupported_types("custom", "custom", custom_mapping, log) # Load ECS in an index mapping format (nested schema) current_version = Version.parse(load_current_package_version(), optional_minor_and_patch=True) From eeec54d58b3470aa0e2a2e689492fee1bb5cfbc9 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Wed, 12 Nov 2025 19:26:18 -0500 Subject: [PATCH 5/6] Add additional unit test for schema conflicts --- tests/test_rules_remote.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_rules_remote.py b/tests/test_rules_remote.py index 91743a8e6a5..e74d9631acd 100644 --- a/tests/test_rules_remote.py +++ b/tests/test_rules_remote.py @@ -218,3 +218,20 @@ def test_esql_filtered_keep(self): """ with pytest.raises(EsqlSchemaError): _ = RuleCollection().load_dict(production_rule) + + def test_esql_non_ecs_schema_conflict_resolution(self): + """Test an ESQL rule that has a known conflict between non_ecs and integrations for correct handling.""" + file_path = get_path(["tests", "data", "command_control_dummy_production_rule.toml"]) + original_production_rule = load_rule_contents(file_path) + production_rule = deepcopy(original_production_rule)[0] + production_rule["metadata"]["integration"] = ["azure", "o365"] + production_rule["rule"]["query"] = """ + from logs-azure.signinlogs-* metadata _id, _version, _index + | where @timestamp > now() - 30 minutes + and event.dataset in ("azure.signinlogs") + and event.outcome == "success" + and azure.signinlogs.properties.user_id is not null + | keep + event.outcome + """ + _ = RuleCollection().load_dict(production_rule) From 97ca5b942417194faeb0434b17581dc64c21fb4c Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Thu, 13 Nov 2025 14:45:11 -0500 Subject: [PATCH 6/6] Update debug string naming --- detection_rules/index_mappings.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/detection_rules/index_mappings.py b/detection_rules/index_mappings.py index d6bae16db51..f3017145ad4 100644 --- a/detection_rules/index_mappings.py +++ b/detection_rules/index_mappings.py @@ -160,17 +160,16 @@ def get_simulated_index_template_mappings(elastic_client: Elasticsearch, name: s def prune_mappings_of_unsupported_types( - integration: str | None, index: str | None, stream_mappings: dict[str, Any], log: Callable[[str], None] + debug_str_data_source: str, stream_mappings: dict[str, Any], log: Callable[[str], None] ) -> dict[str, Any]: """Prune fields with unsupported types (ES|QL) from the provided mappings.""" - debug_str = integration if integration is not None else index nested_multifields = find_nested_multifields(stream_mappings) for field in nested_multifields: parts = str(field).split(".fields.")[0].split(".") base_name = ".properties.".join(parts) field_name = f"{base_name}.fields" log( - f"Warning: Nested multi-field `{field}` found in `{debug_str}`. " + f"Warning: Nested multi-field `{field}` found in `{debug_str_data_source}`. " f"Removing parent field from schema for ES|QL validation." ) delete_nested_key_from_dict(stream_mappings, field_name) @@ -183,7 +182,7 @@ def prune_mappings_of_unsupported_types( field_name = f"{base_name}.fields" property_name = f"{base_name}.properties" log( - f"Warning: flattened field `{field}` found in `{debug_str}` with sub fields. " + f"Warning: flattened field `{field}` found in `{debug_str_data_source}` with sub fields. " f"Removing parent field from schema for ES|QL validation." ) delete_nested_key_from_dict(stream_mappings, field_name) @@ -231,7 +230,7 @@ def prepare_integration_mappings( # noqa: PLR0913 for stream in package_schema: flat_schema = package_schema[stream] stream_mappings = flat_schema_to_index_mapping(flat_schema) - stream_mappings = prune_mappings_of_unsupported_types(f"{integration}-{stream}", None, stream_mappings, log) + stream_mappings = prune_mappings_of_unsupported_types(f"{integration}-{stream}", stream_mappings, log) utils.combine_dicts(integration_mappings, deepcopy(stream_mappings)) index_lookup[f"{integration}-{stream}"] = stream_mappings @@ -314,7 +313,7 @@ def get_filtered_index_schema( # noqa: PLR0913 # Need to use a merge here to not overwrite existing fields utils.combine_dicts(base, deepcopy(non_ecs_mapping.get(match, {}))) utils.combine_dicts(base, deepcopy(custom_mapping.get(match, {}))) - filtered_index_lookup[match] = prune_mappings_of_unsupported_types(None, match, base, log) + filtered_index_lookup[match] = prune_mappings_of_unsupported_types(match, base, log) utils.combine_dicts(combined_mappings, deepcopy(base)) # Reduce the index lookup to only the matched indices (remote/Kibana schema validation source of truth) @@ -500,7 +499,7 @@ def prepare_mappings( # noqa: PLR0913 # and also at a per index level as custom schemas can override non-ecs fields and/or indices non_ecs_schema = ecs.flatten(non_ecs_schema) non_ecs_schema = utils.convert_to_nested_schema(non_ecs_schema) - non_ecs_schema = prune_mappings_of_unsupported_types(None, "non-ecs", non_ecs_schema, log) + non_ecs_schema = prune_mappings_of_unsupported_types("non-ecs", non_ecs_schema, log) # Load custom schema and convert to index mapping format (nested schema) custom_mapping: dict[str, Any] = {}