feat: make mapping of Generic Resolver yaml compliant (#928)

* Add list[tuple[str]] as valid resolve_list type, change test to account for tuple, COMMITED WITH NO-VERIFY

* Add converter func

* Apparently resolve_list is a mapping from str to dict

* Add test to test converter

* update changelog

* Add typevar and rename func

* Rename INPUT_TYPE and add KEY_TYPE, extract to own utils file, with own utils test, add wrapper function and keep merge_dicts relativly clean

* Remove assert isinstance for now, rework converters, update tests

* Add FieldValue as Dict value

* pull up path

* Update tests and add doc

* Update logprep/processor/generic_resolver/rule.py

Co-authored-by: Michael Hoff <mail@michael-hoff.net>

* fix review comments

* Convert from file load, fix tests, fix Key and Value vartypes

* Upgrade uv.lock for protobuf cve fix

* Update logprep/processor/generic_resolver/rule.py

Co-authored-by: Michael Hoff <9436725+mhoff@users.noreply.github.com>

* Rename test id, add pydoc to converters

* Split long comment line into multiple

* Change test to expect ValueError not InvalidConfigurationError

---------

Co-authored-by: Michael Hoff <mail@michael-hoff.net>
Co-authored-by: Michael Hoff <9436725+mhoff@users.noreply.github.com>

Author: Pablu23

CHANGELOG.md

@@ -8,6 +8,7 @@
 * simplify Dockerfile and remove docker build support for `LOGPREP_VERSION`
 
 ### Bugfix
+* generic resolver now follows yaml standard and accepts a list instead of relying on the ordering of a dict
 
 ## 18.0.1
 ### Breaking

examples/exampledata/config/pipeline.yml

@@ -9,13 +9,13 @@ logger:
   format: "%(asctime)-15s %(hostname)-5s %(name)-10s %(levelname)-8s: %(message)s"
   datefmt: "%Y-%m-%d %H:%M:%S"
   loggers:
-    "py.warnings": {"level": "ERROR"}
-    "Runner": {"level": "DEBUG"}
-    "Processor": {"level": "ERROR"}
-    "Exporter": {"level": "ERROR"}
-    "uvicorn": {"level": "ERROR"}
-    "uvicorn.access": {"level": "ERROR"}
-    "OpenSearchOutput": {"level": "ERROR"}
+    "py.warnings": { "level": "ERROR" }
+    "Runner": { "level": "DEBUG" }
+    "Processor": { "level": "ERROR" }
+    "Exporter": { "level": "ERROR" }
+    "uvicorn": { "level": "ERROR" }
+    "uvicorn.access": { "level": "ERROR" }
+    "OpenSearchOutput": { "level": "ERROR" }
 metrics:
   enabled: true
   port: 8001
@@ -73,6 +73,15 @@ pipeline:
           calculator:
             target_field: "calculation"
             calc: "1 + 1"
+  - generic_resolver:
+      type: generic_resolver
+      rules:
+        - filter: "test_label"
+          generic_resolver:
+            field_mapping:
+              test_label: resolved
+            resolve_list:
+              - .*Hello.*: Greeting
 input:
   kafka:
     type: confluentkafka_input

logprep/processor/generic_resolver/rule.py

@@ -28,6 +28,23 @@
       resolve_list:
         .*Hello.*: Greeting
 
+For YAML compliance, it is possible to declare the resolve list as follows
+to maintain ordering when using the configuration file with different programs.
+Both styles will be supported in future; however, this one is recommended for clarity and YAML compliance.
+
+..  code-block:: yaml
+    :linenos:
+    :caption: Example
+
+    filter: to_resolve
+    generic_resolver:
+      field_mapping:
+        to_resolve: resolved
+      resolve_list:
+        - .*Hello.*: Greeting
+        - .*error.*: Error
+        - never_match: Panic
+
 Alternatively, a YML file with a resolve list and a regex pattern can be used to resolve values.
 For this, a field :code:`resolve_from_file` with the subfields :code:`path` and :code:`pattern`
 must be added.
@@ -117,15 +134,19 @@
 """
 
 import re
-from functools import cached_property
+import typing
+from functools import cached_property, partial
 from pathlib import Path
-from typing import List, Optional, Tuple, Union
 
 from attrs import define, field, validators
 
 from logprep.factory_error import InvalidConfigurationError
 from logprep.processor.field_manager.rule import FieldManagerRule
+from logprep.util.converters import (
+    convert_ordered_mapping_or_keep_mapping,
+)
 from logprep.util.getter import GetterFactory, RefreshableGetter
+from logprep.util.helper import FieldValue
 
 
 class GenericResolverRule(FieldManagerRule):
@@ -145,15 +166,22 @@ class Config(FieldManagerRule.Config):
             ]
         )
         """Mapping in form of :code:`{SOURCE_FIELD: DESTINATION_FIELD}`"""
-        resolve_list: dict = field(validator=(validators.instance_of(dict)), factory=dict)
+        resolve_list: dict[str, FieldValue] = field(
+            validator=validators.deep_mapping(
+                key_validator=validators.instance_of(str),
+                mapping_validator=validators.instance_of(dict),
+            ),
+            converter=convert_ordered_mapping_or_keep_mapping,
+            factory=dict,
+        )
         """lookup mapping in form of
         :code:`{REGEX_PATTERN_0: ADDED_VALUE_0, ..., REGEX_PATTERN_N: ADDED_VALUE_N}`"""
         resolve_from_file: dict = field(
             validator=[
                 validators.instance_of(dict),
                 validators.deep_mapping(
                     key_validator=validators.in_(["path", "pattern"]),
-                    value_validator=validators.instance_of(Union[str, int]),
+                    value_validator=validators.instance_of((str, int)),
                 ),
             ],
             factory=dict,
@@ -178,38 +206,39 @@ class Config(FieldManagerRule.Config):
            authenticity and integrity of the loaded values.
 
         """
-        ignore_case: Optional[str] = field(validator=validators.instance_of(bool), default=False)
+        ignore_case: bool = field(validator=validators.instance_of(bool), default=False)
         """(Optional) Ignore case when matching resolve values. Defaults to :code:`False`."""
 
         additions: dict = field(default={}, eq=False, init=False)
         """Contains a dictionary of field names and values that should be added."""
 
         @property
-        def _file_path(self):
+        def _file_path(self) -> None | str:
             """Returns the file path"""
             return self.resolve_from_file.get("path")
 
         def __attrs_post_init__(self):
             if self._file_path:
                 getter = GetterFactory.from_string(self._file_path)
                 if isinstance(getter, RefreshableGetter):
-                    getter.add_callback(self._add_from_path)
-                self._add_from_path()
+                    getter.add_callback(partial(self._add_from_path, self._file_path))
+                self._add_from_path(self._file_path)
 
-        def _add_from_path(self):
+        def _add_from_path(self, path: str):
             self._raise_if_pattern_is_invalid()
-            self._raise_if_file_does_not_exist()
-            additions = self._get_additions()
+            self._raise_if_file_does_not_exist(path)
+            additions = self._get_additions_from_path(path)
             if self.ignore_case:
                 additions = {key.upper(): value for key, value in additions.items()}
             self.additions = additions
 
-        def _get_additions(self) -> dict:
+        def _get_additions_from_path(self, path: str) -> dict:
             try:
-                additions = GetterFactory.from_string(self._file_path).get_dict()
+                additions = GetterFactory.from_string(path).get_collection()
+                additions = convert_ordered_mapping_or_keep_mapping(additions)
             except ValueError as error:
                 raise InvalidConfigurationError(
-                    f"Error loading additions from '{self._file_path}': {error}"
+                    f"Error loading additions from '{path}': {error}"
                 ) from error
             return additions
 
@@ -219,39 +248,44 @@ def _raise_if_pattern_is_invalid(self):
                     f"Mapping group is missing in mapping file pattern! (Rule ID: '{self.id}')"
                 )
 
-        def _raise_if_file_does_not_exist(self):
-            if not (self._file_path.startswith("http") or Path(self._file_path).is_file()):
+        def _raise_if_file_does_not_exist(self, path: str):
+            if not (path.startswith("http") or Path(path).is_file()):
                 raise InvalidConfigurationError(
-                    f"Additions file '{self._file_path}' not found! (Rule ID: '{self.id}')",
+                    f"Additions file '{path}' not found! (Rule ID: '{self.id}')",
                 )
 
+    @property
+    def config(self) -> Config:
+        """Returns the typed GenericResolverRule.Config"""
+        return typing.cast(GenericResolverRule.Config, self._config)
+
     @property
     def field_mapping(self) -> dict:
         """Returns the field mapping"""
-        return self._config.field_mapping
+        return self.config.field_mapping
 
     @property
     def resolve_list(self) -> dict:
         """Returns the resolve list"""
-        return self._config.resolve_list
+        return self.config.resolve_list
 
     @cached_property
-    def compiled_resolve_list(self) -> List[Tuple[re.Pattern, str]]:
+    def compiled_resolve_list(self) -> list[tuple[re.Pattern, FieldValue]]:
         """Returns the resolve list with tuple pairs of compiled patterns and values"""
         return [
             (re.compile(pattern, re.I if self.ignore_case else 0), val)
-            for pattern, val in self._config.resolve_list.items()
+            for pattern, val in self.config.resolve_list.items()
         ]
 
     @property
     def resolve_from_file(self) -> dict:
         """Returns the resolve file"""
-        return self._config.resolve_from_file
+        return self.config.resolve_from_file
 
     @property
     def ignore_case(self) -> bool:
         """Returns if the matching should be case-sensitive or not"""
-        return self._config.ignore_case
+        return self.config.ignore_case
 
     @cached_property
     def pattern(self) -> re.Pattern:
@@ -261,4 +295,4 @@ def pattern(self) -> re.Pattern:
     @property
     def additions(self) -> dict:
         """Returns additions from the resolve file"""
-        return self._config.additions
+        return self.config.additions

logprep/util/converters.py

@@ -0,0 +1,31 @@
+from typing import TypeVar
+
+Key = TypeVar("Key")
+Value = TypeVar("Value")
+
+
+def convert_ordered_mapping_or_keep_mapping(
+    dict_or_sequence: dict[Key, Value] | list[dict[Key, Value]],
+) -> dict[Key, Value]:
+    """Convert a list of key Values to a single dict, with no reocurring keys and only single item dicts, if input is already singular dict, return early"""
+    if isinstance(dict_or_sequence, dict):
+        return dict_or_sequence
+
+    if not isinstance(dict_or_sequence, list):
+        raise ValueError("expected list")
+
+    return convert_ordered_mapping(dict_or_sequence)
+
+
+def convert_ordered_mapping(dicts: list[dict[Key, Value]]) -> dict[Key, Value]:
+    """Convert a list of key Values to a single dict, with no reocurring keys and only single item dicts"""
+    ordered_mapping = {}
+    for element in dicts:
+        keys = list(element.keys())
+        if len(keys) != 1:
+            raise ValueError("dict has not exactly one key")
+        if keys[0] in ordered_mapping:
+            raise ValueError("dict already has key")
+        ordered_mapping[keys[0]] = element[keys[0]]
+
+    return ordered_mapping