Added config item for extra tags

Author: mmguero

scripts/installer/configs/configuration_items/capture.py

@@ -9,6 +9,7 @@
 This module contains all configuration items related to live traffic capture settings,
 including network interface configuration, capture filters, and capture methods.
 """
+from typing import Any, Tuple
 
 from scripts.malcolm_constants import WidgetType
 from scripts.malcolm_utils import get_hostname_without_domain
@@ -24,6 +25,7 @@
     KEY_CONFIG_ITEM_PCAP_IFACE,
     KEY_CONFIG_ITEM_PCAP_NETSNIFF,
     KEY_CONFIG_ITEM_PCAP_NODE_NAME,
+    KEY_CONFIG_ITEM_EXTRA_TAGS,
     KEY_CONFIG_ITEM_PCAP_TCPDUMP,
     KEY_CONFIG_ITEM_TWEAK_IFACE,
 )
@@ -132,6 +134,54 @@
 )
 
 
+class CustomTagsConfigItem(ConfigItem):
+    """Custom ConfigItem that converts comma-separated strings to lists for extra tags"""
+
+    def set_value(self, value: Any) -> Tuple[bool, str]:
+        """Set and validate a new value, with automatic string-to-list conversion."""
+        # Convert string input to list if needed
+        if isinstance(value, str):
+            if value.strip():
+                converted_value = [s.strip() for s in value.split(",") if s.strip()]
+            else:
+                converted_value = []
+        else:
+            converted_value = value
+
+        # Now validate the converted value
+        if self.validator:
+            result = self.validator(converted_value)
+            if isinstance(result, tuple):
+                valid, error = result
+            else:
+                valid = result
+                error = "Invalid value" if not valid else ""
+
+            if not valid:
+                return False, error
+
+        # Store the converted value
+        self.is_modified = True
+        self.value = converted_value
+        return True, ""
+
+
+def _validate_extra_tags_list(x):
+    """Validate that input is a list of strings (or a single string)"""
+    return isinstance(x, str) or (isinstance(x, list) and all(isinstance(val, str) for val in x))
+
+
+CONFIG_ITEM_EXTRA_TAGS = CustomTagsConfigItem(
+    key=KEY_CONFIG_ITEM_EXTRA_TAGS,
+    label="Extra Tags",
+    default_value=[],
+    accept_blank=True,
+    validator=_validate_extra_tags_list,
+    question="Comma-separated list of tags for data generated by Malcolm",
+    widget_type=WidgetType.TEXT,
+)
+
+
 def get_capture_config_item_dict():
     """Get all ConfigItem objects from this module.
 

scripts/installer/configs/constants/config_env_var_keys.py

@@ -101,6 +101,7 @@
 KEY_ENV_PCAP_IFACE = "PCAP_IFACE"  # capture interface(s)
 KEY_ENV_PCAP_FILTER = "PCAP_FILTER"  # capture filter
 KEY_ENV_PCAP_NODE_NAME = "PCAP_NODE_NAME"  # capture source "node name" for locally processed PCAP files
+KEY_ENV_EXTRA_TAGS = "EXTRA_TAGS"  # Comma-separated list of tags for data generated by Malcolm
 KEY_ENV_PCAP_PIPELINE_POLLING = "PCAP_PIPELINE_POLLING"  # Use polling for file watching vs. native
 
 KEY_ENV_PGID = "PGID"  # process Group ID

scripts/installer/configs/constants/configuration_item_keys.py

@@ -30,6 +30,7 @@
 KEY_CONFIG_ITEM_LIVE_ZEEK = "liveZeek"
 KEY_CONFIG_ITEM_LIVE_SURICATA = "liveSuricata"
 KEY_CONFIG_ITEM_PCAP_NODE_NAME = "pcapNodeName"
+KEY_CONFIG_ITEM_EXTRA_TAGS = "extraTags"
 
 # Docker options
 KEY_CONFIG_ITEM_MALCOLM_RESTART_POLICY = "malcolmRestartPolicy"

scripts/installer/core/config_env_mapper.py

@@ -9,6 +9,10 @@
 Includes reverse import from existing env files and lookup helpers.
 """
 
+try:
+    from collections.abc import Iterable
+except ImportError:
+    from collections import Iterable
 from collections import defaultdict
 from dataclasses import dataclass, field
 from typing import Any, Callable, Dict, List, Optional
@@ -51,6 +55,23 @@ def _default_string_reverse_transform(value: str):
     return value
 
 
+def _default_list_of_strings_transform(value: Any) -> List[str]:
+    if value is None:
+        result = []
+    elif isinstance(value, str):
+        result = [value]
+    elif isinstance(value, Iterable):
+        result = [str(v) for v in value]
+    else:
+        result = [str(value)]
+
+    return ",".join([true_or_false_no_quotes(x).strip() for x in result])
+
+
+def _default_list_of_strings_reverse_transform(value: str) -> List[str]:
+    return [s.strip() for s in value.split(",") if s.strip()] if value.strip() else []
+
+
 # 1. Boolean transform logic (single config item only)
 _BOOLEAN_VARS = [
     KEY_ENV_ARKIME_MANAGE_PCAP_FILES,
@@ -130,8 +151,13 @@ def _default_string_reverse_transform(value: str):
     KEY_ENV_ZEEK_VTOT_API2_KEY,
 ]
 
+# 3. List-of-strings transform logic
+_LIST_OF_STRING_VARS = [
+    KEY_ENV_EXTRA_TAGS,
+]
+
 
-# 3. Custom transform logic
+# 4. Custom transform logic
 @dataclass(frozen=True)
 class TransformHook:
     forward: Callable
@@ -366,6 +392,9 @@ def __init__(self):
 
                                     if map_key_constant_value in _CUSTOM_VARS:
                                         self._handle_custom_transform(env_var_instance, map_key_constant_value)
+                                    elif map_key_constant_value in _LIST_OF_STRING_VARS:
+                                        env_var_instance.transform = _default_list_of_strings_transform
+                                        env_var_instance.reverse_transform = _default_list_of_strings_reverse_transform
                                     elif map_key_constant_value in _STRING_VARS:
                                         env_var_instance.transform = _default_string_transform
                                         env_var_instance.reverse_transform = _default_string_reverse_transform
@@ -568,6 +597,7 @@ def __init__(self):
             self.env_var_by_map_key[KEY_ENV_PCAP_IFACE].config_items = [KEY_CONFIG_ITEM_PCAP_IFACE]
             self.env_var_by_map_key[KEY_ENV_PCAP_FILTER].config_items = [KEY_CONFIG_ITEM_PCAP_FILTER]
             self.env_var_by_map_key[KEY_ENV_PCAP_NODE_NAME].config_items = [KEY_CONFIG_ITEM_PCAP_NODE_NAME]
+            self.env_var_by_map_key[KEY_ENV_EXTRA_TAGS].config_items = [KEY_CONFIG_ITEM_EXTRA_TAGS]
             self.env_var_by_map_key[KEY_ENV_PCAP_PIPELINE_POLLING].config_items = [
                 KEY_CONFIG_ITEM_DOCKER_ORCHESTRATION_MODE
             ]

scripts/installer/core/dependencies.py

@@ -265,6 +265,13 @@ def __repr__(self) -> str:
         )
     ),
     # Malcolm profile children
+    KEY_CONFIG_ITEM_EXTRA_TAGS: DependencySpec(
+        visibility=VisibilityRule(
+            depends_on=KEY_CONFIG_ITEM_MALCOLM_PROFILE,
+            condition=lambda profile: profile in (PROFILE_HEDGEHOG, PROFILE_MALCOLM),
+            ui_parent=KEY_CONFIG_ITEM_MALCOLM_PROFILE,
+        )
+    ),
     KEY_CONFIG_ITEM_MALCOLM_MAINTAIN_OPENSEARCH: DependencySpec(
         visibility=VisibilityRule(
             depends_on=KEY_CONFIG_ITEM_MALCOLM_PROFILE,

scripts/installer/core/malcolm_config.py

@@ -32,7 +32,7 @@
     SURICATA_LOG_CONTAINER_PATH,
     YAMLDynamic,
 )
-from scripts.malcolm_utils import deep_get, get_main_script_path, same_file_or_dir, touch
+from scripts.malcolm_utils import deep_get, get_main_script_path, same_file_or_dir, touch, unwrap_method
 
 from scripts.installer.configs.constants.config_env_var_keys import *
 from scripts.installer.configs.constants.configuration_item_keys import *
@@ -70,6 +70,12 @@
 from scripts.installer.core.transform_registry import apply_inbound
 
 
+def overrides_set_value(obj):
+    item_func = unwrap_method(obj.__class__.set_value)
+    base_func = unwrap_method(ConfigItem.set_value)
+    return item_func is not base_func
+
+
 class MalcolmConfig(ObservableStoreMixin):
     """Configuration store managing items, dependencies, and env mapping."""
 
@@ -291,8 +297,16 @@ def apply_default(
             if item.get_value() == value:
                 return
 
-            # Apply without flipping is_modified so future syncs can adjust
-            item.value = value
+            if not overrides_set_value(item):
+                # set value directly without set_value (i.e., without flipping is_modified) so future syncs can adjust
+                item.value = value
+            else:
+                # this config item overrides set_value so they must want to do something special,
+                # so call its set_value but preserve its is_modified flag
+                old_modified = item.is_modified
+                item.set_value(value)
+                item.is_modified = old_modified
+
             try:
                 self._notify_observers(key, item.get_value())
             except Exception as e:

scripts/malcolm_utils.py

@@ -5,6 +5,7 @@
 
 import contextlib
 import fnmatch
+import functools
 import glob
 import hashlib
 import inspect
@@ -884,6 +885,25 @@ def temporary_filename(suffix=None):
         os.unlink(tmp_name)
 
 
+###################################################################################################
+# Returns the raw underlying function behind a method, classmethod, staticmethod, or functools.partial/wrapped method.
+def unwrap_method(method):
+
+    # Handle classmethod / staticmethod
+    if isinstance(method, (classmethod, staticmethod)):
+        method = method.__func__
+
+    # Unwrap functools.partial, wraps, etc.
+    while hasattr(method, "__wrapped__"):
+        method = method.__wrapped__
+
+    # functools.partialmethod stores the underlying function in `func`
+    if isinstance(method, functools.partial):
+        method = method.func
+
+    return method
+
+
 ###################################################################################################
 # open a file and close it, updating its access time
 def touch(filename):