Merge branch 'main' of https://github.com/mmguero-dev/Malcolm into trixie

Author: mmguero

Dockerfiles/zeek.Dockerfile

@@ -33,7 +33,7 @@ USER root
 # see PUSER_CHOWN at the bottom of the file (after the other environment variables it references)
 
 # for download and install
-ARG ZEEK_VERSION=7.2.2-0
+ARG ZEEK_VERSION=8.0.0-0
 ENV ZEEK_VERSION $ZEEK_VERSION
 ARG ZEEK_DEB_ALTERNATE_DOWNLOAD_URL=""
 
@@ -69,6 +69,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/')
       ca-certificates \
       ccache \
       cmake \
+      cppzmq-dev \
       curl \
       file \
       flex \
@@ -89,11 +90,14 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/')
       libmaxminddb0 \
       libpcap-dev \
       libpcap0.8 \
+      librdkafka++1 \
       librdkafka-dev \
+      librdkafka1 \
       libssl-dev \
       libssl3 \
       libtcmalloc-minimal4 \
       libunwind8 \
+      libzmq3-dev \
       libzmq5 \
       locales-all \
       make \

arkime/etc/config.ini

@@ -513,11 +513,6 @@ zeek.dns.answers=db:zeek.dns.answers;group:zeek_dns;kind:termfield;viewerOnly:tr
 zeek.dns.TTLs=db:zeek.dns.TTLs;group:zeek_dns;kind:termfield;viewerOnly:true;friendly:TTL;help:TTL
 zeek.dns.rejected=db:zeek.dns.rejected;group:zeek_dns;kind:termfield;viewerOnly:true;friendly:Rejected;help:Rejected
 
-# dpd.log
-# https://docs.zeek.org/en/stable/scripts/base/frameworks/dpd/main.zeek.html#type-DPD::Info
-zeek.dpd.service=db:zeek.dpd.service;group:zeek_dpd;kind:termfield;viewerOnly:true;friendly:Protocol;help:Protocol
-zeek.dpd.failure_reason=db:zeek.dpd.failure_reason;group:zeek_dpd;kind:termfield;viewerOnly:true;friendly:Failure Reason;help:Failure Reason
-
 # enip.log
 # https://github.com/cisagov/ICSNPP
 zeek.enip.enip_command=db:zeek.enip.enip_command;group:zeek_enip;kind:termfield;viewerOnly:true;friendly:EthernetIP Command;help:EthernetIP Command
@@ -2575,6 +2570,12 @@ zeek.rdp.cert_permanent=db:zeek.rdp.cert_permanent;group:zeek_rdp;kind:termfield
 zeek.rdp.encryption_level=db:zeek.rdp.encryption_level;group:zeek_rdp;kind:termfield;viewerOnly:true;friendly:Encryption Level;help:Encryption Level
 zeek.rdp.encryption_method=db:zeek.rdp.encryption_method;group:zeek_rdp;kind:termfield;viewerOnly:true;friendly:Encryption Method;help:Encryption Method
 
+# redis.log
+# https://docs.zeek.org/en/v8.0.0/scripts/base/protocols/redis/main.zeek.html#type-Redis::Info
+zeek.redis.cmd=db:zeek.redis.cmd;group:zeek_redis;kind:termfield;viewerOnly:true;friendly:Command;help:Command
+zeek.redis.success=db:zeek.redis.success;group:zeek_redis;kind:termfield;viewerOnly:true;friendly:Success;help:Success
+zeek.redis.reply=db:zeek.redis.reply;group:zeek_redis;kind:termfield;viewerOnly:true;friendly:Reply;help:Reply
+
 # rfb.log
 # https://docs.zeek.org/en/stable/scripts/base/protocols/rfb/main.zeek.html#type-RFB::Info
 zeek.rfb.client_major_version=db:zeek.rfb.client_major_version;group:zeek_rfb;kind:termfield;viewerOnly:true;friendly:Client Major Version;help:Client Major Version
@@ -3885,7 +3886,6 @@ o_zeek_dnp3=require:zeek.dnp3;title:Zeek dnp3.log;fields:zeek.dnp3.fc_request,ze
 o_zeek_dnp3_control=require:zeek.dnp3_control;title:Zeek dnp3_control.log;fields:zeek.dnp3_control.block_type,zeek.dnp3_control.function_code,zeek.dnp3_control.index_number,zeek.dnp3_control.trip_control_code,zeek.dnp3_control.operation_type,zeek.dnp3_control.clear_bit,zeek.dnp3_control.execute_count,zeek.dnp3_control.on_time,zeek.dnp3_control.off_time,zeek.dnp3_control.status_code
 o_zeek_dnp3_objects=require:zeek.dnp3_objects;title:Zeek dnp3_objects.log;fields:zeek.dnp3_objects.function_code,zeek.dnp3_objects.object_type,zeek.dnp3_objects.object_count,zeek.dnp3_objects.range_low,zeek.dnp3_objects.range_high
 o_zeek_dns=require:zeek.dns;title:Zeek dns.log;fields:zeek.dns.trans_id,zeek.dns.rtt,zeek.dns.query,zeek.dns.qclass,zeek.dns.qclass_name,zeek.dns.qtype,zeek.dns.qtype_name,zeek.dns.rcode,zeek.dns.rcode_name,zeek.dns.AA,zeek.dns.TC,zeek.dns.RD,zeek.dns.RA,zeek.dns.Z,zeek.dns.answers,zeek.dns.TTLs,zeek.dns.rejected
-o_zeek_dpd=require:zeek.dpd;title:Zeek dpd.log;fields:zeek.dpd.service,zeek.dpd.failure_reason
 o_zeek_ecat_aoe_info=require:zeek.ecat_aoe_info;title:Zeek ecat_aoe_info.log;fields:zeek.ecat_aoe_info.resp_port,zeek.ecat_aoe_info.orig_port,zeek.ecat_aoe_info.command,zeek.ecat_aoe_info.state,zeek.ecat_aoe_info.data
 o_zeek_ecat_arp_info=require:zeek.ecat_arp_info;title:Zeek ecat_arp_info.log;fields:zeek.ecat_arp_info.arp_type,zeek.ecat_arp_info.orig_proto_addr,zeek.ecat_arp_info.orig_hw_addr,zeek.ecat_arp_info.resp_proto_addr,zeek.ecat_arp_info.resp_hw_addr
 o_zeek_ecat_coe_info=require:zeek.ecat_coe_info;title:Zeek ecat_coe_info.log;fields:zeek.ecat_coe_info.number,zeek.ecat_coe_info.type,zeek.ecat_coe_info.req_resp,zeek.ecat_coe_info.index,zeek.ecat_coe_info.subindex,zeek.ecat_coe_info.dataoffset
@@ -3943,6 +3943,7 @@ o_zeek_profinet_dce_rpc=require:zeek.profinet_dce_rpc;title:Zeek profinet_dce_rp
 o_zeek_profinet_io_cm=require:zeek.profinet_io_cm;title:Zeek profinet_io_cm.log;fields:zeek.profinet_io_cm.rpc_version,zeek.profinet_io_cm.packet_type,zeek.profinet_io_cm.reserved_for_impl_1,zeek.profinet_io_cm.last_fragment,zeek.profinet_io_cm.fragment,zeek.profinet_io_cm.no_fragment_requested,zeek.profinet_io_cm.maybe,zeek.profinet_io_cm.idempotent,zeek.profinet_io_cm.broadcast,zeek.profinet_io_cm.reserved_for_impl_2,zeek.profinet_io_cm.cancel_was_pending_at_call_end,zeek.profinet_io_cm.integer_encoding,zeek.profinet_io_cm.character_encoding,zeek.profinet_io_cm.floating_point_encoding,zeek.profinet_io_cm.serial_high,zeek.profinet_io_cm.object_uuid,zeek.profinet_io_cm.interface_uuid,zeek.profinet_io_cm.activity_uuid,zeek.profinet_io_cm.server_boot_time,zeek.profinet_io_cm.uuid_version,zeek.profinet_io_cm.sequence_num,zeek.profinet_io_cm.operation,zeek.profinet_io_cm.interface_hint,zeek.profinet_io_cm.activity_hint,zeek.profinet_io_cm.len_of_body,zeek.profinet_io_cm.fragment_num,zeek.profinet_io_cm.auth_protocol,zeek.profinet_io_cm.serial_low,zeek.profinet_io_cm.vers_fack,zeek.profinet_io_cm.window_size,zeek.profinet_io_cm.max_tsdu,zeek.profinet_io_cm.max_frag_size,zeek.profinet_io_cm.serial_number,zeek.profinet_io_cm.sel_ack_len,zeek.profinet_io_cm.sel_ack
 o_zeek_radius=require:zeek.radius;title:Zeek radius.log;fields:zeek.radius.mac,zeek.radius.framed_addr,zeek.radius.tunnel_client,zeek.radius.connect_info,zeek.radius.reply_msg,zeek.radius.result,zeek.radius.ttl
 o_zeek_rdp=require:zeek.rdp;title:Zeek rdp.log;fields:zeek.rdp.cookie,zeek.rdp.result,zeek.rdp.security_protocol,zeek.rdp.client_channels,zeek.rdp.keyboard_layout,zeek.rdp.client_build,zeek.rdp.client_name,zeek.rdp.client_dig_product_id,zeek.rdp.desktop_width,zeek.rdp.desktop_height,zeek.rdp.requested_color_depth,zeek.rdp.cert_type,zeek.rdp.cert_count,zeek.rdp.cert_permanent,zeek.rdp.encryption_level,zeek.rdp.encryption_method
+o_zeek_redis=require:zeek.redis;title:Zeek redis.log;fields:zeek.redis.cmd,zeek.redis.success,zeek.redis.reply
 o_zeek_rfb=require:zeek.rfb;title:Zeek rfb.log;fields:zeek.rfb.client_major_version,zeek.rfb.client_minor_version,zeek.rfb.server_major_version,zeek.rfb.server_minor_version,zeek.rfb.authentication_method,zeek.rfb.auth,zeek.rfb.share_flag,zeek.rfb.desktop_name,zeek.rfb.width,zeek.rfb.height
 o_zeek_roc_plus=require:zeek.roc_plus;title:Zeek roc_plus.log;fields:zeek.roc_plus.link_id,zeek.roc_plus.point_type,zeek.roc_plus.packet_type,zeek.roc_plus.destination_unit,zeek.roc_plus.destination_group,zeek.roc_plus.source_unit,zeek.roc_plus.source_group,zeek.roc_plus.command,zeek.roc_plus.opcode,zeek.roc_plus.data,zeek.roc_plus.data_length,zeek.roc_plus.lsb_crc,zeek.roc_plus.msb_crc,zeek.roc_plus.error_code,zeek.roc_plus.error_offset,zeek.roc_plus.num_points,zeek.roc_plus.point_logic_number
 o_zeek_roc_plus_configurable_opcode=require:zeek.roc_plus_configurable_opcode;title:Zeek roc_plus_configurable_opcode.log;fields:zeek.roc_plus_configurable_opcode.table_number,zeek.roc_plus_configurable_opcode.starting_table_location,zeek.roc_plus_configurable_opcode.num_table_locations,zeek.roc_plus_configurable_opcode.table_version_number

arkime/wise/source.zeeklogs.js

@@ -993,8 +993,6 @@ class MalcolmSource extends WISESource {
       "zeek.dns.trans_id",
       "zeek.dns.TTLs",
       "zeek.dns.Z",
-      "zeek.dpd.failure_reason",
-      "zeek.dpd.service",
       "zeek.ecat_aoe_info.command",
       "zeek.ecat_aoe_info.data",
       "zeek.ecat_aoe_info.orig_port",
@@ -2735,6 +2733,9 @@ class MalcolmSource extends WISESource {
       "zeek.rdp.requested_color_depth",
       "zeek.rdp.result",
       "zeek.rdp.security_protocol",
+      "zeek.redis.cmd",
+      "zeek.redis.success",
+      "zeek.redis.reply",
       "zeek.rfb.auth",
       "zeek.rfb.authentication_method",
       "zeek.rfb.client_major_version",

dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json

@@ -449,7 +449,7 @@
         "title": "Zeek Analyzer Messages",
         "uiStateJSON": "{}",
         "version": 1,
-        "visState": "{\"title\":\"Zeek Analyzer Messages\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.analyzer.cause\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Cause\"},\"schema\":\"bucket\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.analyzer.analyzer_kind\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Class\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.analyzer.analyzer_name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Analyzer\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}"
+        "visState": "{\"title\":\"Zeek Analyzer Messages\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.analyzer.failure_reason\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Failure Reason\"},\"schema\":\"bucket\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.analyzer.analyzer_kind\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Class\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.analyzer.analyzer_name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Analyzer\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}"
       },
       "id": "abcfca50-d7dd-11ee-b25e-e793ed358448",
       "migrationVersion": {
@@ -475,7 +475,6 @@
           "host.name",
           "zeek.analyzer.analyzer_kind",
           "zeek.analyzer.analyzer_name",
-          "zeek.analyzer.cause",
           "zeek.analyzer.failure_reason",
           "source.ip",
           "destination.ip",

dashboards/templates/composable/component/zeek.json

@@ -13,6 +13,12 @@
           "properties": {
             "analyzer": {
               "properties": {
+                "analyzer_kind": {
+                  "type": "keyword"
+                },
+                "analyzer_name": {
+                  "type": "keyword"
+                },
                 "cause": {
                   "type": "keyword",
                   "ignore_above": 1024,
@@ -22,13 +28,7 @@
                     }
                   }
                 },
-                "analyzer_kind": {
-                  "type": "keyword"
-                },
-                "analyzer_name": {
-                  "type": "keyword"
-                },
-                "failure_reason": {
+                "failure_data": {
                   "type": "keyword",
                   "ignore_above": 1024,
                   "fields": {
@@ -37,7 +37,7 @@
                     }
                   }
                 },
-                "failure_data": {
+                "failure_reason": {
                   "type": "keyword",
                   "ignore_above": 1024,
                   "fields": {
@@ -232,22 +232,6 @@
                 }
               }
             },
-            "dpd": {
-              "properties": {
-                "failure_reason": {
-                  "type": "keyword",
-                  "ignore_above": 1024,
-                  "fields": {
-                    "text": {
-                      "type": "text"
-                    }
-                  }
-                },
-                "service": {
-                  "type": "keyword"
-                }
-              }
-            },
             "files": {
               "properties": {
                 "analyzers": {
@@ -1429,6 +1413,19 @@
                 }
               }
             },
+            "redis": {
+              "properties": {
+                "cmd": {
+                  "type": "keyword"
+                },
+                "success": {
+                  "type": "keyword"
+                },
+                "reply": {
+                  "type": "keyword"
+                }
+              }
+            },
             "rfb": {
               "properties": {
                 "auth": {

hedgehog-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot

@@ -9,7 +9,7 @@ apt-get -y --purge remove \
   libc6-dbg \
   ninja-build \
   sparse \
-  $(dpkg --get-selections | grep -Pv "(^(dpkg|libbroker|libc6|libcrypt|libdbus|libffi|libfl|libgoogle-perftools|libgcc|libkrb5|librdkafka|libmaxminddb|libncurses|libnsl|libobjc|libomp|libpcap|libssl|libstdc|libtinfo|libtirpc|libunwind|libxml|libyaml|libz|linux-libc|python3|zeek|zlib1g)|deinstall$)" | cut -f1 | grep -P -- '-dev(:\w+)?$') || true
+  $(dpkg --get-selections | grep -Pv "(^(cppzmq-dev|dpkg|libbroker|libc6|libcrypt|libdbus|libffi|libfl|libgoogle-perftools|libgcc|libkrb5|librdkafka|libmaxminddb|libncurses|libnsl|libobjc|libomp|libpcap|libssl|libstdc|libtinfo|libzmq5|libzmq3-dev|libtirpc|libunwind|libxml|libyaml|libz|linux-libc|python3|zeek|zlib1g)|deinstall$)" | cut -f1 | grep -P -- '-dev(:\w+)?$') || true
 rm -rf /var/spool/ccache
 
 # remove unwanted packages

hedgehog-iso/config/package-lists/system.list.chroot

@@ -22,6 +22,7 @@ bzip2
 cifs-utils
 coreutils
 cpio
+cppzmq-dev
 cracklib-runtime
 cryptmount
 cryptsetup
@@ -107,6 +108,7 @@ libwww-perl
 libyaml-dev
 libykpers-1-1
 libyubikey0
+libzmq3-dev
 libzmq5
 lm-sensors
 localepurge

logstash/maps/service_ports.yaml

@@ -154,6 +154,8 @@ rdp:
   - 259
   - 2179
   - 3389
+redis:
+  - 6379
 rfb:
   - 2654
   - 5800

logstash/maps/zeek_log_ecs_categories.yaml

@@ -136,6 +136,7 @@
 "profinet_io_cm": ["ot", "network"]
 "radius": ["authentication", "iam", "network"]
 "rdp": ["network"]
+"redis": ["database", "network"]
 "rfb": ["network"]
 "roc_plus": ["ot", "network"]
 "roc_plus_configurable_opcode": ["ot", "network"]

logstash/pipelines/zeek/1000_zeek_prep.conf

@@ -26,7 +26,7 @@ filter {
   # Zeek logs we're going to ignore
   ruby {
     id => "ruby_zeek_log_type_determine_drop"
-    init => "logtypesStr = ENV['LOGSTASH_ZEEK_IGNORED_LOGS'] || 'analyzer,broker,bsap_ip_unknown,bsap_serial_unknown,capture_loss,cluster,config,ecat_arp_info,loaded_scripts,packet_filter,png,print,prof,reporter,roc_plus_unknown_data,stats,stderr,stdout' ; @logtypes = logtypesStr.gsub(/\s+/, '').split(',')"
+    init => "logtypesStr = ENV['LOGSTASH_ZEEK_IGNORED_LOGS'] || 'analyzer,analyzer_debug,broker,bsap_ip_unknown,bsap_serial_unknown,capture_loss,cluster,config,ecat_arp_info,loaded_scripts,packet_filter,png,print,prof,reporter,roc_plus_unknown_data,stats,stderr,stdout' ; @logtypes = logtypesStr.gsub(/\s+/, '').split(',')"
     code => "event.set('[@metadata][drop_zeek_log]', true) if @logtypes.include?(event.get('[log_source]').to_s)"
   }
   if [@metadata][drop_zeek_log] { drop { id => "drop_zeek_ignored_source" } }