for cisagov#703, record stats for network interfaces periodically

Author: mmguero

config/pcap-capture.env.example

@@ -20,4 +20,6 @@ PCAP_ROTATE_MEGABYTES=4096
 #   will be closed for processing and a new PCAP file created
 PCAP_ROTATE_MINUTES=10
 # Specifies a tcpdump-style filter expression for local packet capture ('' to capture all traffic)
-PCAP_FILTER=
+PCAP_FILTER=
+# Get OS-level stats for network interfaces periodically (leave blank to disable)
+PCAP_IFACE_STATS_CRON_EXPRESSION=

dashboards/dashboards/beats/Metricbeat-system-overview.json

@@ -108,7 +108,7 @@
       "attributes": {
         "description": "",
         "kibanaSavedObjectMeta": {
-          "searchSourceJSON": "{\"query\":{\"query\":\"miscbeat.mem:* OR miscbeat.cpu:*\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"
+          "searchSourceJSON": "{\"query\":{\"query\":\"miscbeat.mem:* OR miscbeat.cpu:* OR miscbeat.network:*\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"
         },
         "title": "Number of hosts",
         "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}",

docs/malcolm-config.md

@@ -129,6 +129,7 @@ Although the configuration script automates many of the following configuration
     - `PCAP_IFACE_TWEAK` - if set to `true`, Malcolm will [use `ethtool`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/nic-capture-setup.sh) to disable NIC hardware offloading features and adjust ring buffer sizes for capture interface(s); this should be `true` if the interface(s) are being used for capture only, `false` if they are being used for management/communication
     - `PCAP_ROTATE_MEGABYTES` – used to specify how large a locally captured PCAP file can become (in megabytes) before it is closed for processing and a new PCAP file created
     - `PCAP_ROTATE_MINUTES` – used to specify a time interval (in minutes) after which a locally-captured PCAP file will be closed for processing and a new PCAP file created
+    - `PCAP_IFACE_STATS_CRON_EXPRESSION` - Specifies a [cron expression](https://en.wikipedia.org/wiki/Cron#CRON_expression) (using [`cronexpr`](https://github.com/aptible/supercronic/tree/master/cronexpr#implementation)-compatible syntax) indicating the refresh interval for collecting kernel-level statistics for network interfaces. An empty value for this variable means these statistics will not be generated.
 * **`postgres.env`** - Settings related to the PostgreSQL relational database
 * **`process.env`** - settings for how the processes running inside Malcolm containers are executed
     - `PUID` and `PGID` - Docker runs all its containers as the privileged `root` user by default. For better security, Malcolm immediately drops to non-privileged user accounts for executing internal processes wherever possible. The `PUID` (**p**rocess **u**ser **ID**) and `PGID` (**p**rocess **g**roup **ID**) environment variables allow Malcolm to map internal non-privileged user accounts to a corresponding [user account](https://en.wikipedia.org/wiki/User_identifier) on the host. Note a few (including the `logstash` and `netbox` containers) may take a few extra minutes during startup if `PUID` and `PGID` are set to values other than the default `1000`. This is expected and should not affect operation after the initial startup.

filebeat/filebeat-logs.yml

@@ -129,6 +129,38 @@ filebeat.inputs:
   close_eof: ${FILEBEAT_CLOSE_EOF:true}
   clean_removed: ${FILEBEAT_CLEAN_REMOVED:true}
 
+#-------------------------- network interface stats (as JSON) ---------------
+#                           (see netdev-json.sh)
+- type: log
+  paths:
+    - ${FILEBEAT_ZEEK_LOG_LIVE_PATH:/zeek/live}/netdev-stats*.json
+  symlinks: true
+  json:
+    expand_keys: false
+    add_error_key: false
+    ignore_decoding_error: true
+    keys_under_root: false
+    target: miscbeat.network
+  fields:
+    miscbeat.module: network
+  processors:
+    - rename:
+        fields:
+          - from: "fields.miscbeat.module"
+            to: "miscbeat.module"
+          - from: "json"
+            to: "miscbeat.network"
+  tags: ["_malcolm_beats", "_malcolm_miscbeat"]
+  compression_level: 0
+  scan_frequency: ${FILEBEAT_SCAN_FREQUENCY:10s}
+  clean_inactive: ${FILEBEAT_CLEAN_INACTIVE:180m}
+  ignore_older: ${FILEBEAT_IGNORE_OLDER:120m}
+  close_inactive: ${FILEBEAT_CLOSE_INACTIVE_LIVE:90m}
+  close_renamed: true
+  close_removed: true
+  close_eof: true
+  clean_removed: true
+
 #================================ Outputs ======================================
 
 #-------------------------- Logstash Output ------------------------------------

zeek/scripts/docker_entrypoint.sh

@@ -6,5 +6,16 @@ ZEEK_DIR=${ZEEK_DIR:-"/opt/zeek"}
 setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ZEEK_DIR}"/bin/zeek 2>/dev/null || true
 setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ZEEK_DIR}"/bin/capstats 2>/dev/null || true
 
+# generate packet statistics for network interfaces periodically
+if [[ -d "${ZEEK_LOG_PATH}" ]] && [[ -n "${SUPERCRONIC_CRONTAB}" ]]; then
+
+    touch "${SUPERCRONIC_CRONTAB}" 2>/dev/null || true
+    sed -i -e "/netdev-json\.sh/d" "${SUPERCRONIC_CRONTAB}"
+
+    [[ -n "${PCAP_IFACE_STATS_CRON_EXPRESSION}" ]] && \
+      echo "${PCAP_IFACE_STATS_CRON_EXPRESSION} /usr/local/bin/netdev-json.sh >\"${ZEEK_LOG_PATH}\"/netdev-stats.json.tmp 2>/dev/null && mv -f \"${ZEEK_LOG_PATH}\"/netdev-stats.json.tmp \"${ZEEK_LOG_PATH}\"/netdev-stats.json" \
+         >> "${SUPERCRONIC_CRONTAB}"
+fi
+
 # start supervisor (which will spawn pcap-zeek, cron, etc.) or whatever the default command is
 exec "$@"