standardization and fixes for named volumes

Author: mmguero

Dockerfiles/filebeat.Dockerfile

@@ -21,7 +21,7 @@ ENV PGROUP="filebeat"
 #   in the Dockerfile is getting set with an ownership of 999:999.
 #   This is to override that, although I'm not yet sure if there are
 #   other implications. See containers/podman#23347.
-ENV PUSER_CHOWN="/usr/share/filebeat-logs/data;/usr/share/filebeat-nginx/data;/usr/share/filebeat-tcp/data"
+ENV PUSER_CHOWN="/usr/share/filebeat-logs/data;/usr/share/filebeat-nginx/data;/usr/share/filebeat-syslog-tcp/data;/usr/share/filebeat-syslog-udp/data;/usr/share/filebeat-tcp/data;/usr/share/filebeat-zeek-files-logs/data"
 # not dropping privileges globally: supervisord will take care of it
 # on a case-by-case basis so that one script (filebeat-watch-zeeklogs-uploads-folder.py)
 # can chown uploaded files
@@ -198,7 +198,7 @@ ENV FILEBEAT_ZEEK_UPLOAD_SUBDIR="upload"
 ENV PCAP_NODE_NAME=$PCAP_NODE_NAME
 
 # see PUSER_CHOWN comment above
-VOLUME ["/usr/share/filebeat-logs/data", "/usr/share/filebeat-nginx/data", "/usr/share/filebeat-tcp/data"]
+VOLUME ["/usr/share/filebeat-logs/data", "/usr/share/filebeat-nginx/data", "/usr/share/filebeat-syslog-tcp/data", "/usr/share/filebeat-syslog-udp/data", "/usr/share/filebeat-tcp/data", "/usr/share/filebeat-zeek-files-logs/data"]
 
 ENTRYPOINT ["/usr/bin/tini", \
             "--", \

Dockerfiles/filescan.Dockerfile

@@ -19,6 +19,12 @@ ENV DEFAULT_UID=$DEFAULT_UID
 ENV DEFAULT_GID=$DEFAULT_GID
 ENV PUSER="scan"
 ENV PGROUP="scan"
+# This is to handle an issue when running with rootless podman and
+#   "userns_mode: keep-id". It seems that anything defined as a VOLUME
+#   in the Dockerfile is getting set with an ownership of 999:999.
+#   This is to override that, although I'm not yet sure if there are
+#   other implications. See containers/podman#23347.
+ENV PUSER_CHOWN="/filescan/data"
 ENV PUSER_PRIV_DROP=true
 ENV PUSER_RLIMIT_UNLOCK=true
 USER root
@@ -159,6 +165,7 @@ ADD --chmod=644 shared/bin/watch_common.py /usr/local/bin/
 
 ################################################################################
 
+# see PUSER_CHOWN comment above
 VOLUME ["/filescan/data"]
 
 EXPOSE $FILESCAN_HTTP_SERVER_PORT

Dockerfiles/logstash.Dockerfile

@@ -33,9 +33,14 @@ ENV DEFAULT_UID=$DEFAULT_UID
 ENV DEFAULT_GID=$DEFAULT_GID
 ENV PUSER="logstash"
 ENV PGROUP="logstash"
+# This is to handle an issue when running with rootless podman and
+#   "userns_mode: keep-id". It seems that anything defined as a VOLUME
+#   in the Dockerfile is getting set with an ownership of 999:999.
+#   This is to override that, although I'm not yet sure if there are
+#   other implications. See containers/podman#23347.
+ENV PUSER_CHOWN="/logstash-persistent-queue"
 ENV PUSER_PRIV_DROP=true
 ENV PUSER_RLIMIT_UNLOCK=true
-ENV PUSER_CHOWN="/logstash-persistent-queue"
 USER root
 
 ENV TERM=xterm
@@ -127,6 +132,8 @@ ENV LOGSTASH_OPENSEARCH_PIPELINE_ADDRESS_EXTERNAL=$LOGSTASH_OPENSEARCH_PIPELINE_
 ENV LOGSTASH_OPENSEARCH_OUTPUT_PIPELINE_ADDRESSES=$LOGSTASH_OPENSEARCH_OUTPUT_PIPELINE_ADDRESSES
 
 ENV LOGSTASH_KEYSTORE_PASS="a410a267b1404c949284dee25518a917"
+
+# see PUSER_CHOWN comment above
 VOLUME ["/logstash-persistent-queue"]
 
 EXPOSE 5044 9001 9600

Dockerfiles/strelka-backend.Dockerfile

@@ -41,7 +41,7 @@ ENV YARA_COMPILED_RULES_FILE="rules.compiled"
 ARG EXTRACTED_FILE_MAX_BYTES=134217728
 ENV EXTRACTED_FILE_MAX_BYTES=$EXTRACTED_FILE_MAX_BYTES
 ENV CLAMD_SOCKET_FILE=/tmp/clamd.ctl
-ENV CLAMAV_RULES_DIR "/var/lib/clamav"
+ENV CLAMAV_RULES_DIR="/var/lib/clamav"
 
 ARG STRELKA_BACKEND_PROCS=1
 ENV STRELKA_BACKEND_PROCS=$STRELKA_BACKEND_PROCS
@@ -110,9 +110,7 @@ ADD --chmod=755 container-health-scripts/strelka-backend.sh /usr/local/bin/conta
 ENV PUSER_CHOWN="$CLAMAV_RULES_DIR;$YARA_RULES_DIR;$YARA_RULES_SRC_DIR"
 
 # see PUSER_CHOWN comment above
-VOLUME ["$CLAMAV_RULES_DIR"]
-VOLUME ["$YARA_RULES_DIR"]
-VOLUME ["$YARA_RULES_SRC_DIR"]
+VOLUME ["$CLAMAV_RULES_DIR", "$YARA_RULES_DIR", "$YARA_RULES_SRC_DIR"]
 
 ENTRYPOINT ["/usr/bin/tini", \
             "--", \

Dockerfiles/strelka-frontend.Dockerfile

@@ -42,6 +42,13 @@ ADD --chmod=755 container-health-scripts/strelka-frontend.sh /usr/local/bin/cont
 
 EXPOSE 57314
 
+# This is to handle an issue when running with rootless podman and
+#   "userns_mode: keep-id". It seems that anything defined as a VOLUME
+#   in the Dockerfile is getting set with an ownership of 999:999.
+#   This is to override that, although I'm not yet sure if there are
+#   other implications. See containers/podman#23347.
+ENV PUSER_CHOWN="/var/log/strelka"
+
 VOLUME [ "/var/log/strelka" ]
 
 ENTRYPOINT ["/sbin/tini", \

docker-compose-dev.yml

@@ -106,6 +106,7 @@ services:
     depends_on:
     - opensearch
     volumes:
+    - dashboards-helper-data-init:/data/init
     - type: bind
       bind:
         create_host_path: false
@@ -218,6 +219,7 @@ services:
     ports:
     - 127.0.0.1:5044:5044
     volumes:
+    - logstash-persistent-queue:/logstash-persistent-queue
     - type: bind
       bind:
         create_host_path: false
@@ -323,6 +325,12 @@ services:
     - 127.0.0.1:5045:5045
     volumes:
     - nginx-log-path:/nginx:ro
+    - filebeat-logs-registry:/usr/share/filebeat-logs/data
+    - filebeat-nginx-registry:/usr/share/filebeat-nginx/data
+    - filebeat-syslog-tcp-registry:/usr/share/filebeat-syslog-tcp/data
+    - filebeat-syslog-udp-registry:/usr/share/filebeat-syslog-udp/data
+    - filebeat-tcp-registry:/usr/share/filebeat-tcp/data
+    - filebeat-zeek-files-logs-registry:/usr/share/filebeat-zeek-files-logs/data
     - type: bind
       bind:
         create_host_path: false
@@ -723,6 +731,9 @@ services:
     - ./config/suricata.env
     - ./config/suricata-offline.env
     volumes:
+    - suricata-config:/etc/suricata
+    - suricata-managed:/var/lib/suricata
+    - suricata-run:/var/run/suricata
     - type: bind
       bind:
         create_host_path: false
@@ -791,6 +802,9 @@ services:
     - ./config/suricata.env
     - ./config/suricata-live.env
     volumes:
+    - suricata-live-config:/etc/suricata
+    - suricata-live-managed:/var/lib/suricata
+    - suricata-live-run:/var/run/suricata
     - type: bind
       bind:
         create_host_path: false
@@ -849,6 +863,7 @@ services:
     - ./config/filescan-secret.env
     - ./config/filescan.env
     volumes:
+    - filescan-data:/filescan/data
     - type: bind
       bind:
         create_host_path: false
@@ -904,6 +919,9 @@ services:
     - ./config/pipeline.env
     shm_size: 512mb
     volumes:
+    - strelka-backend-clamav-rules:/var/lib/clamav
+    - strelka-backend-yara-rules-src:/yara-rules-src
+    - strelka-backend-yara-rules:/yara-rules
     - type: bind
       bind:
         create_host_path: false
@@ -954,6 +972,7 @@ services:
     - ./config/redis.env
     - ./config/pipeline.env
     volumes:
+    - strelka-frontend-logs:/var/log/strelka
     - type: bind
       bind:
         create_host_path: false
@@ -966,7 +985,6 @@ services:
       source: ./strelka/config/frontend/
       target: /etc/strelka/configmap
       read_only: true
-    - strelka-logs:/var/log/strelka/
     healthcheck:
       test: ["CMD", "/usr/local/bin/container_health.sh"]
       interval: 30s
@@ -1611,9 +1629,26 @@ services:
       traefik.enable: false
 
 volumes:
-  # shared named volume so filebeat can access nginx access logs
+  dashboards-helper-data-init:
+  filebeat-logs-registry:
+  filebeat-nginx-registry:
+  filebeat-syslog-tcp-registry:
+  filebeat-syslog-udp-registry:
+  filebeat-tcp-registry:
+  filebeat-zeek-files-logs-registry:
+  filescan-data:
+  logstash-persistent-queue:
   nginx-log-path:
-  strelka-logs:
+  strelka-backend-clamav-rules:
+  strelka-backend-yara-rules-src:
+  strelka-backend-yara-rules:
+  strelka-frontend-logs:
+  suricata-config:
+  suricata-live-config:
+  suricata-live-managed:
+  suricata-live-run:
+  suricata-managed:
+  suricata-run:
 
 networks:
   default:

docker-compose.yml

@@ -100,6 +100,7 @@ services:
     depends_on:
     - opensearch
     volumes:
+    - dashboards-helper-data-init:/data/init
     - type: bind
       bind:
         create_host_path: false
@@ -206,6 +207,7 @@ services:
     ports:
     - 127.0.0.1:5044:5044
     volumes:
+    - logstash-persistent-queue:/logstash-persistent-queue
     - type: bind
       bind:
         create_host_path: false
@@ -284,6 +286,12 @@ services:
     - 127.0.0.1:5045:5045
     volumes:
     - nginx-log-path:/nginx:ro
+    - filebeat-logs-registry:/usr/share/filebeat-logs/data
+    - filebeat-nginx-registry:/usr/share/filebeat-nginx/data
+    - filebeat-syslog-tcp-registry:/usr/share/filebeat-syslog-tcp/data
+    - filebeat-syslog-udp-registry:/usr/share/filebeat-syslog-udp/data
+    - filebeat-tcp-registry:/usr/share/filebeat-tcp/data
+    - filebeat-zeek-files-logs-registry:/usr/share/filebeat-zeek-files-logs/data
     - type: bind
       bind:
         create_host_path: false
@@ -621,6 +629,9 @@ services:
     - ./config/suricata.env
     - ./config/suricata-offline.env
     volumes:
+    - suricata-config:/etc/suricata
+    - suricata-managed:/var/lib/suricata
+    - suricata-run:/var/run/suricata
     - type: bind
       bind:
         create_host_path: false
@@ -686,6 +697,9 @@ services:
     - ./config/suricata.env
     - ./config/suricata-live.env
     volumes:
+    - suricata-live-config:/etc/suricata
+    - suricata-live-managed:/var/lib/suricata
+    - suricata-live-run:/var/run/suricata
     - type: bind
       bind:
         create_host_path: false
@@ -741,6 +755,7 @@ services:
     - ./config/filescan-secret.env
     - ./config/filescan.env
     volumes:
+    - filescan-data:/filescan/data
     - type: bind
       bind:
         create_host_path: false
@@ -793,6 +808,9 @@ services:
     - ./config/pipeline.env
     shm_size: 512mb
     volumes:
+    - strelka-backend-clamav-rules:/var/lib/clamav
+    - strelka-backend-yara-rules-src:/yara-rules-src
+    - strelka-backend-yara-rules:/yara-rules
     - type: bind
       bind:
         create_host_path: false
@@ -840,6 +858,7 @@ services:
     - ./config/redis.env
     - ./config/pipeline.env
     volumes:
+    - strelka-frontend-logs:/var/log/strelka
     - type: bind
       bind:
         create_host_path: false
@@ -852,7 +871,6 @@ services:
       source: ./strelka/config/frontend/
       target: /etc/strelka/configmap
       read_only: true
-    - strelka-logs:/var/log/strelka/
     healthcheck:
       test: ["CMD", "/usr/local/bin/container_health.sh"]
       interval: 30s
@@ -1458,9 +1476,26 @@ services:
       traefik.enable: false
 
 volumes:
-  # shared named volume so filebeat can access nginx access logs
+  dashboards-helper-data-init:
+  filebeat-logs-registry:
+  filebeat-nginx-registry:
+  filebeat-syslog-tcp-registry:
+  filebeat-syslog-udp-registry:
+  filebeat-tcp-registry:
+  filebeat-zeek-files-logs-registry:
+  filescan-data:
+  logstash-persistent-queue:
   nginx-log-path:
-  strelka-logs:
+  strelka-backend-clamav-rules:
+  strelka-backend-yara-rules-src:
+  strelka-backend-yara-rules:
+  strelka-frontend-logs:
+  suricata-config:
+  suricata-live-config:
+  suricata-live-managed:
+  suricata-live-run:
+  suricata-managed:
+  suricata-run:
 
 networks:
   default: