Fix dashboards export, and don't explicitly include arkime_stats-v3 index pattern (the index template will handle its automatic creation by arkime)

mmguero · mmguero · commit ffbaa062cfc6 · 2025-09-04T19:48:37.000Z
diff --git a/api/project/__init__.py b/api/project/__init__.py
@@ -1297,6 +1297,7 @@ def dashboard_export(dashid):
                                         if doReplacers
                                         else app.config['MALCOLM_OTHER_INDEX_PATTERN']
                                     ),
+                                    app.config['ARKIME_STATS_INDEX_PATTERN'],
                                 ]
                             )
                         )
diff --git a/api/project/config.py b/api/project/config.py
@@ -5,48 +5,49 @@
 
 
 class Config(object):
-    ARKIME_FIELDS_INDEX = f"{os.getenv('ARKIME_FIELDS_INDEX', 'arkime_fields')}"
-    MALCOLM_NETWORK_INDEX_PATTERN = f"{os.getenv('MALCOLM_NETWORK_INDEX_PATTERN', 'arkime_sessions3-*')}"
-    MALCOLM_NETWORK_INDEX_TIME_FIELD = f"{os.getenv('MALCOLM_NETWORK_INDEX_TIME_FIELD', 'firstPacket')}"
-    MALCOLM_OTHER_INDEX_PATTERN = f"{os.getenv('MALCOLM_OTHER_INDEX_PATTERN', 'malcolm_beats_*')}"
-    MALCOLM_OTHER_INDEX_TIME_FIELD = f"{os.getenv('MALCOLM_OTHER_INDEX_TIME_FIELD', '@timestamp')}"
-    ARKIME_NETWORK_INDEX_PATTERN = f"{os.getenv('ARKIME_NETWORK_INDEX_PATTERN', 'arkime_sessions3-*')}"
-    ARKIME_NETWORK_INDEX_TIME_FIELD = f"{os.getenv('ARKIME_NETWORK_INDEX_TIME_FIELD', 'firstPacket')}"
+    ARKIME_FIELDS_INDEX = os.getenv('ARKIME_FIELDS_INDEX', 'arkime_fields')
+    MALCOLM_NETWORK_INDEX_PATTERN = os.getenv('MALCOLM_NETWORK_INDEX_PATTERN', 'arkime_sessions3-*')
+    MALCOLM_NETWORK_INDEX_TIME_FIELD = os.getenv('MALCOLM_NETWORK_INDEX_TIME_FIELD', 'firstPacket')
+    MALCOLM_OTHER_INDEX_PATTERN = os.getenv('MALCOLM_OTHER_INDEX_PATTERN', 'malcolm_beats_*')
+    MALCOLM_OTHER_INDEX_TIME_FIELD = os.getenv('MALCOLM_OTHER_INDEX_TIME_FIELD', '@timestamp')
+    ARKIME_NETWORK_INDEX_PATTERN = os.getenv('ARKIME_NETWORK_INDEX_PATTERN', 'arkime_sessions3-*')
+    ARKIME_NETWORK_INDEX_TIME_FIELD = os.getenv('ARKIME_NETWORK_INDEX_TIME_FIELD', 'firstPacket')
+    ARKIME_STATS_INDEX_PATTERN = "arkime_stats_*"
 
-    ARKIME_SSL = f"{os.getenv('ARKIME_SSL', 'true')}"
-    ARKIME_HOST = f"{os.getenv('ARKIME_HOST', 'arkime')}"
-    ARKIME_PORT = int(f"{os.getenv('ARKIME_VIEWER_PORT', os.getenv('ARKIME_PORT', '8005'))}".split(':')[-1])
-    BUILD_DATE = f"{os.getenv('BUILD_DATE', 'unknown')}"
-    DASHBOARDS_URL = f"{os.getenv('DASHBOARDS_URL', 'http://dashboards:5601/dashboards')}"
-    DASHBOARDS_HELPER_HOST = f"{os.getenv('DASHBOARDS_HELPER_HOST', 'dashboards-helper')}"
-    DASHBOARDS_MAPS_PORT = int(f"{os.getenv('DASHBOARDS_MAPS_PORT', '28991')}")
-    DOCTYPE_DEFAULT = f"{os.getenv('DOCTYPE_DEFAULT', 'network')}"
-    FILEBEAT_HOST = f"{os.getenv('FILEBEAT_HOST', 'filebeat')}"
-    FILEBEAT_TCP_JSON_PORT = int(f"{os.getenv('FILEBEAT_TCP_JSON_PORT', '5045')}")
-    FREQ_URL = f"{os.getenv('FREQ_URL', 'http://freq:10004')}"
-    LOGSTASH_API_PORT = int(f"{os.getenv('LOGSTASH_API_PORT', '9600')}")
-    LOGSTASH_HOST = f"{os.getenv('LOGSTASH_HOST', 'logstash')}"
-    LOGSTASH_LJ_PORT = int(f"{os.getenv('LOGSTASH_LJ_PORT', '5044')}")
-    MALCOLM_API_DEBUG = f"{os.getenv('MALCOLM_API_DEBUG', 'false')}"
-    MALCOLM_API_PREFIX = f"{os.getenv('MALCOLM_API_PREFIX', 'mapi')}"
-    MALCOLM_TEMPLATE = f"{os.getenv('MALCOLM_TEMPLATE', 'malcolm_template')}"
-    MALCOLM_VERSION = f"{os.getenv('MALCOLM_VERSION', 'unknown')}"
+    ARKIME_SSL = os.getenv('ARKIME_SSL', 'true')
+    ARKIME_HOST = os.getenv('ARKIME_HOST', 'arkime')
+    ARKIME_PORT = int(os.getenv('ARKIME_VIEWER_PORT', os.getenv('ARKIME_PORT', '8005')).split(':')[-1])
+    BUILD_DATE = os.getenv('BUILD_DATE', 'unknown')
+    DASHBOARDS_URL = os.getenv('DASHBOARDS_URL', 'http://dashboards:5601/dashboards')
+    DASHBOARDS_HELPER_HOST = os.getenv('DASHBOARDS_HELPER_HOST', 'dashboards-helper')
+    DASHBOARDS_MAPS_PORT = int(os.getenv('DASHBOARDS_MAPS_PORT', '28991'))
+    DOCTYPE_DEFAULT = os.getenv('DOCTYPE_DEFAULT', 'network')
+    FILEBEAT_HOST = os.getenv('FILEBEAT_HOST', 'filebeat')
+    FILEBEAT_TCP_JSON_PORT = int(os.getenv('FILEBEAT_TCP_JSON_PORT', '5045'))
+    FREQ_URL = os.getenv('FREQ_URL', 'http://freq:10004')
+    LOGSTASH_API_PORT = int(os.getenv('LOGSTASH_API_PORT', '9600'))
+    LOGSTASH_HOST = os.getenv('LOGSTASH_HOST', 'logstash')
+    LOGSTASH_LJ_PORT = int(os.getenv('LOGSTASH_LJ_PORT', '5044'))
+    MALCOLM_API_DEBUG = os.getenv('MALCOLM_API_DEBUG', 'false')
+    MALCOLM_API_PREFIX = os.getenv('MALCOLM_API_PREFIX', 'mapi')
+    MALCOLM_TEMPLATE = os.getenv('MALCOLM_TEMPLATE', 'malcolm_template')
+    MALCOLM_VERSION = os.getenv('MALCOLM_VERSION', 'unknown')
     NETBOX_URL = os.getenv('NETBOX_URL') or 'http://netbox:8080/netbox'
-    NETBOX_TOKEN = f"{os.getenv('NETBOX_TOKEN') or os.getenv('SUPERUSER_API_TOKEN', '')}"
-    OPENSEARCH_CREDS_CONFIG_FILE = (
-        f"{os.getenv('OPENSEARCH_CREDS_CONFIG_FILE', '/var/local/curlrc/.opensearch.primary.curlrc')}"
+    NETBOX_TOKEN = os.getenv('NETBOX_TOKEN') or os.getenv('SUPERUSER_API_TOKEN', '')
+    OPENSEARCH_CREDS_CONFIG_FILE = os.getenv(
+        'OPENSEARCH_CREDS_CONFIG_FILE', '/var/local/curlrc/.opensearch.primary.curlrc'
     )
-    OPENSEARCH_PRIMARY = f"{os.getenv('OPENSEARCH_PRIMARY', 'opensearch-local')}"
-    OPENSEARCH_SSL_CERTIFICATE_VERIFICATION = f"{os.getenv('OPENSEARCH_SSL_CERTIFICATE_VERIFICATION', 'false')}"
-    OPENSEARCH_URL = f"{os.getenv('OPENSEARCH_URL', 'https://opensearch:9200')}"
-    PCAP_MONITOR_HOST = f"{os.getenv('PCAP_MONITOR_HOST', 'pcap-monitor')}"
-    PCAP_TOPIC_PORT = int(f"{os.getenv('PCAP_TOPIC_PORT', '30441')}")
-    RESULT_SET_LIMIT = int(f"{os.getenv('RESULT_SET_LIMIT', '500')}")
-    VCS_REVISION = f"{os.getenv('VCS_REVISION', 'unknown')}"
-    ZEEK_EXTRACTED_FILE_LOGGER_HOST = f"{os.getenv('ZEEK_EXTRACTED_FILE_LOGGER_HOST', 'file-monitor')}"
-    ZEEK_EXTRACTED_FILE_LOGGER_TOPIC_PORT = int(f"{os.getenv('ZEEK_EXTRACTED_FILE_LOGGER_TOPIC_PORT', '5988')}")
-    ZEEK_EXTRACTED_FILE_MONITOR_HOST = f"{os.getenv('ZEEK_EXTRACTED_FILE_MONITOR_HOST', 'file-monitor')}"
-    ZEEK_EXTRACTED_FILE_TOPIC_PORT = int(f"{os.getenv('ZEEK_EXTRACTED_FILE_TOPIC_PORT', '5987')}")
+    OPENSEARCH_PRIMARY = os.getenv('OPENSEARCH_PRIMARY', 'opensearch-local')
+    OPENSEARCH_SSL_CERTIFICATE_VERIFICATION = os.getenv('OPENSEARCH_SSL_CERTIFICATE_VERIFICATION', 'false')
+    OPENSEARCH_URL = os.getenv('OPENSEARCH_URL', 'https://opensearch:9200')
+    PCAP_MONITOR_HOST = os.getenv('PCAP_MONITOR_HOST', 'pcap-monitor')
+    PCAP_TOPIC_PORT = int(os.getenv('PCAP_TOPIC_PORT', '30441'))
+    RESULT_SET_LIMIT = int(os.getenv('RESULT_SET_LIMIT', '500'))
+    VCS_REVISION = os.getenv('VCS_REVISION', 'unknown')
+    ZEEK_EXTRACTED_FILE_LOGGER_HOST = os.getenv('ZEEK_EXTRACTED_FILE_LOGGER_HOST', 'file-monitor')
+    ZEEK_EXTRACTED_FILE_LOGGER_TOPIC_PORT = int(os.getenv('ZEEK_EXTRACTED_FILE_LOGGER_TOPIC_PORT', '5988'))
+    ZEEK_EXTRACTED_FILE_MONITOR_HOST = os.getenv('ZEEK_EXTRACTED_FILE_MONITOR_HOST', 'file-monitor')
+    ZEEK_EXTRACTED_FILE_TOPIC_PORT = int(os.getenv('ZEEK_EXTRACTED_FILE_TOPIC_PORT', '5987'))
 
     ROLE_BASED_ACCESS = os.getenv('ROLE_BASED_ACCESS', 'false')
     ROLE_ADMIN = os.getenv('ROLE_ADMIN', '')
diff --git a/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json b/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json
@@ -614,24 +614,6 @@
       "type": "search",
       "updated_at": "2025-08-25T14:54:49.880Z",
       "version": "WzEwMzIsMV0="
-    },
-    {
-      "attributes": {
-        "fields": "[{\"count\":0,\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_score\",\"type\":\"number\",\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_type\",\"type\":\"string\",\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"closeQueue\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"cpu\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":1,\"name\":\"currentTime\",\"type\":\"date\",\"esTypes\":[\"date\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"deltaBytes\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"deltaDropped\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"deltaDupDropped\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"deltaESDropped\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"deltaFragsDropped\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"deltaMS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"deltaOverloadDropped\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"deltaPackets\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"deltaSessionBytes\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"deltaSessions\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"deltaUnwrittenBytes\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"deltaWrittenBytes\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"diskQueue\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"esHealthMS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"esQueue\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"espSessions\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"frags\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"fragsQueue\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"freeSpaceM\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"freeSpaceP\",\"type\":\"number\",\"esTypes\":[\"float\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":1,\"name\":\"hostname\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"icmpSessions\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"interval\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"memory\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"memoryP\",\"type\":\"number\",\"esTypes\":[\"float\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"monitoring\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"needSave\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":1,\"name\":\"nodeName\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"otherSessions\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"packetQueue\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"sctpSessions\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"startTime\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"tcpSessions\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"totalDropped\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":1,\"name\":\"totalK\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":1,\"name\":\"totalPackets\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":1,\"name\":\"totalSessions\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"udpSessions\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"usedSpaceM\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"ver\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"ver.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"ver\"}}}]",
-        "timeFieldName": null,
-        "title": "arkime_stats_*"
-      },
-      "id": "arkime_stats_*",
-      "migrationVersion": {
-        "index-pattern": "7.6.0"
-      },
-      "namespaces": [
-        "default"
-      ],
-      "references": [],
-      "type": "index-pattern",
-      "updated_at": "2025-08-26T15:29:11.296Z",
-      "version": "WzExNjYsNV0="
     }
   ],
   "version": "3.1.0"

Original file line number	Diff line number	Diff line change
`@@ -1297,6 +1297,7 @@ def dashboard_export(dashid):`
`1297`	`1297`	`if doReplacers`
`1298`	`1298`	`else app.config['MALCOLM_OTHER_INDEX_PATTERN']`
`1299`	`1299`	`),`
	`1300`	`+ app.config['ARKIME_STATS_INDEX_PATTERN'],`
`1300`	`1301`	`]`
`1301`	`1302`	`)`
`1302`	`1303`	`)`