From 555384b354f71cd1dc6b66b6fbc081fb574a055d Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Tue, 5 Nov 2024 20:22:31 -0700
Subject: [PATCH 01/42] bump opensearch and dashboards to v2.18.0

---
 Dockerfiles/dashboards.Dockerfile | 10 +++++-----
 Dockerfiles/opensearch.Dockerfile |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/Dockerfiles/dashboards.Dockerfile b/Dockerfiles/dashboards.Dockerfile
index 6a82cfe9f..3bfb5f610 100644
--- a/Dockerfiles/dashboards.Dockerfile
+++ b/Dockerfiles/dashboards.Dockerfile
@@ -1,4 +1,4 @@
-FROM opensearchproject/opensearch-dashboards:2.17.1
+FROM opensearchproject/opensearch-dashboards:2.18.0
 
 LABEL maintainer="malcolm@inl.gov"
 LABEL org.opencontainers.image.authors='malcolm@inl.gov'
@@ -42,10 +42,10 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/')
     # Malcolm manages authentication and encryption via NGINX reverse proxy
     /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove securityDashboards --allow-root && \
     cd /tmp && \
-        # unzip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \
-        # sed -i "s/2\.16\.0/2\.17\.0/g" opensearch-dashboards/transformVis/opensearch_dashboards.json && \
-        # sed -i "s/2\.16\.0/2\.17\.0/g" opensearch-dashboards/transformVis/package.json && \
-        # zip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \
+        unzip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \
+        sed -i "s/2\.17\.1/2\.18\.0/g" opensearch-dashboards/transformVis/opensearch_dashboards.json && \
+        sed -i "s/2\.17\.1/2\.18\.0/g" opensearch-dashboards/transformVis/package.json && \
+        zip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \
         cd /usr/share/opensearch-dashboards/plugins && \
         /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install file:///tmp/transformVis.zip --allow-root && \
         rm -rf /tmp/transformVis /tmp/opensearch-dashboards && \
diff --git a/Dockerfiles/opensearch.Dockerfile b/Dockerfiles/opensearch.Dockerfile
index 2daa274a5..84264336d 100644
--- a/Dockerfiles/opensearch.Dockerfile
+++ b/Dockerfiles/opensearch.Dockerfile
@@ -1,4 +1,4 @@
-FROM opensearchproject/opensearch:2.17.1
+FROM opensearchproject/opensearch:2.18.0
 
 # Copyright (c) 2024 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="malcolm@inl.gov"

From c02b1b8fd234f2c7e16059a3ad74dfb852d71d14 Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Wed, 6 Nov 2024 10:49:26 -0700
Subject: [PATCH 02/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358

---
 Dockerfiles/zeek.Dockerfile               |   1 +
 docs/zeek-intel.md                        |  33 +++-
 shared/bin/zeek_intel_from_threat_feed.py |  25 ++-
 shared/bin/zeek_intel_setup.sh            |   6 +-
 shared/bin/zeek_threat_feed_utils.py      | 206 +++++++++++++++++++++-
 5 files changed, 259 insertions(+), 12 deletions(-)

diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile
index cba872952..6745f62db 100644
--- a/Dockerfiles/zeek.Dockerfile
+++ b/Dockerfiles/zeek.Dockerfile
@@ -108,6 +108,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/')
       python3-setuptools \
       python3-tz \
       python3-wheel \
+      python3-yaml \
       python3-zmq \
       rsync \
       supervisor \
diff --git a/docs/zeek-intel.md b/docs/zeek-intel.md
index 7d00bb1c2..468f9529c 100644
--- a/docs/zeek-intel.md
+++ b/docs/zeek-intel.md
@@ -3,6 +3,8 @@
 * [Zeek Intelligence Framework](#ZeekIntel)
     - [STIX™ and TAXII™](#ZeekIntelSTIX)
     - [MISP](#ZeekIntelMISP)
+    - [Mandiant](#ZeekIntelMandiant)
+    - [Endorsement Disclaimer](#IntelFeedDisclaimer)
 
 To quote Zeek's [Intelligence Framework](https://docs.zeek.org/en/master/frameworks/intel.html) documentation, "The goals of Zeek’s Intelligence Framework are to consume intelligence data, make it available for matching, and provide infrastructure to improve performance and memory utilization. Data in the Intelligence Framework is an atomic piece of intelligence such as an IP address or an e-mail address. This atomic data will be packed with metadata such as a freeform source field, a freeform descriptive field, and a URL which might lead to more information about the specific item." Zeek [intelligence](https://docs.zeek.org/en/master/scripts/base/frameworks/intel/main.zeek.html) [indicator types](https://docs.zeek.org/en/master/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Type) include IP addresses, URLs, file names, hashes, email addresses, and more.
 
@@ -42,6 +44,8 @@ Malcolm will attempt to query the TAXII feed(s) for `indicator` STIX objects and
 
 Note that only **indicators** of [**cyber-observable objects**](https://docs.oasis-open.org/cti/stix/v2.1/cs01/stix-v2.1-cs01.html#_mlbmudhl16lr) matched with the **equals (`=`)** [comparison operator](https://docs.oasis-open.org/cti/stix/v2.1/cs01/stix-v2.1-cs01.html#_t11hn314cr7w) against a **single value** can be expressed as Zeek intelligence items. More complex STIX indicators will be silently ignored.
 
+Malcolm uses the [stix2](https://pypi.org/project/stix2/) and [taxii2-client](https://pypi.org/project/taxii2-client/) Python libraries to access STIX™/TAXII™ threat intelligence feeds.
+
 ## <a name="ZeekIntelMISP"></a>MISP
 
 In addition to loading Zeek intelligence files on startup, Malcolm will [automatically generate]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/zeek_intel_from_threat_feed.py) a Zeek intelligence file for all [Malware Information Sharing Platform (MISP)](https://www.misp-project.org/datamodels/) JSON files found under `./zeek/intel/MISP`.
@@ -72,4 +76,31 @@ Upon Malcolm connects to the URLs for the MISP feeds in `.misp_input.txt`, it wi
 * a list of [Events](https://www.misp-project.org/openapi/#tag/Events) returned for a request via the [MISP Automation API](https://www.misp-project.org/openapi/) made to a MISP platform's [`/events` endpoint](https://www.misp-project.org/openapi/#tag/Events/operation/restSearchEvents)
 * a list of [Attributes](https://www.misp-project.org/openapi/#tag/Attributes) returned for a request via the [MISP Automation API](https://www.misp-project.org/openapi/) made to a MISP platform's [`/attributes` endpoint](https://www.misp-project.org/openapi/#tag/Attributes/operation/restSearchAttributes)
 
-Note that only a subset of MISP [attribute types](https://www.misp-project.org/datamodels/#attribute-categories-vs-types) can be expressed with the Zeek intelligence [indicator types](https://docs.zeek.org/en/master/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Type). MISP attributes with other types will be silently ignored.
\ No newline at end of file
+Note that only a subset of MISP [attribute types](https://www.misp-project.org/datamodels/#attribute-categories-vs-types) can be expressed with the Zeek intelligence [indicator types](https://docs.zeek.org/en/master/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Type). MISP attributes with other types will be silently ignored.
+
+Malcolm uses the [MISP/PyMISP](https://github.com/MISP/PyMISP) Python library to access MISP threat intelligence feeds.
+
+## <a name="ZeekIntelMandiant"></a>Mandiant
+
+If a [YAML](https://yaml.org/) file named `mandiant.yaml` is found in `./zeek/intel/Mandiant`, that file will be read and processed as parameters for the [Mandiant Threat Intelligence](https://www.mandiant.com/threats) service. This file should minimally include:
+
+```yaml
+type: mandiant
+api_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+secret_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+```
+
+These other parameters can also optionally be provided:
+
+```yaml
+minimum_mscore: 60
+exclude_osint: False
+bearer_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+api_base_url: https://api.intelligence.mandiant.com
+```
+
+Malcolm uses the [google/mandiant-ti-client](https://github.com/google/mandiant-ti-client) Python library to access Mandiant threat intelligence feeds.
+
+## <a name="IntelFeedDisclaimer"></a>Disclaimer
+
+Neither Malcolm's development team nor its funding sources endorse any commercial product or service, nor do they attest to the suitability or effectiveness of these products and services for any particular use case. Any reference to specific commercial products, processes, or services by trademark, manufacturer, or otherwise should not be interpreted as an endorsement, recommendation, or preference.
\ No newline at end of file
diff --git a/shared/bin/zeek_intel_from_threat_feed.py b/shared/bin/zeek_intel_from_threat_feed.py
index ca6153f1f..7fbf76fa6 100755
--- a/shared/bin/zeek_intel_from_threat_feed.py
+++ b/shared/bin/zeek_intel_from_threat_feed.py
@@ -12,7 +12,9 @@
 import argparse
 import logging
 import os
+import re
 import sys
+import yaml
 import zeek_threat_feed_utils
 
 import malcolm_utils
@@ -103,7 +105,7 @@ def main():
         nargs='*',
         type=str,
         default=None,
-        help="Read --input arguments from a local or external file (one per line)",
+        help="Read --input arguments from a local or external file (one per line, or YAML definitions)",
     )
     parser.add_argument(
         '-o',
@@ -148,6 +150,7 @@ def main():
 
     if args.input is None:
         args.input = []
+    yamlInputs = []
     since = (
         ParseDate(args.since).astimezone(UTCTimeZone) if (args.since is not None) and (len(args.since) > 0) else None
     )
@@ -164,9 +167,15 @@ def main():
             for infileArg in args.inputFile:
                 try:
                     if os.path.isfile(infileArg):
-                        # read inputs from local file
-                        with open(infileArg) as f:
-                            args.input.extend(f.read().splitlines())
+                        # read inputs from local file (delimited lines or YAML file)
+                        infileParts = os.path.splitext(infileArg)
+                        if re.search(r"\.ya?ml$", infileParts[1], re.IGNORECASE):
+                            with open(infileArg, 'r') as f:
+                                inputParams = yaml.safe_load(f)
+                            yamlInputs.append(inputParams)
+                        else:
+                            with open(infileArg) as f:
+                                args.input.extend(f.read().splitlines())
 
                     elif '://' in infileArg:
                         # download from URL and read input from remote file
@@ -189,11 +198,15 @@ def main():
         # deduplicate input sources
         seenInput = {}
         args.input = [seenInput.setdefault(x, x) for x in args.input if x not in seenInput]
-        logging.debug(f"Input: {args.input}")
 
         # we'll queue and then process all of the input arguments in workers
         inputQueue = deque()
-        inputQueue.extend(args.input)
+        if args.input:
+            inputQueue.extend(args.input)
+        if yamlInputs:
+            inputQueue.extend(yamlInputs)
+        logging.debug(f"Inputs: {list(inputQueue)}")
+
         workerThreadCount = malcolm_utils.AtomicInt(value=0)
         ThreadPool(
             args.threads,
diff --git a/shared/bin/zeek_intel_setup.sh b/shared/bin/zeek_intel_setup.sh
index f15e4e8d3..0f11ebc21 100755
--- a/shared/bin/zeek_intel_setup.sh
+++ b/shared/bin/zeek_intel_setup.sh
@@ -47,7 +47,7 @@ if mkdir -- "$LOCK_DIR" 2>/dev/null; then
         done < <(echo "${CONFIG_MAP_DIR:-configmap;secretmap}" | tr ';' '\n')
 
         rsync --recursive --delete --delete-excluded "${EXCLUDES[@]}" "${INTEL_PRESEED_DIR}"/ "${INTEL_DIR}"/
-        mkdir -p "${INTEL_DIR}"/MISP "${INTEL_DIR}"/STIX || true
+        mkdir -p "${INTEL_DIR}"/MISP "${INTEL_DIR}"/STIX "${INTEL_DIR}"/Mandiant || true
     fi
 
     # create directive to @load every subdirectory in /opt/zeek/share/zeek/site/intel
@@ -95,14 +95,14 @@ EOF
         done
 
         # process STIX and MISP inputs by converting them to Zeek intel format
-        if ( (( ${#THREAT_JSON_FILES[@]} )) || [[ -r ./STIX/.stix_input.txt ]] || [[ -r ./MISP/.misp_input.txt ]] ) && [[ -x "${THREAT_FEED_TO_ZEEK_SCRIPT}" ]]; then
+        if ( (( ${#THREAT_JSON_FILES[@]} )) || [[ -r ./STIX/.stix_input.txt ]] || [[ -r ./MISP/.misp_input.txt ]] || [[ -r ./Mandiant/mandiant.yaml ]] ) && [[ -x "${THREAT_FEED_TO_ZEEK_SCRIPT}" ]]; then
             "${THREAT_FEED_TO_ZEEK_SCRIPT}" \
                 --ssl-verify ${ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION} \
                 --since "${ZEEK_INTEL_FEED_SINCE}" \
                 --threads ${ZEEK_INTEL_REFRESH_THREADS} \
                 --output ./.threat_autogen.zeek."${INSTANCE_UID}" \
                 --input "${THREAT_JSON_FILES[@]}" \
-                --input-file ./STIX/.stix_input.txt ./MISP/.misp_input.txt
+                --input-file ./STIX/.stix_input.txt ./MISP/.misp_input.txt ./Mandiant/mandiant.yaml
             if [[ $? -eq 0 ]]; then
                 rm -f ./.threat_autogen.zeek.old
                 mv --backup=simple --suffix=.old ./.threat_autogen.zeek."${INSTANCE_UID}" ./.threat_autogen.zeek
diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py
index 1341a243b..fe6402299 100644
--- a/shared/bin/zeek_threat_feed_utils.py
+++ b/shared/bin/zeek_threat_feed_utils.py
@@ -11,6 +11,7 @@
 from collections.abc import Iterable
 from contextlib import contextmanager, nullcontext
 from datetime import datetime
+from dateutil.relativedelta import relativedelta
 from multiprocessing import RawValue
 from pymisp import MISPEvent, MISPAttribute, PyMISP
 from pytz import utc as UTCTimeZone
@@ -28,10 +29,13 @@
 from taxii2client.v21 import Server as TaxiiServer_v21
 from threading import Lock
 from time import sleep, mktime
+from types import GeneratorType, FunctionType, LambdaType
 from typing import Tuple, Union
 from urllib.parse import urljoin, urlparse
 from logging import DEBUG as LOGGING_DEBUG
+import copy
 import json
+import mandiant_threatintel
 import os
 import re
 import requests
@@ -39,7 +43,7 @@
 
 from malcolm_utils import eprint, base64_decode_if_prefixed, LoadStrIfJson, LoadFileIfJson, isprivateip
 
-# keys for dict returned by map_stix_indicator_to_zeek for Zeek intel file fields
+# keys for dict returned by map_*_indicator_to_zeek for Zeek intel file fields
 ZEEK_INTEL_INDICATOR = 'indicator'
 ZEEK_INTEL_INDICATOR_TYPE = 'indicator_type'
 ZEEK_INTEL_META_SOURCE = 'meta.source'
@@ -115,6 +119,16 @@
     "x509-fingerprint-sha1": "CERT_HASH",
 }
 
+# See the documentation for the Zeek INTEL framework [1] and Mandiant threat intel API [2]
+# [1] https://docs.zeek.org/en/current/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Type
+# [2] https://docs.mandiant.com/home/mati-threat-intelligence-api-v4#tag/Indicators
+MANDIANT_ZEEK_INTEL_TYPE_MAP = {
+    mandiant_threatintel.FQDNIndicator: 'DOMAIN',
+    mandiant_threatintel.URLIndicator: 'URL',
+    mandiant_threatintel.IPIndicator: 'ADDR',
+    mandiant_threatintel.MD5Indicator: 'FILE_HASH',
+}
+
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 
@@ -168,6 +182,135 @@ def download_to_file(url, session=None, local_filename=None, chunk_bytes=4096, s
         return None
 
 
+def mandiant_json_serializer(obj):
+    """
+    JSON serializer for mandiant_threatintel.APIResponse object (for debug output)
+    """
+
+    if isinstance(obj, datetime):
+        return obj.astimezone(UTCTimeZone).isoformat()
+
+    elif isinstance(obj, GeneratorType):
+        return list(map(mandiant_json_serializer, obj))
+
+    elif isinstance(obj, list):
+        return [mandiant_json_serializer(item) for item in obj]
+
+    elif isinstance(obj, dict):
+        return {key: mandiant_json_serializer(value) for key, value in obj.items()}
+
+    elif isinstance(obj, set):
+        return {mandiant_json_serializer(item) for item in obj}
+
+    elif isinstance(obj, tuple):
+        return tuple(mandiant_json_serializer(item) for item in obj)
+
+    elif isinstance(obj, FunctionType):
+        return f"function {obj.__name__}" if obj.__name__ != "<lambda>" else "lambda"
+
+    elif isinstance(obj, LambdaType):
+        return "lambda"
+
+    elif (not hasattr(obj, "__str__") or obj.__str__ is object.__str__) and (
+        not hasattr(obj, "__repr__") or obj.__repr__ is object.__repr__
+    ):
+        return obj.__class__.__name__
+
+    else:
+        return str(obj)
+
+
+def mandiant_indicator_as_json_str(indicator):
+    return json.dumps(
+        {
+            key: getattr(indicator, key)
+            for key in indicator.__dir__()
+            if (not key.startswith("_"))
+            and (not key == 'attributed_associations')
+            and (not callable(getattr(indicator, key)))
+        },
+        default=mandiant_json_serializer,
+    )
+
+
+def map_mandiant_indicator_to_zeek(
+    indicator: mandiant_threatintel.APIResponse,
+    logger=None,
+) -> Union[Tuple[defaultdict], None]:
+    """
+    Maps a Mandiant threat intelligence indicator object to Zeek intel items
+    @see https://docs.zeek.org/en/current/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Type
+    @param indicator The indicator object (mandiant_threatintel.APIResponse) to convert
+    @return a list containing the Zeek intel dict(s) from the indicator
+    """
+    results = []
+
+    # get matching Zeek intel type
+    if zeek_type := MANDIANT_ZEEK_INTEL_TYPE_MAP.get(type(indicator), None):
+
+        if logger is not None:
+            logger.debug(mandiant_indicator_as_json_str(indicator))
+
+        zeekItem = defaultdict(lambda: '-')
+        tags = []
+
+        zeekItem[ZEEK_INTEL_INDICATOR_TYPE] = "Intel::" + zeek_type
+        if hasattr(indicator, 'mscore'):
+            zeekItem[ZEEK_INTEL_CIF_CONFIDENCE] = str(round(indicator.mscore / 10))
+        if hasattr(indicator, 'first_seen'):
+            zeekItem[ZEEK_INTEL_CIF_FIRSTSEEN] = str(mktime(indicator.first_seen.timetuple()))
+        if hasattr(indicator, 'last_seen'):
+            zeekItem[ZEEK_INTEL_CIF_LASTSEEN] = str(mktime(indicator.last_seen.timetuple()))
+        if hasattr(indicator, 'sources'):
+            zeekItem[ZEEK_INTEL_META_SOURCE] = ','.join(
+                list({entry['source_name'] for entry in indicator.sources if 'source_name' in entry})
+            )
+            if categories := list(
+                {
+                    category
+                    for item in indicator.sources
+                    if 'category' in item and item['category']
+                    for category in item['category']
+                }
+            ):
+                tags.extend(categories)
+
+        if hasattr(indicator, 'misp'):
+            if trueMispAttrs := [key for key, value in indicator.misp.items() if value]:
+                tags.extend(trueMispAttrs)
+
+        if tags:
+            zeekItem[ZEEK_INTEL_CIF_TAGS] = ','.join(tags)
+
+        # ZEEK_INTEL_META_DESC = 'meta.desc'
+        # ZEEK_INTEL_META_URL = 'meta.url'
+
+        if isinstance(indicator, mandiant_threatintel.MD5Indicator):
+            pass
+            # the MD5Indicator class can actually have multiple types of hashes,
+            #   and we want to create a zeek intel item for each
+            for hashName in ["md5", "sha1", "sha256"]:
+                if hasattr(indicator, hashName) and (val := getattr(indicator, hashName)):
+                    tmpItem = copy.deepcopy(zeekItem)
+                    tmpItem[ZEEK_INTEL_INDICATOR] = val
+                    results.append(tmpItem)
+                    if logger is not None:
+                        logger.debug(tmpItem)
+
+        elif hasattr(indicator, 'value') and (val := indicator.value):
+            # handle other types besides the file hash
+            zeekItem[ZEEK_INTEL_INDICATOR] = val
+            results.append(zeekItem)
+            if logger is not None:
+                logger.debug(zeekItem)
+
+    else:
+        if logger is not None:
+            logger.warning(f"No matching Zeek type found for Mandiant indicator type '{indicator.__class__.__name__}'")
+
+    return results
+
+
 def stix_pattern_from_str(indicator_type: type, pattern_str: str) -> Union[STIX_Pattern_v21, STIX_Pattern_v20, None]:
     """
     Creates a stix2patterns.v20.pattern.Pattern (STIX_Pattern_v20) or a
@@ -482,6 +625,26 @@ def PrintHeader(self):
                     print('\t'.join(['#fields'] + self.fields), file=self.outFile)
                     self.printedHeader = True
 
+    def ProcessMandiant(
+        self,
+        indicator,
+    ):
+        result = False
+        try:
+            if isinstance(indicator, mandiant_threatintel.APIResponse):
+                # map indicator object to Zeek value(s)
+                if vals := map_mandiant_indicator_to_zeek(indicator=indicator, logger=self.logger):
+                    for val in vals:
+                        self.PrintHeader()
+                        with self.lock:
+                            # print the intelligence item fields according to the columns in 'fields'
+                            print('\t'.join([val[key] for key in self.fields]), file=self.outFile)
+
+        except Exception as e:
+            if self.logger is not None:
+                self.logger.warning(f"{type(e).__name__} for {mandiant_indicator_as_json_str(indicator)}: {e}")
+        return result
+
     def ProcessSTIX(
         self,
         toParse,
@@ -635,7 +798,11 @@ def ProcessThreatInputWorker(threatInputWorkerArgs):
                 sleep(1)
             else:
                 try:
-                    with open(inarg) if ((inarg is not None) and os.path.isfile(inarg)) else nullcontext() as infile:
+                    with (
+                        open(inarg)
+                        if (isinstance(inarg, (str, bytes, os.PathLike, int)) and os.path.isfile(inarg))
+                        else nullcontext()
+                    ) as infile:
                         if infile:
                             ##################################################################################
                             # JSON FILE (STIX or MISP)
@@ -664,6 +831,41 @@ def ProcessThreatInputWorker(threatInputWorkerArgs):
                             else:
                                 raise Exception(f"Could not parse JSON in '{inarg}'")
 
+                        elif isinstance(inarg, dict):
+                            ##################################################################################
+                            # Connection parameters specified in dict (e.g., Mandiant Threat Intel)
+                            if 'type' in inarg:
+
+                                if str(inarg['type']).lower() == 'mandiant':
+                                    if mati_client := mandiant_threatintel.ThreatIntelClient(
+                                        api_key=inarg.get('api_key', None),
+                                        secret_key=inarg.get('secret_key', None),
+                                        bearer_token=inarg.get('bearer_token', None),
+                                        api_base_url=inarg.get('api_base_url', mandiant_threatintel.API_BASE_URL),
+                                        client_name=inarg.get('client_name', mandiant_threatintel.CLIENT_APP_NAME),
+                                    ):
+                                        print(since)
+                                        for indicator in mati_client.Indicators.get_list(
+                                            minimum_mscore=inarg.get('minimum_mscore', 60),
+                                            exclude_osint=inarg.get('exclude_osint', False),
+                                            start_epoch=since if since else datetime.now() - relativedelta(years=10),
+                                        ):
+                                            try:
+                                                if zeekPrinter.ProcessMandiant(indicator):
+                                                    successCount.increment()
+                                            except Exception as e:
+                                                if logger is not None:
+                                                    logger.warning(
+                                                        f"[{workerId}]: {type(e).__name__} for Mandiant indicator {indicator.id if isinstance(indicator, mandiant_threatintel.APIResponse) else ''}: {e}"
+                                                    )
+
+                                    else:
+                                        raise Exception(f"Could not connect to Mandiant threat intelligence service")
+                                else:
+                                    raise Exception(f"Could not handle identify threat feed type '{inarg["type"]}'")
+                            else:
+                                raise Exception(f"Could not identify threat feed type in '{inarg}'")
+
                         elif inarg.lower().startswith('misp'):
                             ##################################################################################
                             # MISP URL

From 1e14ac48fcc9f1c656246289b2e1f33b2b9526af Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Wed, 6 Nov 2024 11:43:41 -0700
Subject: [PATCH 03/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358

---
 shared/bin/zeek_threat_feed_utils.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py
index fe6402299..836854f96 100644
--- a/shared/bin/zeek_threat_feed_utils.py
+++ b/shared/bin/zeek_threat_feed_utils.py
@@ -255,6 +255,9 @@ def map_mandiant_indicator_to_zeek(
         tags = []
 
         zeekItem[ZEEK_INTEL_INDICATOR_TYPE] = "Intel::" + zeek_type
+
+        if hasattr(indicator, 'id'):
+            zeekItem[ZEEK_INTEL_META_DESC] = indicator.id
         if hasattr(indicator, 'mscore'):
             zeekItem[ZEEK_INTEL_CIF_CONFIDENCE] = str(round(indicator.mscore / 10))
         if hasattr(indicator, 'first_seen'):
@@ -282,9 +285,6 @@ def map_mandiant_indicator_to_zeek(
         if tags:
             zeekItem[ZEEK_INTEL_CIF_TAGS] = ','.join(tags)
 
-        # ZEEK_INTEL_META_DESC = 'meta.desc'
-        # ZEEK_INTEL_META_URL = 'meta.url'
-
         if isinstance(indicator, mandiant_threatintel.MD5Indicator):
             pass
             # the MD5Indicator class can actually have multiple types of hashes,

From 7a2610c3ba101f21cdb3093be6909a6800b51c4a Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Wed, 6 Nov 2024 11:46:33 -0700
Subject: [PATCH 04/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358

---
 shared/bin/zeek_threat_feed_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py
index 836854f96..4f87c9e7b 100644
--- a/shared/bin/zeek_threat_feed_utils.py
+++ b/shared/bin/zeek_threat_feed_utils.py
@@ -258,6 +258,7 @@ def map_mandiant_indicator_to_zeek(
 
         if hasattr(indicator, 'id'):
             zeekItem[ZEEK_INTEL_META_DESC] = indicator.id
+            zeekItem[ZEEK_INTEL_META_URL] = f'https://advantage.mandiant.com/indicator/{indicator.id}'
         if hasattr(indicator, 'mscore'):
             zeekItem[ZEEK_INTEL_CIF_CONFIDENCE] = str(round(indicator.mscore / 10))
         if hasattr(indicator, 'first_seen'):

From 4d3219d0d197a280690aa4253bce34ff0337156f Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Wed, 6 Nov 2024 12:07:01 -0700
Subject: [PATCH 05/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358

---
 Dockerfiles/zeek.Dockerfile                                    | 1 +
 hedgehog-iso/config/hooks/normal/0910-sensor-build.hook.chroot | 2 +-
 malcolm-iso/build.sh                                           | 1 +
 scripts/malcolm_appliance_packager.sh                          | 1 +
 shared/bin/zeekdeploy.sh                                       | 2 +-
 zeek/intel/Mandiant/.gitignore                                 | 3 +++
 6 files changed, 8 insertions(+), 2 deletions(-)
 create mode 100644 zeek/intel/Mandiant/.gitignore

diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile
index 6745f62db..2515d862c 100644
--- a/Dockerfiles/zeek.Dockerfile
+++ b/Dockerfiles/zeek.Dockerfile
@@ -140,6 +140,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/')
       ( find "${ZEEK_DIR}"/lib/zeek/plugins/packages -type f -name "*.hlto" -exec chmod 755 "{}" \; || true ) && \
     mkdir -p "${ZEEK_DIR}"/share/zeek/site/intel/STIX && \
       mkdir -p "${ZEEK_DIR}"/share/zeek/site/intel/MISP && \
+      mkdir -p "${ZEEK_DIR}"/share/zeek/site/intel/Mandiant && \
       mkdir -p "${ZEEK_DIR}"/share/zeek/site/custom && \
       touch "${ZEEK_DIR}"/share/zeek/site/intel/__load__.zeek && \
       touch "${ZEEK_DIR}"/share/zeek/site/custom/__load__.zeek && \
diff --git a/hedgehog-iso/config/hooks/normal/0910-sensor-build.hook.chroot b/hedgehog-iso/config/hooks/normal/0910-sensor-build.hook.chroot
index 8c40a1f7c..464898ffc 100755
--- a/hedgehog-iso/config/hooks/normal/0910-sensor-build.hook.chroot
+++ b/hedgehog-iso/config/hooks/normal/0910-sensor-build.hook.chroot
@@ -50,7 +50,7 @@ find "${ZEEK_DIR}/" -type f -exec file "{}" \; | grep -Pi "ELF 64-bit.*not strip
 
 # set up default zeek local policy and sensor-related directories
 cp -f /usr/local/etc/zeek/*.zeek /usr/local/etc/zeek/*.txt "${ZEEK_DIR}"/share/zeek/site/
-mkdir -p /opt/sensor/sensor_ctl/zeek/custom /opt/sensor/sensor_ctl/zeek/intel/STIX /opt/sensor/sensor_ctl/zeek/intel/MISP /opt/sensor/sensor_ctl/fluentbit
+mkdir -p /opt/sensor/sensor_ctl/zeek/custom /opt/sensor/sensor_ctl/zeek/intel/STIX /opt/sensor/sensor_ctl/zeek/intel/MISP /opt/sensor/sensor_ctl/zeek/intel/Mandiant /opt/sensor/sensor_ctl/fluentbit
 touch /opt/sensor/sensor_ctl/zeek/intel/__load__.zeek
 touch /opt/sensor/sensor_ctl/zeek/custom/__load__.zeek
 [[ -f /usr/local/bin/zeek_intel_setup.sh ]] && mv /usr/local/bin/zeek_intel_setup.sh "${ZEEK_DIR}"/bin/
diff --git a/malcolm-iso/build.sh b/malcolm-iso/build.sh
index db293d456..e737d13e8 100755
--- a/malcolm-iso/build.sh
+++ b/malcolm-iso/build.sh
@@ -127,6 +127,7 @@ if [ -d "$WORKDIR" ]; then
   mkdir -p "$MALCOLM_DEST_DIR/zeek-logs/processed/"
   mkdir -p "$MALCOLM_DEST_DIR/zeek-logs/upload/"
   mkdir -p "$MALCOLM_DEST_DIR/zeek/custom/"
+  mkdir -p "$MALCOLM_DEST_DIR/zeek/intel/Mandiant/"
   mkdir -p "$MALCOLM_DEST_DIR/zeek/intel/MISP/"
   mkdir -p "$MALCOLM_DEST_DIR/zeek/intel/STIX/"
   cp ./docker-compose.yml "$MALCOLM_DEST_DIR/docker-compose.yml"
diff --git a/scripts/malcolm_appliance_packager.sh b/scripts/malcolm_appliance_packager.sh
index b903a927c..7bd1f26c3 100755
--- a/scripts/malcolm_appliance_packager.sh
+++ b/scripts/malcolm_appliance_packager.sh
@@ -94,6 +94,7 @@ if mkdir "$DESTDIR"; then
   mkdir $VERBOSE -p "$DESTDIR/zeek-logs/processed/"
   mkdir $VERBOSE -p "$DESTDIR/zeek-logs/upload/"
   mkdir $VERBOSE -p "$DESTDIR/zeek/custom/"
+  mkdir $VERBOSE -p "$DESTDIR/zeek/intel/Mandiant/"
   mkdir $VERBOSE -p "$DESTDIR/zeek/intel/MISP/"
   mkdir $VERBOSE -p "$DESTDIR/zeek/intel/STIX/"
 
diff --git a/shared/bin/zeekdeploy.sh b/shared/bin/zeekdeploy.sh
index c931e592c..2500f5167 100755
--- a/shared/bin/zeekdeploy.sh
+++ b/shared/bin/zeekdeploy.sh
@@ -155,7 +155,7 @@ fi
 # make sure "intel" directory exists, even if empty
 [[ -n "$ZEEK_INTEL_PATH" ]] && INTEL_DIR="$ZEEK_INTEL_PATH" || INTEL_DIR=/opt/sensor/sensor_ctl/zeek/intel
 export INTEL_DIR
-mkdir -p "$INTEL_DIR"/STIX "$INTEL_DIR"/MISP
+mkdir -p "$INTEL_DIR"/STIX "$INTEL_DIR"/MISP "$INTEL_DIR"/Mandiant
 touch "$INTEL_DIR"/__load__.zeek 2>/dev/null || true
 # autoconfigure load directives for intel files
 [[ -x "$ZEEK_INSTALL_PATH"/bin/zeek_intel_setup.sh ]] && \
diff --git a/zeek/intel/Mandiant/.gitignore b/zeek/intel/Mandiant/.gitignore
new file mode 100644
index 000000000..a5baada18
--- /dev/null
+++ b/zeek/intel/Mandiant/.gitignore
@@ -0,0 +1,3 @@
+*
+!.gitignore
+

From c73b9ce670dd16ea8a7e451645db80aec1d26ad3 Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Wed, 6 Nov 2024 12:07:39 -0700
Subject: [PATCH 06/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358

---
 docs/README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/README.md b/docs/README.md
index a7004736c..f8be3f476 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -106,6 +106,7 @@ Malcolm can also easily be deployed locally on an ordinary consumer workstation
     - [Zeek Intelligence Framework](zeek-intel.md#ZeekIntel)
         + [STIX™ and TAXII™](zeek-intel.md#ZeekIntelSTIX)
         + [MISP](zeek-intel.md#ZeekIntelMISP)
+        + [Mandiant](zeek-intel.md#ZeekIntelMandiant)
     - ["Best Guess" Fingerprinting for ICS Protocols](ics-best-guess.md#ICSBestGuess)
     - [Asset Interaction Analysis](asset-interaction-analysis.md#AssetInteractionAnalysis)
         + [Enriching network traffic metadata via NetBox lookups](asset-interaction-analysis.md#NetBoxEnrichment)

From 866e30f87d93f8bbe89a488ab5ec78716d26d5e7 Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Wed, 6 Nov 2024 12:45:24 -0700
Subject: [PATCH 07/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358

---
 shared/bin/zeek_intel_from_threat_feed.py | 2 ++
 shared/bin/zeek_threat_feed_utils.py      | 3 +--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/shared/bin/zeek_intel_from_threat_feed.py b/shared/bin/zeek_intel_from_threat_feed.py
index 7fbf76fa6..5319064a6 100755
--- a/shared/bin/zeek_intel_from_threat_feed.py
+++ b/shared/bin/zeek_intel_from_threat_feed.py
@@ -34,6 +34,7 @@ def main():
                 'Outputs a Zeek intelligence framework file from various formats used to represent threat information:',
                 ' - "Indicator" objects in STIX™ v2.0/v2.1 JSON files',
                 ' - MISP attributes or core format JSON files',
+                ' - Indicators from Mantiant Threat Intelligence',
                 '',
                 'See:',
                 ' - Malcolm documentation: https://idaholab.github.io/Malcolm/docs/zeek-intel.html#ZeekIntel',
@@ -47,6 +48,7 @@ def main():
                 ' - MISP default feeds: https://www.misp-project.org/feeds/',
                 ' - Managing MISP feeds: https://misp.gitbooks.io/misp-book/content/managing-feeds/',
                 ' - Expand MISP usage: https://github.com/idaholab/Malcolm/issues/336',
+                ' - Mandiant Threat Intelligence Indicators API: https://docs.mandiant.com/home/mati-threat-intelligence-api-v4#tag/Indicators'
                 '',
                 'Note: The Zeek intelligence framework only supports simple indicators matched against a single value.',
                 'The STIX™ standard can express more complex indicators that cannot be expressed with Zeek intelligence items.',
diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py
index 4f87c9e7b..4ca41cd73 100644
--- a/shared/bin/zeek_threat_feed_utils.py
+++ b/shared/bin/zeek_threat_feed_utils.py
@@ -845,11 +845,10 @@ def ProcessThreatInputWorker(threatInputWorkerArgs):
                                         api_base_url=inarg.get('api_base_url', mandiant_threatintel.API_BASE_URL),
                                         client_name=inarg.get('client_name', mandiant_threatintel.CLIENT_APP_NAME),
                                     ):
-                                        print(since)
                                         for indicator in mati_client.Indicators.get_list(
                                             minimum_mscore=inarg.get('minimum_mscore', 60),
                                             exclude_osint=inarg.get('exclude_osint', False),
-                                            start_epoch=since if since else datetime.now() - relativedelta(years=10),
+                                            start_epoch=since if since else datetime.now() - relativedelta(months=1),
                                         ):
                                             try:
                                                 if zeekPrinter.ProcessMandiant(indicator):

From 208f9b67ca253ad4e50f81577960fb8544b44c26 Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Wed, 6 Nov 2024 14:04:24 -0700
Subject: [PATCH 08/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358

---
 shared/bin/zeek_threat_feed_utils.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py
index 4ca41cd73..a744fa841 100644
--- a/shared/bin/zeek_threat_feed_utils.py
+++ b/shared/bin/zeek_threat_feed_utils.py
@@ -835,9 +835,9 @@ def ProcessThreatInputWorker(threatInputWorkerArgs):
                         elif isinstance(inarg, dict):
                             ##################################################################################
                             # Connection parameters specified in dict (e.g., Mandiant Threat Intel)
-                            if 'type' in inarg:
+                            if ('type' in inarg) and (threatFeedType := str(inarg['type'])):
 
-                                if str(inarg['type']).lower() == 'mandiant':
+                                if threatFeedType.lower() == 'mandiant':
                                     if mati_client := mandiant_threatintel.ThreatIntelClient(
                                         api_key=inarg.get('api_key', None),
                                         secret_key=inarg.get('secret_key', None),
@@ -860,9 +860,9 @@ def ProcessThreatInputWorker(threatInputWorkerArgs):
                                                     )
 
                                     else:
-                                        raise Exception(f"Could not connect to Mandiant threat intelligence service")
+                                        raise Exception("Could not connect to Mandiant threat intelligence service")
                                 else:
-                                    raise Exception(f"Could not handle identify threat feed type '{inarg["type"]}'")
+                                    raise Exception(f"Could not handle identify threat feed type '{threatFeedType}'")
                             else:
                                 raise Exception(f"Could not identify threat feed type in '{inarg}'")
 

From fcd95d81fd9c03d2b3fb211cd219837021da0b6b Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Wed, 6 Nov 2024 15:04:14 -0700
Subject: [PATCH 09/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358

---
 shared/bin/zeek_threat_feed_utils.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py
index a744fa841..a6fd66bc8 100644
--- a/shared/bin/zeek_threat_feed_utils.py
+++ b/shared/bin/zeek_threat_feed_utils.py
@@ -846,9 +846,14 @@ def ProcessThreatInputWorker(threatInputWorkerArgs):
                                         client_name=inarg.get('client_name', mandiant_threatintel.CLIENT_APP_NAME),
                                     ):
                                         for indicator in mati_client.Indicators.get_list(
+                                            start_epoch=since if since else datetime.now() - relativedelta(weeks=1),
                                             minimum_mscore=inarg.get('minimum_mscore', 60),
                                             exclude_osint=inarg.get('exclude_osint', False),
-                                            start_epoch=since if since else datetime.now() - relativedelta(months=1),
+                                            include_campaigns=inarg.get('include_campaigns', False),
+                                            include_reports=inarg.get('include_reports', False),
+                                            include_threat_rating=inarg.get('include_threat_rating', False),
+                                            include_misp=inarg.get('include_misp', True),
+                                            include_category=inarg.get('include_category', True),
                                         ):
                                             try:
                                                 if zeekPrinter.ProcessMandiant(indicator):

From 68f416a27443f6fb8ba90775a7a3f439ce734ef8 Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Thu, 7 Nov 2024 08:15:04 -0700
Subject: [PATCH 10/42] have threat intel happen once under supervisord on
 startup, not in container entrypoint for zeek non-live container, to support
 cisagov/Malcolm#358

---
 Dockerfiles/zeek.Dockerfile       |  7 ++++---
 config/zeek-offline.env.example   |  6 +++---
 zeek/scripts/container_health.sh  |  2 +-
 zeek/scripts/docker_entrypoint.sh | 11 -----------
 zeek/supervisord.conf             | 11 +++++++++++
 5 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile
index 2515d862c..d3253806c 100644
--- a/Dockerfiles/zeek.Dockerfile
+++ b/Dockerfiles/zeek.Dockerfile
@@ -203,8 +203,9 @@ ARG ZEEK_PCAP_PROCESSOR=true
 #Whether or not to run "zeek -r XXXXX.pcap local" on each pcap file
 ARG ZEEK_AUTO_ANALYZE_PCAP_FILES=false
 ARG ZEEK_AUTO_ANALYZE_PCAP_THREADS=1
-#Whether or not to refresh intel at various points during processing
-ARG ZEEK_INTEL_REFRESH_ON_ENTRYPOINT=false
+#Whether or not to do first intel refresh under supervisord
+ARG ZEEK_INTEL_REFRESH_ON_STARTUP=false
+#Whether or not to do first intel refresh under zeekdeploy.sh
 ARG ZEEK_INTEL_REFRESH_ON_DEPLOY=false
 ARG ZEEK_INTEL_REFRESH_CRON_EXPRESSION=
 ARG ZEEK_INTEL_ITEM_EXPIRATION=-1min
@@ -227,7 +228,7 @@ ARG PCAP_NODE_NAME=malcolm
 
 ENV AUTO_TAG $AUTO_TAG
 ENV ZEEK_PCAP_PROCESSOR $ZEEK_PCAP_PROCESSOR
-ENV ZEEK_INTEL_REFRESH_ON_ENTRYPOINT $ZEEK_INTEL_REFRESH_ON_ENTRYPOINT
+ENV ZEEK_INTEL_REFRESH_ON_STARTUP $ZEEK_INTEL_REFRESH_ON_STARTUP
 ENV ZEEK_INTEL_REFRESH_ON_DEPLOY $ZEEK_INTEL_REFRESH_ON_DEPLOY
 ENV ZEEK_INTEL_REFRESH_CRON_EXPRESSION $ZEEK_INTEL_REFRESH_CRON_EXPRESSION
 ENV ZEEK_AUTO_ANALYZE_PCAP_FILES $ZEEK_AUTO_ANALYZE_PCAP_FILES
diff --git a/config/zeek-offline.env.example b/config/zeek-offline.env.example
index 75c2e130a..6e4efc7c5 100644
--- a/config/zeek-offline.env.example
+++ b/config/zeek-offline.env.example
@@ -11,9 +11,9 @@ ZEEK_ROTATED_PCAP=true
 
 ZEEK_PCAP_PROCESSOR=true
 
-# Specifies whether or not to refresh Zeek Intelligence Framework files in
-#   the container entrypoint
-ZEEK_INTEL_REFRESH_ON_ENTRYPOINT=true
+# Specifies whether or not to refresh Zeek Intelligence Framework files
+#   as soon as the container starts up
+ZEEK_INTEL_REFRESH_ON_STARTUP=true
 # Specifies a cron expression indicating the refresh interval for generating the
 #   Zeek Intelligence Framework files (or blank to disable automatic refresh)
 ZEEK_INTEL_REFRESH_CRON_EXPRESSION=
\ No newline at end of file
diff --git a/zeek/scripts/container_health.sh b/zeek/scripts/container_health.sh
index 2fa834f4c..75464199f 100755
--- a/zeek/scripts/container_health.sh
+++ b/zeek/scripts/container_health.sh
@@ -5,7 +5,7 @@
 if [[ "${ZEEK_LIVE_CAPTURE:-false}" == "true" ]]; then
     supervisorctl status live-zeek >/dev/null 2>&1
 else
-    if [[ "${ZEEK_INTEL_REFRESH_ON_ENTRYPOINT:-false}" == "true" ]]; then
+    if [[ "${ZEEK_INTEL_REFRESH_ON_STARTUP:-false}" == "true" ]]; then
         ( ps a 2>/dev/null | grep -q '[z]eek_intel_from_threat_feed.py' ) || supervisorctl status pcap-zeek >/dev/null 2>&1
     else
         supervisorctl status pcap-zeek >/dev/null 2>&1
diff --git a/zeek/scripts/docker_entrypoint.sh b/zeek/scripts/docker_entrypoint.sh
index 34aa5f82b..88be6bdd1 100755
--- a/zeek/scripts/docker_entrypoint.sh
+++ b/zeek/scripts/docker_entrypoint.sh
@@ -6,16 +6,5 @@ ZEEK_DIR=${ZEEK_DIR:-"/opt/zeek"}
 setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ZEEK_DIR}"/bin/zeek 2>/dev/null || true
 setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ZEEK_DIR}"/bin/capstats 2>/dev/null || true
 
-if [[ "${ZEEK_INTEL_REFRESH_ON_ENTRYPOINT:-false}" == "true" ]] && \
-   [[ -x "${ZEEK_DIR}"/bin/zeek_intel_setup.sh ]]; then
-    if [[ "$(id -u)" == "0" ]] && [[ -n "$PUSER" ]]; then
-        su -s /bin/bash -p ${PUSER} << EOF
-            "${ZEEK_DIR}"/bin/zeek_intel_setup.sh /bin/true
-EOF
-    else
-        "${ZEEK_DIR}"/bin/zeek_intel_setup.sh /bin/true
-    fi
-fi
-
 # start supervisor (which will spawn pcap-zeek, cron, etc.) or whatever the default command is
 exec "$@"
diff --git a/zeek/supervisord.conf b/zeek/supervisord.conf
index ec258c16d..3c9a89b1c 100644
--- a/zeek/supervisord.conf
+++ b/zeek/supervisord.conf
@@ -53,6 +53,17 @@ stdout_logfile_maxbytes=0
 redirect_stderr=true
 user=%(ENV_PUSER)s
 
+[program:intel-initialization]
+command="%(ENV_ZEEK_DIR)s"/bin/zeek_intel_setup.sh /bin/true
+autostart=%(ENV_ZEEK_INTEL_REFRESH_ON_STARTUP)s
+autorestart=false
+stopasgroup=true
+killasgroup=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+user=%(ENV_PUSER)s
+
 [program:live-zeek]
 command=/opt/zeek/bin/zeekdeploy.sh
 autostart=%(ENV_ZEEK_LIVE_CAPTURE)s

From 45ab9ce5f61ca454967cbd5b1b14f875ed37e055 Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Thu, 7 Nov 2024 08:26:09 -0700
Subject: [PATCH 11/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358

---
 shared/bin/zeek_intel_setup.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/shared/bin/zeek_intel_setup.sh b/shared/bin/zeek_intel_setup.sh
index 0f11ebc21..ca9dc1bb2 100755
--- a/shared/bin/zeek_intel_setup.sh
+++ b/shared/bin/zeek_intel_setup.sh
@@ -86,8 +86,9 @@ EOF
             elif [[ -f "${DIR}"/__load__.zeek ]]; then
                 # this intel feed has its own load directive and should take care of itself
                 echo "@load ${DIR}" >> ./__load__.zeek."${INSTANCE_UID}"
-            else
-                # this directory contains "loose" intel files we'll need to load explicitly
+
+            elif [[ "${DIR}" != "./Mandiant" ]]; then
+                # this custom directory contains "loose" intel files we'll need to load explicitly
                 while IFS= read -r line; do
                     LOOSE_INTEL_FILES+=( "$line" )
                 done < <( find "${INTEL_DIR}/${DIR}" -type f ! -name ".*" 2>/dev/null )

From 6fb54ab9e12f6987089b97cc5bd771e352967c3c Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Thu, 7 Nov 2024 12:49:23 -0700
Subject: [PATCH 12/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358

---
 shared/bin/zeek_threat_feed_utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py
index a6fd66bc8..2eb7d5f79 100644
--- a/shared/bin/zeek_threat_feed_utils.py
+++ b/shared/bin/zeek_threat_feed_utils.py
@@ -845,8 +845,10 @@ def ProcessThreatInputWorker(threatInputWorkerArgs):
                                         api_base_url=inarg.get('api_base_url', mandiant_threatintel.API_BASE_URL),
                                         client_name=inarg.get('client_name', mandiant_threatintel.CLIENT_APP_NAME),
                                     ):
+                                        nowTime = datetime.now().astimezone(UTCTimeZone)
                                         for indicator in mati_client.Indicators.get_list(
-                                            start_epoch=since if since else datetime.now() - relativedelta(weeks=1),
+                                            start_epoch=since if since else nowTime - relativedelta(hours=24),
+                                            end_epoch=nowTime,
                                             minimum_mscore=inarg.get('minimum_mscore', 60),
                                             exclude_osint=inarg.get('exclude_osint', False),
                                             include_campaigns=inarg.get('include_campaigns', False),

From c3aeb5cb039f89e4d235ea4201585e83c04978ac Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Thu, 7 Nov 2024 14:22:02 -0700
Subject: [PATCH 13/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358

---
 shared/bin/zeek_threat_feed_utils.py | 35 +++++++++++++++++++++-------
 1 file changed, 26 insertions(+), 9 deletions(-)

diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py
index 2eb7d5f79..bb8a36184 100644
--- a/shared/bin/zeek_threat_feed_utils.py
+++ b/shared/bin/zeek_threat_feed_utils.py
@@ -57,12 +57,20 @@
 ZEEK_INTEL_CIF_FIRSTSEEN = 'meta.cif_firstseen'
 ZEEK_INTEL_CIF_LASTSEEN = 'meta.cif_lastseen'
 
+ZEEK_INTEL_WORKER_THREADS_DEFAULT = 2
+
 TAXII_INDICATOR_FILTER = {'type': 'indicator'}
 TAXII_PAGE_SIZE = 50
 MISP_PAGE_SIZE_ATTRIBUTES = 500
 MISP_PAGE_SIZE_EVENTS = 10
-ZEEK_INTEL_WORKER_THREADS_DEFAULT = 2
-
+MANDIANT_PAGE_SIZE_DEFAULT = 100
+MANDIANT_MINIMUM_MSCORE_DEFAULT = 60
+MANDIANT_EXCLUDE_OSINT_DEFAULT = False
+MANDIANT_INCLUDE_CAMPAIGNS_DEFAULT = False
+MANDIANT_INCLUDE_REPORTS_DEFAULT = False
+MANDIANT_INCLUDE_THREAT_RATING_DEFAULT = False
+MANDIANT_INCLUDE_MISP_DEFAULT = True
+MANDIANT_INCLUDE_CATEGORY_DEFAULT = True
 
 # See the documentation for the Zeek INTEL framework [1] and STIX-2 cyber observable objects [2]
 # [1] https://docs.zeek.org/en/stable/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Type
@@ -849,13 +857,22 @@ def ProcessThreatInputWorker(threatInputWorkerArgs):
                                         for indicator in mati_client.Indicators.get_list(
                                             start_epoch=since if since else nowTime - relativedelta(hours=24),
                                             end_epoch=nowTime,
-                                            minimum_mscore=inarg.get('minimum_mscore', 60),
-                                            exclude_osint=inarg.get('exclude_osint', False),
-                                            include_campaigns=inarg.get('include_campaigns', False),
-                                            include_reports=inarg.get('include_reports', False),
-                                            include_threat_rating=inarg.get('include_threat_rating', False),
-                                            include_misp=inarg.get('include_misp', True),
-                                            include_category=inarg.get('include_category', True),
+                                            page_size=inarg.get('page_size', MANDIANT_PAGE_SIZE_DEFAULT),
+                                            minimum_mscore=inarg.get('minimum_mscore', MANDIANT_MINIMUM_MSCORE_DEFAULT),
+                                            exclude_osint=inarg.get('exclude_osint', MANDIANT_EXCLUDE_OSINT_DEFAULT),
+                                            include_campaigns=inarg.get(
+                                                'include_campaigns', MANDIANT_INCLUDE_CAMPAIGNS_DEFAULT
+                                            ),
+                                            include_reports=inarg.get(
+                                                'include_reports', MANDIANT_INCLUDE_REPORTS_DEFAULT
+                                            ),
+                                            include_threat_rating=inarg.get(
+                                                'include_threat_rating', MANDIANT_INCLUDE_THREAT_RATING_DEFAULT
+                                            ),
+                                            include_misp=inarg.get('include_misp', MANDIANT_INCLUDE_MISP_DEFAULT),
+                                            include_category=inarg.get(
+                                                'include_category', MANDIANT_INCLUDE_CATEGORY_DEFAULT
+                                            ),
                                         ):
                                             try:
                                                 if zeekPrinter.ProcessMandiant(indicator):

From fa0d731c0e90e281dcb2d0f28b8cb585d5fc011e Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Thu, 7 Nov 2024 15:49:36 -0700
Subject: [PATCH 14/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358

---
 shared/bin/zeek_threat_feed_utils.py | 59 +++++++++++++++++-----------
 1 file changed, 35 insertions(+), 24 deletions(-)

diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py
index bb8a36184..a04ef00de 100644
--- a/shared/bin/zeek_threat_feed_utils.py
+++ b/shared/bin/zeek_threat_feed_utils.py
@@ -228,13 +228,13 @@ def mandiant_json_serializer(obj):
         return str(obj)
 
 
-def mandiant_indicator_as_json_str(indicator):
+def mandiant_indicator_as_json_str(indicator, skip_attr_map={}):
     return json.dumps(
         {
             key: getattr(indicator, key)
             for key in indicator.__dir__()
-            if (not key.startswith("_"))
-            and (not key == 'attributed_associations')
+            if (skip_attr_map.get(key, False) == False)
+            and (not key.startswith("_"))
             and (not callable(getattr(indicator, key)))
         },
         default=mandiant_json_serializer,
@@ -243,6 +243,7 @@ def mandiant_indicator_as_json_str(indicator):
 
 def map_mandiant_indicator_to_zeek(
     indicator: mandiant_threatintel.APIResponse,
+    skip_attr_map={},
     logger=None,
 ) -> Union[Tuple[defaultdict], None]:
     """
@@ -257,7 +258,7 @@ def map_mandiant_indicator_to_zeek(
     if zeek_type := MANDIANT_ZEEK_INTEL_TYPE_MAP.get(type(indicator), None):
 
         if logger is not None:
-            logger.debug(mandiant_indicator_as_json_str(indicator))
+            logger.debug(mandiant_indicator_as_json_str(indicator, skip_attr_map=skip_attr_map))
 
         zeekItem = defaultdict(lambda: '-')
         tags = []
@@ -634,15 +635,14 @@ def PrintHeader(self):
                     print('\t'.join(['#fields'] + self.fields), file=self.outFile)
                     self.printedHeader = True
 
-    def ProcessMandiant(
-        self,
-        indicator,
-    ):
+    def ProcessMandiant(self, indicator, skip_attr_map={}):
         result = False
         try:
             if isinstance(indicator, mandiant_threatintel.APIResponse):
                 # map indicator object to Zeek value(s)
-                if vals := map_mandiant_indicator_to_zeek(indicator=indicator, logger=self.logger):
+                if vals := map_mandiant_indicator_to_zeek(
+                    indicator=indicator, skip_attr_map=skip_attr_map, logger=self.logger
+                ):
                     for val in vals:
                         self.PrintHeader()
                         with self.lock:
@@ -651,7 +651,9 @@ def ProcessMandiant(
 
         except Exception as e:
             if self.logger is not None:
-                self.logger.warning(f"{type(e).__name__} for {mandiant_indicator_as_json_str(indicator)}: {e}")
+                self.logger.warning(
+                    f"{type(e).__name__} for {indicator.id if hasattr(indicator, 'id') else 'indicator'}: {e}"
+                )
         return result
 
     def ProcessSTIX(
@@ -853,6 +855,23 @@ def ProcessThreatInputWorker(threatInputWorkerArgs):
                                         api_base_url=inarg.get('api_base_url', mandiant_threatintel.API_BASE_URL),
                                         client_name=inarg.get('client_name', mandiant_threatintel.CLIENT_APP_NAME),
                                     ):
+                                        skip_attr_map = defaultdict(lambda: False)
+                                        skip_attr_map['campaigns'] = not bool(
+                                            inarg.get('include_campaigns', MANDIANT_INCLUDE_CAMPAIGNS_DEFAULT)
+                                        )
+                                        skip_attr_map['category'] = not bool(
+                                            inarg.get('include_category', MANDIANT_INCLUDE_CATEGORY_DEFAULT)
+                                        )
+                                        skip_attr_map['misp'] = not bool(
+                                            inarg.get('include_misp', MANDIANT_INCLUDE_MISP_DEFAULT)
+                                        )
+                                        skip_attr_map['reports'] = not bool(
+                                            inarg.get('include_reports', MANDIANT_INCLUDE_REPORTS_DEFAULT)
+                                        )
+                                        skip_attr_map['threat_rating'] = not bool(
+                                            inarg.get('include_threat_rating', MANDIANT_INCLUDE_THREAT_RATING_DEFAULT)
+                                        )
+                                        skip_attr_map['attributed_associations'] = True
                                         nowTime = datetime.now().astimezone(UTCTimeZone)
                                         for indicator in mati_client.Indicators.get_list(
                                             start_epoch=since if since else nowTime - relativedelta(hours=24),
@@ -860,22 +879,14 @@ def ProcessThreatInputWorker(threatInputWorkerArgs):
                                             page_size=inarg.get('page_size', MANDIANT_PAGE_SIZE_DEFAULT),
                                             minimum_mscore=inarg.get('minimum_mscore', MANDIANT_MINIMUM_MSCORE_DEFAULT),
                                             exclude_osint=inarg.get('exclude_osint', MANDIANT_EXCLUDE_OSINT_DEFAULT),
-                                            include_campaigns=inarg.get(
-                                                'include_campaigns', MANDIANT_INCLUDE_CAMPAIGNS_DEFAULT
-                                            ),
-                                            include_reports=inarg.get(
-                                                'include_reports', MANDIANT_INCLUDE_REPORTS_DEFAULT
-                                            ),
-                                            include_threat_rating=inarg.get(
-                                                'include_threat_rating', MANDIANT_INCLUDE_THREAT_RATING_DEFAULT
-                                            ),
-                                            include_misp=inarg.get('include_misp', MANDIANT_INCLUDE_MISP_DEFAULT),
-                                            include_category=inarg.get(
-                                                'include_category', MANDIANT_INCLUDE_CATEGORY_DEFAULT
-                                            ),
+                                            include_campaigns=not skip_attr_map['campaigns'],
+                                            include_reports=not skip_attr_map['reports'],
+                                            include_threat_rating=not skip_attr_map['threat_rating'],
+                                            include_misp=not skip_attr_map['misp'],
+                                            include_category=skip_attr_map['category'],
                                         ):
                                             try:
-                                                if zeekPrinter.ProcessMandiant(indicator):
+                                                if zeekPrinter.ProcessMandiant(indicator, skip_attr_map=skip_attr_map):
                                                     successCount.increment()
                                             except Exception as e:
                                                 if logger is not None:

From b07504f9d689f485b2f10a7a78c544adbf3f86ee Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Fri, 8 Nov 2024 08:31:09 -0700
Subject: [PATCH 15/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358

---
 shared/bin/zeek_threat_feed_utils.py | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py
index a04ef00de..07165c51a 100644
--- a/shared/bin/zeek_threat_feed_utils.py
+++ b/shared/bin/zeek_threat_feed_utils.py
@@ -295,14 +295,25 @@ def map_mandiant_indicator_to_zeek(
         if tags:
             zeekItem[ZEEK_INTEL_CIF_TAGS] = ','.join(tags)
 
-        if isinstance(indicator, mandiant_threatintel.MD5Indicator):
-            pass
-            # the MD5Indicator class can actually have multiple types of hashes,
-            #   and we want to create a zeek intel item for each
-            for hashName in ["md5", "sha1", "sha256"]:
-                if hasattr(indicator, hashName) and (val := getattr(indicator, hashName)):
+        # The MD5Indicator class can actually have multiple types of hashes,
+        #   and we want to create a zeek intel item for each. I'm accessing
+        #   the underlying API response directly here (rather than through getattr)
+        #   to avoid extra GET requests to the API attempting to find a value
+        #   that didn't come with the initial request.
+        #   Performance-wise, if we didn't get it with the indicator object in
+        #   the first place it's not something we need to make an entire extra
+        #   network communication to attempt.
+        if (
+            isinstance(indicator, mandiant_threatintel.MD5Indicator)
+            and indicator._api_response
+            and (hashes := indicator._api_response.get('associated_hashes', []))
+        ):
+            for hashish in hashes:
+                if hashVal := hashish.get('value', None):
                     tmpItem = copy.deepcopy(zeekItem)
-                    tmpItem[ZEEK_INTEL_INDICATOR] = val
+                    tmpItem[ZEEK_INTEL_INDICATOR] = hashish
+                    if newId := hashish.get('id', None):
+                        tmpItem[ZEEK_INTEL_META_URL] = f'https://advantage.mandiant.com/indicator/{newId}'
                     results.append(tmpItem)
                     if logger is not None:
                         logger.debug(tmpItem)

From 7810d023895db95dc605674d8a9ff6a70ae9fdcb Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Fri, 8 Nov 2024 08:39:28 -0700
Subject: [PATCH 16/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358

---
 shared/bin/zeek_threat_feed_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py
index 07165c51a..5ab872066 100644
--- a/shared/bin/zeek_threat_feed_utils.py
+++ b/shared/bin/zeek_threat_feed_utils.py
@@ -311,7 +311,7 @@ def map_mandiant_indicator_to_zeek(
             for hashish in hashes:
                 if hashVal := hashish.get('value', None):
                     tmpItem = copy.deepcopy(zeekItem)
-                    tmpItem[ZEEK_INTEL_INDICATOR] = hashish
+                    tmpItem[ZEEK_INTEL_INDICATOR] = hashVal
                     if newId := hashish.get('id', None):
                         tmpItem[ZEEK_INTEL_META_URL] = f'https://advantage.mandiant.com/indicator/{newId}'
                     results.append(tmpItem)

From 3c8d301f55cca48361d32b3f76708b13abe91c80 Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Fri, 8 Nov 2024 08:56:07 -0700
Subject: [PATCH 17/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358

---
 shared/bin/zeek_threat_feed_utils.py | 52 +++-------------------------
 1 file changed, 4 insertions(+), 48 deletions(-)

diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py
index 5ab872066..82c3b1e31 100644
--- a/shared/bin/zeek_threat_feed_utils.py
+++ b/shared/bin/zeek_threat_feed_utils.py
@@ -190,55 +190,11 @@ def download_to_file(url, session=None, local_filename=None, chunk_bytes=4096, s
         return None
 
 
-def mandiant_json_serializer(obj):
-    """
-    JSON serializer for mandiant_threatintel.APIResponse object (for debug output)
-    """
-
-    if isinstance(obj, datetime):
-        return obj.astimezone(UTCTimeZone).isoformat()
-
-    elif isinstance(obj, GeneratorType):
-        return list(map(mandiant_json_serializer, obj))
-
-    elif isinstance(obj, list):
-        return [mandiant_json_serializer(item) for item in obj]
-
-    elif isinstance(obj, dict):
-        return {key: mandiant_json_serializer(value) for key, value in obj.items()}
-
-    elif isinstance(obj, set):
-        return {mandiant_json_serializer(item) for item in obj}
-
-    elif isinstance(obj, tuple):
-        return tuple(mandiant_json_serializer(item) for item in obj)
-
-    elif isinstance(obj, FunctionType):
-        return f"function {obj.__name__}" if obj.__name__ != "<lambda>" else "lambda"
-
-    elif isinstance(obj, LambdaType):
-        return "lambda"
-
-    elif (not hasattr(obj, "__str__") or obj.__str__ is object.__str__) and (
-        not hasattr(obj, "__repr__") or obj.__repr__ is object.__repr__
-    ):
-        return obj.__class__.__name__
-
-    else:
-        return str(obj)
-
-
 def mandiant_indicator_as_json_str(indicator, skip_attr_map={}):
-    return json.dumps(
-        {
-            key: getattr(indicator, key)
-            for key in indicator.__dir__()
-            if (skip_attr_map.get(key, False) == False)
-            and (not key.startswith("_"))
-            and (not callable(getattr(indicator, key)))
-        },
-        default=mandiant_json_serializer,
-    )
+    if indicator and indicator._api_response:
+        return json.dumps(indicator._api_response)
+    else:
+        return 'unknown indicator'
 
 
 def map_mandiant_indicator_to_zeek(

From 0505fee43bcf2692ed1536f1304cadb565a64c26 Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Fri, 8 Nov 2024 09:01:43 -0700
Subject: [PATCH 18/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358

---
 shared/bin/zeek_threat_feed_utils.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py
index 82c3b1e31..04b531db8 100644
--- a/shared/bin/zeek_threat_feed_utils.py
+++ b/shared/bin/zeek_threat_feed_utils.py
@@ -177,7 +177,7 @@ def download_to_file(url, session=None, local_filename=None, chunk_bytes=4096, s
                 f.write(chunk)
     fExists = os.path.isfile(tmpDownloadedFileSpec)
     fSize = os.path.getsize(tmpDownloadedFileSpec)
-    if logger is not None:
+    if (logger is not None) and (LOGGING_DEBUG >= logger.root.level):
         logger.debug(
             f"Download of {url} to {tmpDownloadedFileSpec} {'succeeded' if fExists else 'failed'} ({fSize} bytes)"
         )
@@ -213,7 +213,7 @@ def map_mandiant_indicator_to_zeek(
     # get matching Zeek intel type
     if zeek_type := MANDIANT_ZEEK_INTEL_TYPE_MAP.get(type(indicator), None):
 
-        if logger is not None:
+        if (logger is not None) and (LOGGING_DEBUG >= logger.root.level):
             logger.debug(mandiant_indicator_as_json_str(indicator, skip_attr_map=skip_attr_map))
 
         zeekItem = defaultdict(lambda: '-')
@@ -271,14 +271,14 @@ def map_mandiant_indicator_to_zeek(
                     if newId := hashish.get('id', None):
                         tmpItem[ZEEK_INTEL_META_URL] = f'https://advantage.mandiant.com/indicator/{newId}'
                     results.append(tmpItem)
-                    if logger is not None:
+                    if (logger is not None) and (LOGGING_DEBUG >= logger.root.level):
                         logger.debug(tmpItem)
 
         elif hasattr(indicator, 'value') and (val := indicator.value):
             # handle other types besides the file hash
             zeekItem[ZEEK_INTEL_INDICATOR] = val
             results.append(zeekItem)
-            if logger is not None:
+            if (logger is not None) and (LOGGING_DEBUG >= logger.root.level):
                 logger.debug(zeekItem)
 
     else:
@@ -420,7 +420,7 @@ def map_stix_indicator_to_zeek(
             )
         return None
 
-    if logger is not None:
+    if (logger is not None) and (LOGGING_DEBUG >= logger.root.level):
         logger.debug(indicator)
 
     results = []
@@ -474,7 +474,7 @@ def map_stix_indicator_to_zeek(
         # TODO: confidence?
 
         results.append(zeekItem)
-        if logger is not None:
+        if (logger is not None) and (LOGGING_DEBUG >= logger.root.level):
             logger.debug(zeekItem)
 
     return results
@@ -495,7 +495,7 @@ def map_misp_attribute_to_zeek(
     @param attribute The MISPAttribute to convert
     @return a list containing the Zeek intel dict(s) from the MISPAttribute object
     """
-    if logger is not None:
+    if (logger is not None) and (LOGGING_DEBUG >= logger.root.level):
         logger.debug(attribute.to_json())
 
     results = []
@@ -550,7 +550,7 @@ def map_misp_attribute_to_zeek(
             zeekItem[ZEEK_INTEL_CIF_CONFIDENCE] = str(confidence)
 
         results.append(zeekItem)
-        if logger is not None:
+        if (logger is not None) and (LOGGING_DEBUG >= logger.root.level):
             logger.debug(zeekItem)
 
     return results
@@ -765,7 +765,7 @@ def ProcessThreatInputWorker(threatInputWorkerArgs):
     )
 
     with workerThreadCount as workerId:
-        if logger is not None:
+        if (logger is not None) and (LOGGING_DEBUG >= logger.root.level):
             logger.debug(f"[{workerId}]: started")
 
         # the queue was fully populated before we started, so we can run until there are no more elements
@@ -1007,7 +1007,7 @@ def ProcessThreatInputWorker(threatInputWorkerArgs):
                                                         # error or unrecognized results, set this to short circuit
                                                         resultCount = 0
 
-                                                    if logger is not None:
+                                                    if (logger is not None) and (LOGGING_DEBUG >= logger.root.level):
                                                         logger.debug(
                                                             f"[{workerId}]: MISP search page {mispPage} returned {resultCount}"
                                                         )
@@ -1222,5 +1222,5 @@ def ProcessThreatInputWorker(threatInputWorkerArgs):
                     if logger is not None:
                         logger.warning(f"[{workerId}]: {type(e).__name__} for '{inarg}': {e}")
 
-        if logger is not None:
+        if (logger is not None) and (LOGGING_DEBUG >= logger.root.level):
             logger.debug(f"[{workerId}]: finished")

From 875c0d17f76d8be7967878d13b6bb335d746b052 Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Fri, 8 Nov 2024 09:12:45 -0700
Subject: [PATCH 19/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358

---
 shared/bin/zeek_threat_feed_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py
index 04b531db8..d27d0bfec 100644
--- a/shared/bin/zeek_threat_feed_utils.py
+++ b/shared/bin/zeek_threat_feed_utils.py
@@ -63,7 +63,7 @@
 TAXII_PAGE_SIZE = 50
 MISP_PAGE_SIZE_ATTRIBUTES = 500
 MISP_PAGE_SIZE_EVENTS = 10
-MANDIANT_PAGE_SIZE_DEFAULT = 100
+MANDIANT_PAGE_SIZE_DEFAULT = 1000
 MANDIANT_MINIMUM_MSCORE_DEFAULT = 60
 MANDIANT_EXCLUDE_OSINT_DEFAULT = False
 MANDIANT_INCLUDE_CAMPAIGNS_DEFAULT = False

From 231ad1349a430c258aaf327431787bb804183994 Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Fri, 8 Nov 2024 10:05:33 -0700
Subject: [PATCH 20/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358

---
 shared/bin/zeek_threat_feed_utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py
index d27d0bfec..66f7f9c6d 100644
--- a/shared/bin/zeek_threat_feed_utils.py
+++ b/shared/bin/zeek_threat_feed_utils.py
@@ -615,6 +615,8 @@ def ProcessMandiant(self, indicator, skip_attr_map={}):
                         with self.lock:
                             # print the intelligence item fields according to the columns in 'fields'
                             print('\t'.join([val[key] for key in self.fields]), file=self.outFile)
+                        if not result:
+                            result = True
 
         except Exception as e:
             if self.logger is not None:

From 13bf9a754fe18ab859e84f8f647059cb664c5397 Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Fri, 8 Nov 2024 11:33:21 -0700
Subject: [PATCH 21/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358

---
 logstash/pipelines/zeek/1029_zeek_intel.conf | 12 ++++--------
 shared/bin/zeek_threat_feed_utils.py         | 10 ++++++++--
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/logstash/pipelines/zeek/1029_zeek_intel.conf b/logstash/pipelines/zeek/1029_zeek_intel.conf
index d284f10a4..7bab04681 100644
--- a/logstash/pipelines/zeek/1029_zeek_intel.conf
+++ b/logstash/pipelines/zeek/1029_zeek_intel.conf
@@ -40,16 +40,12 @@ filter {
           code => "event.set('[zeek_cols]', @zeek_intel_field_names.zip(event.get('[message]')).to_h)"
         }
       }
-      mutate { id => "mutate_split_zeek_intel_commas"
-               split => { "[zeek_cols][sources]" => ","
-                          "[zeek_cols][matched]" => "," } }
     }
 
-    # For some reason, even in JSON, I have cif_tags strings like:
-    #   Network activity,osint:source-type=\"block-or-filter-list\"
-    # so whatever reason it's not already an array. Split it here.
-    mutate { id => "mutate_split_zeek_intel_cif_tags"
-             split => { "[zeek_cols][cif_tags]" => "," } }
+    mutate { id => "mutate_split_zeek_intel_commas"
+             split => { "[zeek_cols][sources]" => ","
+                        "[zeek_cols][matched]" => ","
+                        "[zeek_cols][cif_tags]" => "," } }
 
   }
 
diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py
index 66f7f9c6d..18d342757 100644
--- a/shared/bin/zeek_threat_feed_utils.py
+++ b/shared/bin/zeek_threat_feed_utils.py
@@ -232,7 +232,13 @@ def map_mandiant_indicator_to_zeek(
             zeekItem[ZEEK_INTEL_CIF_LASTSEEN] = str(mktime(indicator.last_seen.timetuple()))
         if hasattr(indicator, 'sources'):
             zeekItem[ZEEK_INTEL_META_SOURCE] = ','.join(
-                list({entry['source_name'] for entry in indicator.sources if 'source_name' in entry})
+                list(
+                    {
+                        entry['source_name'].replace(',', '\\x2c')
+                        for entry in indicator.sources
+                        if 'source_name' in entry
+                    }
+                )
             )
             if categories := list(
                 {
@@ -249,7 +255,7 @@ def map_mandiant_indicator_to_zeek(
                 tags.extend(trueMispAttrs)
 
         if tags:
-            zeekItem[ZEEK_INTEL_CIF_TAGS] = ','.join(tags)
+            zeekItem[ZEEK_INTEL_CIF_TAGS] = ','.join([x.replace(',', '\\x2c') for x in tags])
 
         # The MD5Indicator class can actually have multiple types of hashes,
         #   and we want to create a zeek intel item for each. I'm accessing

From d970abaa8061144aa32a1ffb4be20a72819a31a1 Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Fri, 8 Nov 2024 11:42:26 -0700
Subject: [PATCH 22/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358

---
 config/zeek.env.example                                       | 4 ++--
 docs/capabilities-and-limitations.md                          | 2 +-
 docs/malcolm-config.md                                        | 2 +-
 docs/zeek-intel.md                                            | 2 +-
 .../config/hooks/normal/0169-pip-installs.hook.chroot         | 1 +
 kubernetes/10-zeek.yml                                        | 2 +-
 kubernetes/21-zeek-live.yml                                   | 2 +-
 scripts/control.py                                            | 2 +-
 shared/bin/zeek_intel_setup.sh                                | 2 +-
 9 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/config/zeek.env.example b/config/zeek.env.example
index 1ca4e171b..5c69531bf 100644
--- a/config/zeek.env.example
+++ b/config/zeek.env.example
@@ -7,11 +7,11 @@ ZEEK_LOCAL_NETS=
 ZEEK_JSON=
 # Specifies the value for Zeek's Intel::item_expiration timeout (-1min to disable)
 ZEEK_INTEL_ITEM_EXPIRATION=-1min
-# When querying a TAXII or MISP feed, only process threat indicators that have
+# When querying a threat intelligence feed, only process threat indicators that have
 #   been created or modified since the time represented by this value;
 #   it may be either a fixed date/time (01/01/2021) or relative interval (30 days ago)
 ZEEK_INTEL_FEED_SINCE=
-# Whether or not to require SSL certificate verification when querying a TAXII or MISP feed
+# Whether or not to require SSL certificate verification when querying an intelligence feed
 ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION=false
 # Number of threads to use for querying feeds for generating Zeek Intelligence Framework files
 ZEEK_INTEL_REFRESH_THREADS=2
diff --git a/docs/capabilities-and-limitations.md b/docs/capabilities-and-limitations.md
index 7b13a9c29..9edb33f35 100644
--- a/docs/capabilities-and-limitations.md
+++ b/docs/capabilities-and-limitations.md
@@ -45,7 +45,7 @@ In short, Malcolm provides an easily deployable traffic analysis tool suite for
     - Limitation: Anomaly detection and machine learning algorithms rely on enough data (for network data, this generally means at least several weeks' worth or more) to be able to build a baseline of what is normal before they can accurately flag anomalies, and each network is different. Anomaly detection and ML are typically not useful for limited deployments without the available traffic to build that baseline.
     - Limitation: While Malcolm provides some powerful tools in the anomaly detection and ML realm, as of yet they have not been built out to provide the value that they will probably one day realize.
 * Threat ingestion
-    - Malcolm can ingest threat indicators in the form of static MISP- or STIX-formatted files. It can also subscribe to and periodically update threat indicators from [MISP](zeek-intel.md#ZeekIntelMISP) and [TAXII](zeek-intel.md#ZeekIntelSTIX) feeds. These indicators are converted into a format that is read by Zeek, and matches in network traffic are [surfaced through the Zeek intelligence framework](zeek-intel.md#ZeekIntel) for logging.
+    - Malcolm can ingest threat indicators in the form of static MISP- or STIX-formatted files. It can also subscribe to and periodically update threat indicators from [MISP](zeek-intel.md#ZeekIntelMISP), [TAXII](zeek-intel.md#ZeekIntelSTIX), and [Mandiant](zeek-intel.md#ZeekIntelMandiant) feeds. These indicators are converted into a format that is read by Zeek, and matches in network traffic are [surfaced through the Zeek intelligence framework](zeek-intel.md#ZeekIntel) for logging.
     - Limitation: Some formats for threat indicators allow for complex definitions and logic. For STIX/TAXII, only indicators of cyber-observable objects matched with the equals (=) comparison operator against a single value can be expressed as Zeek intelligence items. Similarly, only a subset of MISP attribute types can be expressed with the Zeek intelligence indicator types. While this is generally sufficient to cover most indicators interest, more complex indicators are silently ignored.
 * Network Modeling
     - Malcolm provides an instance of [NetBox](https://netboxlabs.com/oss/netbox/), an open-source "solution for modeling and documenting modern networks" which is used to model instrumented networks and enrich passively-observed network traffic from that model, a technique Malcolm calls ["Asset Interaction Analysis"](asset-interaction-analysis.md#AssetInteractionAnalysis). Users can pivot between the network visualization tools (the Asset Interaction Analysis and Zeek Known Summary dashboards in OpenSearch Dashboards, and the Arkime Sessions interface) and the NetBox UI to investigate and examine network assets.
diff --git a/docs/malcolm-config.md b/docs/malcolm-config.md
index b598dc06e..55e2b7b96 100644
--- a/docs/malcolm-config.md
+++ b/docs/malcolm-config.md
@@ -127,7 +127,7 @@ Although the configuration script automates many of the following configuration
     - `ZEEK_DISABLE_ICS_ALL` and `ZEEK_DISABLE_ICS_…` - if set to `true`, these variables can be used to disable Zeek's protocol analyzers for Operational Technology/Industrial Control Systems (OT/ICS) protocols
     - `ZEEK_DISABLE_BEST_GUESS_ICS` - see ["Best Guess" Fingerprinting for ICS Protocols](ics-best-guess.md#ICSBestGuess)
     - `ZEEK_EXTRACTOR_MODE` – determines the file extraction behavior for file transfers detected by Zeek; see [Automatic file extraction and scanning](file-scanning.md#ZeekFileExtraction) for more details
-    - `ZEEK_INTEL_FEED_SINCE` - when querying a [TAXII](zeek-intel.md#ZeekIntelSTIX) or [MISP](zeek-intel.md#ZeekIntelMISP) feed, only process threat indicators created or modified since the time represented by this value; it may be either a fixed date/time (`01/01/2021`) or relative interval (`30 days ago`)
+    - `ZEEK_INTEL_FEED_SINCE` - when querying a [TAXII](zeek-intel.md#ZeekIntelSTIX), [MISP](zeek-intel.md#ZeekIntelMISP), or [Mandiant](zeek-intel.md#ZeekIntelMandiant) threat intelligence feed, only process threat indicators created or modified since the time represented by this value; it may be either a fixed date/time (`01/01/2021`) or relative interval (`30 days ago`)
     - `ZEEK_INTEL_ITEM_EXPIRATION` - specifies the value for Zeek's [`Intel::item_expiration`](https://docs.zeek.org/en/current/scripts/base/frameworks/intel/main.zeek.html#id-Intel::item_expiration) timeout as used by the [Zeek Intelligence Framework](zeek-intel.md#ZeekIntel) (default `-1min`, which disables item expiration)
     - `ZEEK_INTEL_REFRESH_CRON_EXPRESSION` - specifies a [cron expression](https://en.wikipedia.org/wiki/Cron#CRON_expression) indicating the refresh interval for generating the [Zeek Intelligence Framework](zeek-intel.md#ZeekIntel) files (defaults to empty, which disables automatic refresh)
     - `ZEEK_JA4SSH_PACKET_COUNT` - the Zeek [JA4+ plugin](https://github.com/FoxIO-LLC/ja4) calculates the JA4SSH value once for every *x* SSH packets; *x* is set here (default `200`)
diff --git a/docs/zeek-intel.md b/docs/zeek-intel.md
index 468f9529c..bdcc16b67 100644
--- a/docs/zeek-intel.md
+++ b/docs/zeek-intel.md
@@ -10,7 +10,7 @@ To quote Zeek's [Intelligence Framework](https://docs.zeek.org/en/master/framewo
 
 Malcolm doesn't come bundled with intelligence files from any particular feed, but they can be easily included into a local instance. On [startup]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/zeek_intel_setup.sh), Malcolm's `ghcr.io/idaholab/malcolm/zeek` container enumerates the subdirectories under `./zeek/intel` (which is [bind mounted](https://docs.docker.com/storage/bind-mounts/) into the container's runtime) and configures Zeek so those intelligence files will be automatically included in its local policy. Subdirectories under `./zeek/intel` that contain their own `__load__.zeek` file will be `@load`-ed as-is, while subdirectories containing "loose" intelligence files will be [loaded](https://docs.zeek.org/en/master/frameworks/intel.html#loading-intelligence) automatically with a `redef Intel::read_files` directive.
 
-Note that Malcolm does not manage updates for these intelligence files. Users use the update mechanism suggested by the feeds' maintainers to keep intelligence files up to date, or use a [TAXII](#ZeekIntelSTIX) or [MISP](#ZeekIntelMISP) feed as described below.
+Note that Malcolm does not manage updates for these intelligence files. Users use the update mechanism suggested by the feeds' maintainers to keep intelligence files up to date, or use a [TAXII](#ZeekIntelSTIX), [MISP](#ZeekIntelMISP), or [Mandiant](#ZeekIntelMandiant) feed as described below.
 
 Adding and deleting intelligence files under this directory will take effect upon [restarting Malcolm](running.md#StopAndRestart). Alternately, users can use the `ZEEK_INTEL_REFRESH_CRON_EXPRESSION` environment variable containing a [cron expression](https://en.wikipedia.org/wiki/Cron#CRON_expression) to specify the interval at which the intel files should be refreshed. This can also be done manually without restarting Malcolm by running the following command from the Malcolm installation directory:
 
diff --git a/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot b/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot
index 4fdaec171..f5a69daf0 100755
--- a/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot
+++ b/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot
@@ -13,6 +13,7 @@ pip3 install --break-system-packages --no-compile --no-cache-dir --force-reinsta
   dateparser \
   debinterface \
   dominate \
+  git+https://github.com/google/mandiant-ti-client \
   humanfriendly \
   pymisp \
   python-dotenv \
diff --git a/kubernetes/10-zeek.yml b/kubernetes/10-zeek.yml
index c08a95774..00b366dd0 100644
--- a/kubernetes/10-zeek.yml
+++ b/kubernetes/10-zeek.yml
@@ -71,7 +71,7 @@ spec:
               name: process-env
         env:
           - name: PUSER_MKDIR
-            value: "/data/config:zeek/intel/MISP,zeek/intel/STIX;/data/pcap:processed;/data/zeek-logs:current,extract_files/preserved,extract_files/quarantine,live,processed,upload"
+            value: "/data/config:zeek/intel/Mandiant,zeek/intel/MISP,zeek/intel/STIX;/data/pcap:processed;/data/zeek-logs:current,extract_files/preserved,extract_files/quarantine,live,processed,upload"
         volumeMounts:
           - name: zeek-offline-intel-volume
             mountPath: "/data/config"
diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml
index 1997710fe..3045705ae 100644
--- a/kubernetes/21-zeek-live.yml
+++ b/kubernetes/21-zeek-live.yml
@@ -70,7 +70,7 @@ spec:
               name: process-env
         env:
           - name: PUSER_MKDIR
-            value: "/data/config:zeek/intel/MISP,zeek/intel/STIX;/data/zeek-logs:current,extract_files/preserved,extract_files/quarantine,live,processed,upload"
+            value: "/data/config:zeek/intel/Mandiant,zeek/intel/MISP,zeek/intel/STIX;/data/zeek-logs:current,extract_files/preserved,extract_files/quarantine,live,processed,upload"
         volumeMounts:
           - name: zeek-live-intel-volume
             mountPath: "/data/config"
diff --git a/scripts/control.py b/scripts/control.py
index 204d10d60..2430685c8 100755
--- a/scripts/control.py
+++ b/scripts/control.py
@@ -1072,7 +1072,7 @@ def start():
                 BoundPath("zeek", "/zeek/extract_files", False, None, None),
                 BoundPath("zeek", "/zeek/upload", False, None, None),
                 BoundPath("zeek", "/opt/zeek/share/zeek/site/custom", False, None, None),
-                BoundPath("zeek", "/opt/zeek/share/zeek/site/intel", False, ["MISP", "STIX"], None),
+                BoundPath("zeek", "/opt/zeek/share/zeek/site/intel", False, ["Mandiant", "MISP", "STIX"], None),
                 BoundPath("zeek-live", "/zeek/live", False, ["spool"], None),
                 BoundPath(
                     "filebeat", "/zeek", False, ["processed", "current", "live", "extract_files", "upload"], None
diff --git a/shared/bin/zeek_intel_setup.sh b/shared/bin/zeek_intel_setup.sh
index ca9dc1bb2..1d81ca9cc 100755
--- a/shared/bin/zeek_intel_setup.sh
+++ b/shared/bin/zeek_intel_setup.sh
@@ -95,7 +95,7 @@ EOF
             fi
         done
 
-        # process STIX and MISP inputs by converting them to Zeek intel format
+        # process STIX/MISP/Mandiant inputs by converting them to Zeek intel format
         if ( (( ${#THREAT_JSON_FILES[@]} )) || [[ -r ./STIX/.stix_input.txt ]] || [[ -r ./MISP/.misp_input.txt ]] || [[ -r ./Mandiant/mandiant.yaml ]] ) && [[ -x "${THREAT_FEED_TO_ZEEK_SCRIPT}" ]]; then
             "${THREAT_FEED_TO_ZEEK_SCRIPT}" \
                 --ssl-verify ${ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION} \

From e87053218d05e85f14ce3c09557aee16393086b1 Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Fri, 8 Nov 2024 13:01:02 -0700
Subject: [PATCH 23/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358

---
 logstash/pipelines/zeek/1029_zeek_intel.conf | 5 ++++-
 shared/bin/zeek_threat_feed_utils.py         | 6 +++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/logstash/pipelines/zeek/1029_zeek_intel.conf b/logstash/pipelines/zeek/1029_zeek_intel.conf
index 7bab04681..da376c60e 100644
--- a/logstash/pipelines/zeek/1029_zeek_intel.conf
+++ b/logstash/pipelines/zeek/1029_zeek_intel.conf
@@ -42,8 +42,11 @@ filter {
       }
     }
 
+    # "sources" is handled differently because of some weirdness between creating the meta.source field in the
+    #   intel file (which seems to be a string) and the sources field in intel.log (which is a set of string)
+    #   so we're doing our own | pseudo-delimiter to work around it.
     mutate { id => "mutate_split_zeek_intel_commas"
-             split => { "[zeek_cols][sources]" => ","
+             split => { "[zeek_cols][sources]" => "|"
                         "[zeek_cols][matched]" => ","
                         "[zeek_cols][cif_tags]" => "," } }
 
diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py
index 18d342757..04a746c73 100644
--- a/shared/bin/zeek_threat_feed_utils.py
+++ b/shared/bin/zeek_threat_feed_utils.py
@@ -231,7 +231,7 @@ def map_mandiant_indicator_to_zeek(
         if hasattr(indicator, 'last_seen'):
             zeekItem[ZEEK_INTEL_CIF_LASTSEEN] = str(mktime(indicator.last_seen.timetuple()))
         if hasattr(indicator, 'sources'):
-            zeekItem[ZEEK_INTEL_META_SOURCE] = ','.join(
+            zeekItem[ZEEK_INTEL_META_SOURCE] = '\\x7c'.join(
                 list(
                     {
                         entry['source_name'].replace(',', '\\x2c')
@@ -455,7 +455,7 @@ def map_stix_indicator_to_zeek(
         zeekItem = defaultdict(lambda: '-')
 
         zeekItem[ZEEK_INTEL_META_SOURCE] = (
-            ','.join([x.replace(',', '\\x2c') for x in source])
+            '\\x7c'.join([x.replace(',', '\\x2c') for x in source])
             if source is not None and len(source) > 0
             else str(indicator.id)
         )
@@ -539,7 +539,7 @@ def map_misp_attribute_to_zeek(
         zeekItem = defaultdict(lambda: '-')
 
         if source is not None and len(source) > 0:
-            zeekItem[ZEEK_INTEL_META_SOURCE] = ','.join([x.replace(',', '\\x2c') for x in source])
+            zeekItem[ZEEK_INTEL_META_SOURCE] = '\\x7c'.join([x.replace(',', '\\x2c') for x in source])
         if description is not None:
             zeekItem[ZEEK_INTEL_META_DESC] = description
         if url is not None:

From 76399a9e13164aacb2f41d0681a8ab1eed0b8421 Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Fri, 8 Nov 2024 15:10:15 -0700
Subject: [PATCH 24/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358; major remapping of the zeek.intel fields to ECS, some
 stuff may be busted

---
 arkime/etc/config.ini                         |  10 +-
 arkime/wise/source.zeeklogs.js                |   9 -
 .../36ed695f-edcc-47c1-b0ec-50d20c93ce0f.json |  16 +-
 .../templates/composable/component/zeek.json  |   9 -
 logstash/maps/zeek_intel_indicator_types.yaml |  11 +
 logstash/pipelines/zeek/1200_zeek_mutate.conf |  12 +-
 .../pipelines/zeek/1300_zeek_normalize.conf   | 214 +++++++++++++-----
 .../pipelines/zeek/1400_zeek_convert.conf     |  27 +++
 shared/bin/zeek_threat_feed_utils.py          |  24 +-
 9 files changed, 230 insertions(+), 102 deletions(-)
 create mode 100644 logstash/maps/zeek_intel_indicator_types.yaml

diff --git a/arkime/etc/config.ini b/arkime/etc/config.ini
index 78c93ac50..3af5c6c5e 100644
--- a/arkime/etc/config.ini
+++ b/arkime/etc/config.ini
@@ -1237,19 +1237,11 @@ zeek.http.server_header_names=db:zeek.http.server_header_names;group:zeek_http;k
 
 # intel.log
 # https://docs.zeek.org/en/stable/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Info
-zeek.intel.seen_indicator=db:zeek.intel.seen_indicator;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:Indicator;help:Indicator
-zeek.intel.seen_indicator_type=db:zeek.intel.seen_indicator_type;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:Indicator Type;help:Indicator Type
 zeek.intel.seen_where=db:zeek.intel.seen_where;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:Where Discovered;help:Where Discovered
 zeek.intel.seen_node=db:zeek.intel.seen_node;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:Discovered Node;help:Discovered Node
-zeek.intel.matched=db:zeek.intel.matched;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:Match Indicator;help:Match Indicator
-zeek.intel.sources=db:zeek.intel.sources;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:Match Source;help:Match Source
 zeek.intel.file_mime_type=db:zeek.intel.file_mime_type;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:File Magic;help:File Magic
 zeek.intel.file_description=db:zeek.intel.file_description;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:File Description;help:File Description
-zeek.intel.cif_tags=db:zeek.intel.cif_tags;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:CIF Tags;help:CIF Tags
 zeek.intel.cif_confidence=db:zeek.intel.cif_confidence;group:zeek_intel;kind:float;viewerOnly:true;friendly:CIF Confidence;help:CIF Confidence
-zeek.intel.cif_source=db:zeek.intel.cif_source;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:CIF Source;help:CIF Source
-zeek.intel.cif_description=db:zeek.intel.cif_description;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:CIF Description;help:CIF Description
-zeek.intel.cif_firstseen=db:zeek.intel.cif_firstseen;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:CIF First Seen;help:CIF First Seen
 zeek.intel.cif_lastseen=db:zeek.intel.cif_lastseen;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:CIF Last Seen;help:CIF Last Seen
 
 # ipsec.log
@@ -3344,7 +3336,7 @@ o_zeek_hart_ip=require:zeek.hart_ip;title:Zeek hart_ip.log;fields:zeek.hart_ip.c
 o_zeek_hart_ip_session_record=require:zeek.hart_ip_session_record;title:Zeek hart_ip_session_record.log;fields:zeek.hart_ip.session_log_record_link_id,zeek.hart_ip_session_record.session_log_record_client_i_pv4_address,zeek.hart_ip_session_record.session_log_record_client_i_pv6_address,zeek.hart_ip_session_record.session_log_record_client_port,zeek.hart_ip_session_record.session_log_record_server_port,zeek.hart_ip_session_record.session_log_record_connect_time,zeek.hart_ip_session_record.session_log_record_disconnect_time,zeek.hart_ip_session_record.session_log_record_session_status_summary_undefined_bits,zeek.hart_ip_session_record.session_log_record_session_status_summary_insecure_session,zeek.hart_ip_session_record.session_log_record_session_status_summary_session_timeout,zeek.hart_ip_session_record.session_log_record_session_status_summary_aborted_session,zeek.hart_ip_session_record.session_log_record_session_status_summary_bad_session_initialization,zeek.hart_ip_session_record.session_log_record_session_status_summary_writes_occured,zeek.hart_ip_session_record.session_log_record_start_configuration_change_count,zeek.hart_ip_session_record.session_log_record_end_configuration_change_count,zeek.hart_ip_session_record.session_log_record_num_publish_pdu,zeek.hart_ip_session_record.session_log_record_num_request_pdu,zeek.hart_ip_session_record.session_log_record_num_response_pdu
 o_zeek_hart_ip_universal_commands=require:zeek.hart_ip_universal_commands;title:Zeek hart_ip_universal_commands.log;fields:zeek.hart_ip.command_number_link_id,zeek.hart_ip_universal_commands.read_unique_identifier_response_254,zeek.hart_ip_universal_commands.read_unique_identifier_response_expanded_device_type,zeek.hart_ip_universal_commands.read_unique_identifier_response_minimum_preambles_master_slave,zeek.hart_ip_universal_commands.read_unique_identifier_response_hart_protocol_major_revision,zeek.hart_ip_universal_commands.read_unique_identifier_response_device_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_response_software_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_response_hardware_revision_level_and_physical_signaling_codes_hardware_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_response_hardware_revision_level_and_physical_signaling_codes_physical_signaling_code,zeek.hart_ip_universal_commands.read_unique_identifier_response_flags_c8_psk_in_multi_drop_only,zeek.hart_ip_universal_commands.read_unique_identifier_response_flags_c8_psk_capable_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_response_flags_undefined_5,zeek.hart_ip_universal_commands.read_unique_identifier_response_flags_safehart_capable_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_response_flags_ieee_802_15_4_dsss_o_qpsk_modulation,zeek.hart_ip_universal_commands.read_unique_identifier_response_flags_protocol_bridge_device,zeek.hart_ip_universal_commands.read_unique_identifier_response_flags_eeprom_control,zeek.hart_ip_universal_commands.read_unique_identifier_response_flags_mutli_sensor_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_response_device_id,zeek.hart_ip_universal_commands.read_unique_identifier_response_number_preambles_slave_master,zeek.hart_ip_universal_commands.read_unique_identifier_response_last_device_variable_this,zeek.hart_ip_universal_commands.read_unique_identifier_response_configuration_change_counter,zeek.hart_ip_universal_commands.read_unique_identifier_response_extended_field_device_status_undefined_bits,zeek.hart_ip_universal_commands.read_unique_identifier_response_extended_field_device_status_function_check,zeek.hart_ip_universal_commands.read_unique_identifier_response_extended_field_device_status_out_of_specification,zeek.hart_ip_universal_commands.read_unique_identifier_response_extended_field_device_status_failure,zeek.hart_ip_universal_commands.read_unique_identifier_response_extended_field_device_status_critical_power_failure,zeek.hart_ip_universal_commands.read_unique_identifier_response_extended_field_device_status_device_variable_alert,zeek.hart_ip_universal_commands.read_unique_identifier_response_extended_field_device_status_maintenance_required,zeek.hart_ip_universal_commands.read_unique_identifier_response_manufacturer_identification_code,zeek.hart_ip_universal_commands.read_unique_identifier_response_private_label_distributor_code,zeek.hart_ip_universal_commands.read_unique_identifier_response_device_profile,zeek.hart_ip_universal_commands.read_primary_variable_response_primary_variable_units,zeek.hart_ip_universal_commands.read_primary_variable_response_primary_variable,zeek.hart_ip_universal_commands.read_loop_current_response_primary_variable_loop_current,zeek.hart_ip_universal_commands.read_loop_current_response_primary_variable_percent_range,zeek.hart_ip_universal_commands.read_dynamic_variable_response_primary_variable_loop_current,zeek.hart_ip_universal_commands.read_dynamic_variable_response_primary_variable_units,zeek.hart_ip_universal_commands.read_dynamic_variable_response_primary_variable,zeek.hart_ip_universal_commands.read_dynamic_variable_response_secondary_variable_units,zeek.hart_ip_universal_commands.read_dynamic_variable_response_secondary_variable,zeek.hart_ip_universal_commands.read_dynamic_variable_response_tertiary_variable_units,zeek.hart_ip_universal_commands.read_dynamic_variable_response_tertiary_variable,zeek.hart_ip_universal_commands.read_dynamic_variable_response_quaternary_variable_units,zeek.hart_ip_universal_commands.read_dynamic_variable_response_quaternary_variable,zeek.hart_ip_universal_commands.write_polling_address_polling_address_device,zeek.hart_ip_universal_commands.write_polling_address_loop_current_mode,zeek.hart_ip_universal_commands.read_loop_configuration_response_polling_address_device,zeek.hart_ip_universal_commands.read_loop_configuration_response_loop_current_mode,zeek.hart_ip_universal_commands.read_dynamic_variable_classifications_response_primary_variable_classification,zeek.hart_ip_universal_commands.read_dynamic_variable_classifications_response_secondary_variable_classification,zeek.hart_ip_universal_commands.read_dynamic_variable_classifications_response_tertiary_variable_classification,zeek.hart_ip_universal_commands.read_dynamic_variable_classifications_response_quaternary_variable_classification,zeek.hart_ip_universal_commands.read_device_variable_request_slot0_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_request_slot1_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_request_slot2_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_request_slot3_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_request_slot4_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_request_slot5_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_request_slot6_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_request_slot7_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_extended_field_device_status_undefined_bits,zeek.hart_ip_universal_commands.read_device_variable_response_extended_field_device_status_function_check,zeek.hart_ip_universal_commands.read_device_variable_response_extended_field_device_status_out_of_specification,zeek.hart_ip_universal_commands.read_device_variable_response_extended_field_device_status_failure,zeek.hart_ip_universal_commands.read_device_variable_response_extended_field_device_status_critical_power_failure,zeek.hart_ip_universal_commands.read_device_variable_response_extended_field_device_status_device_variable_alert,zeek.hart_ip_universal_commands.read_device_variable_response_extended_field_device_status_maintenance_required,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_device_variable_class,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_units_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_device_variable,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_device_variable_status_process_data_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_device_variable_status_limit_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_device_variable_status_more_device_variable_status_available,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_device_variable_status_device_family_specific_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot1_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot1_device_variable_class,zeek.hart_ip_universal_commands.read_device_variable_response_slot1_units_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot1_device_variable,zeek.hart_ip_universal_commands.read_device_variable_response_slot1_device_variable_status_process_data_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot1_device_variable_status_limit_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot1_device_variable_status_more_device_variable_status_available,zeek.hart_ip_universal_commands.read_device_variable_response_slot1_device_variable_status_device_family_specific_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot2_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot2_device_variable_class,zeek.hart_ip_universal_commands.read_device_variable_response_slot2_units_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot2_device_variable,zeek.hart_ip_universal_commands.read_device_variable_response_slot2_device_variable_status_process_data_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot2_device_variable_status_limit_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot2_device_variable_status_more_device_variable_status_available,zeek.hart_ip_universal_commands.read_device_variable_response_slot2_device_variable_status_device_family_specific_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot3_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot3_device_variable_class,zeek.hart_ip_universal_commands.read_device_variable_response_slot3_units_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot3_device_variable,zeek.hart_ip_universal_commands.read_device_variable_response_slot3_device_variable_status_process_data_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot3_device_variable_status_limit_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot3_device_variable_status_more_device_variable_status_available,zeek.hart_ip_universal_commands.read_device_variable_response_slot3_device_variable_status_device_family_specific_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot4_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot4_device_variable_class,zeek.hart_ip_universal_commands.read_device_variable_response_slot4_units_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot4_device_variable,zeek.hart_ip_universal_commands.read_device_variable_response_slot4_device_variable_status_process_data_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot4_device_variable_status_limit_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot4_device_variable_status_more_device_variable_status_available,zeek.hart_ip_universal_commands.read_device_variable_response_slot4_device_variable_status_device_family_specific_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot5_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot5_device_variable_class,zeek.hart_ip_universal_commands.read_device_variable_response_slot5_units_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot5_device_variable,zeek.hart_ip_universal_commands.read_device_variable_response_slot5_device_variable_status_process_data_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot5_device_variable_status_limit_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot5_device_variable_status_more_device_variable_status_available,zeek.hart_ip_universal_commands.read_device_variable_response_slot5_device_variable_status_device_family_specific_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot6_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot6_device_variable_class,zeek.hart_ip_universal_commands.read_device_variable_response_slot6_units_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot6_device_variable,zeek.hart_ip_universal_commands.read_device_variable_response_slot6_device_variable_status_process_data_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot6_device_variable_status_limit_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot6_device_variable_status_more_device_variable_status_available,zeek.hart_ip_universal_commands.read_device_variable_response_slot6_device_variable_status_device_family_specific_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot7_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot7_device_variable_class,zeek.hart_ip_universal_commands.read_device_variable_response_slot7_units_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot7_device_variable,zeek.hart_ip_universal_commands.read_device_variable_response_slot7_device_variable_status_process_data_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot7_device_variable_status_limit_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot7_device_variable_status_more_device_variable_status_available,zeek.hart_ip_universal_commands.read_device_variable_response_slot7_device_variable_status_device_family_specific_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_time,zeek.hart_ip_universal_commands.read_unique_identifier_tag_request_tag,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_254,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_expanded_device_type,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_minimum_preambles_master_slave,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_hart_protocol_major_revision,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_device_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_software_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_hardware_revision_level_and_physical_signaling_codes_hardware_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_hardware_revision_level_and_physical_signaling_codes_physical_signaling_code,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_flags_c8_psk_in_multi_drop_only,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_flags_c8_psk_capable_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_flags_undefined_5,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_flags_safehart_capable_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_flags_ieee_802_15_4_dsss_o_qpsk_modulation,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_flags_protocol_bridge_device,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_flags_eeprom_control,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_flags_mutli_sensor_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_device_id,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_number_preambles_slave_master,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_last_device_variable_this,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_configuration_change_counter,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_extended_field_device_status_undefined_bits,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_extended_field_device_status_function_check,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_extended_field_device_status_out_of_specification,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_extended_field_device_status_failure,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_extended_field_device_status_critical_power_failure,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_extended_field_device_status_device_variable_alert,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_extended_field_device_status_maintenance_required,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_manufacturer_identification_code,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_private_label_distributor_code,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_device_profile,zeek.hart_ip_universal_commands.read_message_response_message,zeek.hart_ip_universal_commands.read_tag_response_tag,zeek.hart_ip_universal_commands.read_tag_response_descriptor,zeek.hart_ip_universal_commands.read_tag_response_date_code,zeek.hart_ip_universal_commands.read_primary_variable_transducer_information_response_p_v_transducer_serial_number,zeek.hart_ip_universal_commands.read_primary_variable_transducer_information_response_p_v_transducer_limits_units,zeek.hart_ip_universal_commands.read_primary_variable_transducer_information_response_p_v_upper_transducer_limit,zeek.hart_ip_universal_commands.read_primary_variable_transducer_information_response_p_v_lower_transducer_limit,zeek.hart_ip_universal_commands.read_primary_variable_transducer_information_response_p_v_minimum_span,zeek.hart_ip_universal_commands.read_device_information_response_p_v_alarm_selection_code,zeek.hart_ip_universal_commands.read_device_information_response_p_v_transfer_function_code,zeek.hart_ip_universal_commands.read_device_information_response_p_v_upper_lower_range,zeek.hart_ip_universal_commands.read_device_information_response_p_v_upper_range_value,zeek.hart_ip_universal_commands.read_device_information_response_p_v_lower_range_value,zeek.hart_ip_universal_commands.read_device_information_response_p_v_damping_value,zeek.hart_ip_universal_commands.read_device_information_response_write_protect_code,zeek.hart_ip_universal_commands.read_device_information_response_250,zeek.hart_ip_universal_commands.read_device_information_response_p_v_analog_channel_flags_undefined_bits,zeek.hart_ip_universal_commands.read_device_information_response_p_v_analog_channel_flags_analog_channel,zeek.hart_ip_universal_commands.read_final_assembly_number_response_final_assembly_number,zeek.hart_ip_universal_commands.write_message_message_string,zeek.hart_ip_universal_commands.write_tag_descriptor_date_tag,zeek.hart_ip_universal_commands.write_tag_descriptor_date_record_keeping_descriptor,zeek.hart_ip_universal_commands.write_tag_descriptor_date_date_code,zeek.hart_ip_universal_commands.write_final_assembly_number_final_assembly_number,zeek.hart_ip_universal_commands.read_long_tag_response_long_tag,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_request_long_tag,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_254,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_expanded_device_type,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_minimum_preambles_master_slave,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_hart_protocol_major_revision,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_device_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_software_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_hardware_revision_level_and_physical_signaling_codes_hardware_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_hardware_revision_level_and_physical_signaling_codes_physical_signaling_code,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_flags_c8_psk_in_multi_drop_only,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_flags_c8_psk_capable_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_flags_undefined_5,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_flags_safehart_capable_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_flags_ieee_802_15_4_dsss_o_qpsk_modulation,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_flags_protocol_bridge_device,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_flags_eeprom_control,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_flags_mutli_sensor_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_device_id,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_number_preambles_slave_master,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_last_device_variable_this,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_configuration_change_counter,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_extended_field_device_status_undefined_bits,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_extended_field_device_status_function_check,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_extended_field_device_status_out_of_specification,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_extended_field_device_status_failure,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_extended_field_device_status_critical_power_failure,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_extended_field_device_status_device_variable_alert,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_extended_field_device_status_maintenance_required,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_manufacturer_identification_code,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_private_label_distributor_code,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_device_profile,zeek.hart_ip_universal_commands.write_long_tag_long_tag,zeek.hart_ip_universal_commands.reset_configuration_changed_flag_configuration_change_counter,zeek.hart_ip_universal_commands.read_additional_device_status_contents_device_specific_status_0,zeek.hart_ip_universal_commands.read_additional_device_status_contents_extended_field_device_status_undefined_bits,zeek.hart_ip_universal_commands.read_additional_device_status_contents_extended_field_device_status_function_check,zeek.hart_ip_universal_commands.read_additional_device_status_contents_extended_field_device_status_out_of_specification,zeek.hart_ip_universal_commands.read_additional_device_status_contents_extended_field_device_status_failure,zeek.hart_ip_universal_commands.read_additional_device_status_contents_extended_field_device_status_critical_power_failure,zeek.hart_ip_universal_commands.read_additional_device_status_contents_extended_field_device_status_device_variable_alert,zeek.hart_ip_universal_commands.read_additional_device_status_contents_extended_field_device_status_maintenance_required,zeek.hart_ip_universal_commands.read_additional_device_status_contents_device_operating_mode,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status0_device_configuration_lock,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status0_electronic_defect,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status0_environmental_conditions_out_of_range,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status0_power_supply_conditions_out_of_range,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status0_watchdog_reset_executed,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status0_volatile_memory_defect,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status0_non_volatile_memory_defect,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status0_device_variable_simulation_active,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status1_undefined_bits,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status1_reserved,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status1_battery_or_power_supply_needs_maintenance,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status1_event_notification_overflow,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status1_discrete_variable_simulation_active,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status1_status_simulation_active,zeek.hart_ip_universal_commands.read_additional_device_status_contents_analog_channel_saturated_undefined_bits,zeek.hart_ip_universal_commands.read_additional_device_status_contents_analog_channel_saturated_quinary_analog,zeek.hart_ip_universal_commands.read_additional_device_status_contents_analog_channel_saturated_quaternary_analog,zeek.hart_ip_universal_commands.read_additional_device_status_contents_analog_channel_saturated_tertiary_analog,zeek.hart_ip_universal_commands.read_additional_device_status_contents_analog_channel_saturated_secondary_analog,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status2_undefined_bits,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status2_stale_data_notice,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status2_sub_device_with_duplicate_id,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status2_sub_device_mismatch,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status2_duplicate_master_detected,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status2_sub_device_list_changed,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status3_undefined_bits,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status3_radio_failure,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status3_block_transfer_pending,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status3_bandwith_allocation_pending,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status3_resereved,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status3_capacity_denied,zeek.hart_ip_universal_commands.read_additional_device_status_contents_analog_channel_undefined_bits,zeek.hart_ip_universal_commands.read_additional_device_status_contents_analog_channel_analog_channel,zeek.hart_ip_universal_commands.read_additional_device_status_contents_device_specific_status_1
 o_zeek_http=require:zeek.http;title:Zeek http.log;fields:zeek.http.trans_depth,zeek.http.method,zeek.http.host,zeek.http.uri,zeek.http.origin,zeek.http.post_password_plain,zeek.http.post_username,zeek.http.referrer,zeek.http.version,zeek.http.user_agent,zeek.http.request_body_len,zeek.http.response_body_len,zeek.http.status_code,zeek.http.status_msg,zeek.http.info_code,zeek.http.info_msg,zeek.http.tags,zeek.http.proxied,zeek.http.orig_fuids,zeek.http.orig_filenames,zeek.http.orig_mime_types,zeek.http.resp_fuids,zeek.http.resp_filenames,zeek.http.resp_mime_types,zeek.http.ja4h,zeek.http.client_header_names,zeek.http.server_header_names
-o_zeek_intel=require:zeek.intel;title:Zeek intel.log;fields:zeek.intel.cif_tags,zeek.intel.cif_confidence,zeek.intel.cif_source,zeek.intel.cif_description,zeek.intel.cif_firstseen,zeek.intel.cif_lastseen,zeek.intel.seen_indicator,zeek.intel.seen_indicator_type,zeek.intel.seen_where,zeek.intel.seen_node,zeek.intel.matched,zeek.intel.sources,zeek.intel.file_mime_type,zeek.intel.file_description
+o_zeek_intel=require:zeek.intel;title:Zeek intel.log;fields:tags,zeek.intel.cif_confidence,threat.indicator.provider,threat.indicator.description,threat.indicator.first_seen,threat.indicator.last_seen,threat.indicator.name,threat.indicator.type,zeek.intel.seen_where,zeek.intel.seen_node,zeek.intel.file_mime_type,zeek.intel.file_description
 o_zeek_ipsec=require:zeek.ipsec;title:Zeek ipsec.log;fields:zeek.ipsec.initiator_spi,zeek.ipsec.responder_spi,zeek.ipsec.maj_ver,zeek.ipsec.min_ver,zeek.ipsec.exchange_type,zeek.ipsec.doi,zeek.ipsec.flag_e,zeek.ipsec.flag_c,zeek.ipsec.flag_a,zeek.ipsec.flag_i,zeek.ipsec.flag_v,zeek.ipsec.flag_r,zeek.ipsec.flags,zeek.ipsec.message_id,zeek.ipsec.vendor_ids,zeek.ipsec.notify_messages,zeek.ipsec.transforms,zeek.ipsec.ke_dh_groups,zeek.ipsec.proposals,zeek.ipsec.protocol_id,zeek.ipsec.certificates,zeek.ipsec.situation,zeek.ipsec.transform_attributes,zeek.ipsec.length,zeek.ipsec.hash
 o_zeek_irc=require:zeek.irc;title:Zeek irc.log;fields:zeek.irc.nick,zeek.irc.command,zeek.irc.value,zeek.irc.addl,zeek.irc.dcc_file_name,zeek.irc.dcc_file_size,zeek.irc.dcc_mime_type
 o_zeek_kerberos=require:zeek.kerberos;title:Zeek kerberos.log;fields:zeek.kerberos.cname,zeek.kerberos.sname,zeek.kerberos.success,zeek.kerberos.error_msg,zeek.kerberos.from,zeek.kerberos.till,zeek.kerberos.cipher,zeek.kerberos.forwardable,zeek.kerberos.renewable,zeek.kerberos.request_type,zeek.kerberos.client_cert_subject,zeek.kerberos.client_cert_fuid,zeek.kerberos.server_cert_subject,zeek.kerberos.server_cert_fuid
diff --git a/arkime/wise/source.zeeklogs.js b/arkime/wise/source.zeeklogs.js
index 33e520ca9..134af896a 100644
--- a/arkime/wise/source.zeeklogs.js
+++ b/arkime/wise/source.zeeklogs.js
@@ -1666,19 +1666,10 @@ class MalcolmSource extends WISESource {
       "zeek.http.user_agent",
       "zeek.http.version",
       "zeek.intel.cif_confidence",
-      "zeek.intel.cif_description",
-      "zeek.intel.cif_firstseen",
-      "zeek.intel.cif_lastseen",
-      "zeek.intel.cif_source",
-      "zeek.intel.cif_tags",
       "zeek.intel.file_description",
       "zeek.intel.file_mime_type",
-      "zeek.intel.matched",
-      "zeek.intel.seen_indicator",
-      "zeek.intel.seen_indicator_type",
       "zeek.intel.seen_node",
       "zeek.intel.seen_where",
-      "zeek.intel.sources",
       "zeek.ipsec.certificates",
       "zeek.ipsec.doi",
       "zeek.ipsec.exchange_type",
diff --git a/dashboards/dashboards/36ed695f-edcc-47c1-b0ec-50d20c93ce0f.json b/dashboards/dashboards/36ed695f-edcc-47c1-b0ec-50d20c93ce0f.json
index d55123c06..dbe6f2015 100644
--- a/dashboards/dashboards/36ed695f-edcc-47c1-b0ec-50d20c93ce0f.json
+++ b/dashboards/dashboards/36ed695f-edcc-47c1-b0ec-50d20c93ce0f.json
@@ -13,7 +13,7 @@
         "title": "Zeek Intelligence",
         "hits": 0,
         "description": "",
-        "panelsJSON": "[{\"embeddableConfig\":{},\"gridData\":{\"h\":26,\"i\":\"2\",\"w\":8,\"x\":0,\"y\":0},\"panelIndex\":\"2\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{\"table\":null,\"vis\":{\"legendOpen\":false}},\"gridData\":{\"h\":8,\"i\":\"3\",\"w\":32,\"x\":16,\"y\":0},\"panelIndex\":\"3\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"5\",\"w\":16,\"x\":8,\"y\":8},\"panelIndex\":\"5\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_2\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":18,\"i\":\"6\",\"w\":15,\"x\":33,\"y\":26},\"panelIndex\":\"6\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_3\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":18,\"i\":\"7\",\"w\":13,\"x\":14,\"y\":44},\"panelIndex\":\"7\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_4\"},{\"embeddableConfig\":{\"table\":null,\"vis\":{\"params\":{\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":18,\"i\":\"8\",\"w\":21,\"x\":27,\"y\":44},\"panelIndex\":\"8\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_5\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"11\",\"w\":25,\"x\":8,\"y\":26},\"panelIndex\":\"11\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_6\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"12\",\"w\":14,\"x\":0,\"y\":44},\"panelIndex\":\"12\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_7\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":18,\"i\":\"13\",\"w\":8,\"x\":0,\"y\":26},\"panelIndex\":\"13\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_8\"},{\"embeddableConfig\":{\"columns\":[\"source.ip\",\"destination.ip\",\"destination.port\",\"zeek.intel.seen_indicator\",\"zeek.intel.seen_indicator_type\",\"zeek.intel.sources\",\"zeek.intel.seen_where\",\"event.id\"],\"sort\":[\"MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER\",\"desc\"]},\"gridData\":{\"h\":24,\"i\":\"14\",\"w\":48,\"x\":0,\"y\":62},\"panelIndex\":\"14\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_9\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"16\",\"w\":24,\"x\":24,\"y\":8},\"panelIndex\":\"16\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_10\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":8,\"i\":\"17\",\"w\":8,\"x\":8,\"y\":0},\"panelIndex\":\"17\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_11\"}]",
+        "panelsJSON": "[{\"embeddableConfig\":{},\"gridData\":{\"h\":26,\"i\":\"2\",\"w\":8,\"x\":0,\"y\":0},\"panelIndex\":\"2\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{\"table\":null,\"vis\":{\"legendOpen\":false}},\"gridData\":{\"h\":8,\"i\":\"3\",\"w\":32,\"x\":16,\"y\":0},\"panelIndex\":\"3\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"5\",\"w\":16,\"x\":8,\"y\":8},\"panelIndex\":\"5\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_2\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":18,\"i\":\"6\",\"w\":15,\"x\":33,\"y\":26},\"panelIndex\":\"6\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_3\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":18,\"i\":\"7\",\"w\":13,\"x\":14,\"y\":44},\"panelIndex\":\"7\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_4\"},{\"embeddableConfig\":{\"table\":null,\"vis\":{\"params\":{\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":18,\"i\":\"8\",\"w\":21,\"x\":27,\"y\":44},\"panelIndex\":\"8\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_5\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"11\",\"w\":25,\"x\":8,\"y\":26},\"panelIndex\":\"11\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_6\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"12\",\"w\":14,\"x\":0,\"y\":44},\"panelIndex\":\"12\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_7\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":18,\"i\":\"13\",\"w\":8,\"x\":0,\"y\":26},\"panelIndex\":\"13\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_8\"},{\"embeddableConfig\":{\"columns\":[\"source.ip\",\"destination.ip\",\"destination.port\",\"threat.indicator.name\",\"threat.indicator.type\",\"threat.indicator.provider\",\"zeek.intel.seen_where\",\"event.id\"],\"sort\":[\"MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER\",\"desc\"]},\"gridData\":{\"h\":24,\"i\":\"14\",\"w\":48,\"x\":0,\"y\":62},\"panelIndex\":\"14\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_9\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"16\",\"w\":24,\"x\":24,\"y\":8},\"panelIndex\":\"16\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_10\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":8,\"i\":\"17\",\"w\":8,\"x\":8,\"y\":0},\"panelIndex\":\"17\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_11\"}]",
         "optionsJSON": "{\"useMargins\":true}",
         "version": 1,
         "timeRestore": false,
@@ -179,7 +179,7 @@
       "updated_at": "2022-01-12T18:22:26.156Z",
       "version": "WzIzOSwxXQ==",
       "attributes": {
-        "visState": "{\"title\":\"Intel - Source\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"zeek.intel.sources\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Source\"}}],\"listeners\":{}}",
+        "visState": "{\"title\":\"Intel - Source\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"threat.indicator.provider\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Source\"}}],\"listeners\":{}}",
         "description": "",
         "title": "Intel - Source",
         "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
@@ -270,7 +270,7 @@
       "version": "WzI0MiwxXQ==",
       "attributes": {
         "title": "Intel - Indicator",
-        "visState": "{\"title\":\"Intel - Indicator\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.intel.seen_indicator_type\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Type\"},\"schema\":\"bucket\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.intel.seen_where\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Seen Where\"},\"schema\":\"bucket\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.intel.seen_indicator\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Indicator\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false,\"percentageCol\":\"\"}}",
+        "visState": "{\"title\":\"Intel - Indicator\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"threat.indicator.type\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Type\"},\"schema\":\"bucket\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.intel.seen_where\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Seen Where\"},\"schema\":\"bucket\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"threat.indicator.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Indicator\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false,\"percentageCol\":\"\"}}",
         "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":3,\"direction\":\"desc\"}}}}",
         "description": "",
         "version": 1,
@@ -329,7 +329,7 @@
       "updated_at": "2022-01-12T18:22:26.156Z",
       "version": "WzI0NCwxXQ==",
       "attributes": {
-        "visState": "{\"title\":\"Intel - Matched\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"zeek.intel.matched\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Type Matched\"}}],\"listeners\":{}}",
+        "visState": "{\"title\":\"Intel - Matched\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"threat.indicator.type\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Type Matched\"}}],\"listeners\":{}}",
         "description": "",
         "title": "Intel - Matched",
         "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
@@ -376,9 +376,9 @@
           "source.ip",
           "destination.ip",
           "destination.port",
-          "zeek.intel.seen_indicator",
-          "zeek.intel.seen_indicator_type",
-          "zeek.intel.sources",
+          "threat.indicator.name",
+          "threat.indicator.type",
+          "threat.indicator.provider",
           "zeek.intel.seen_where",
           "event.id"
         ]
@@ -404,7 +404,7 @@
       "version": "WzEwMTIsMV0=",
       "attributes": {
         "title": "Intel - Indicator Type",
-        "visState": "{\"title\":\"Intel - Indicator Type\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.intel.seen_indicator_type\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Indicator Type\"},\"schema\":\"segment\"}],\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100,\"rotate\":0},\"title\":{\"text\":\"Indicator Type\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"type\":\"histogram\",\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}",
+        "visState": "{\"title\":\"Intel - Indicator Type\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"threat.indicator.type\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Indicator Type\"},\"schema\":\"segment\"}],\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100,\"rotate\":0},\"title\":{\"text\":\"Indicator Type\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"type\":\"histogram\",\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}",
         "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}",
         "description": "",
         "version": 1,
diff --git a/dashboards/templates/composable/component/zeek.json b/dashboards/templates/composable/component/zeek.json
index 83b43e30e..d4fae78d4 100644
--- a/dashboards/templates/composable/component/zeek.json
+++ b/dashboards/templates/composable/component/zeek.json
@@ -139,19 +139,10 @@
         "zeek.http.user_agent": { "type": "keyword", "ignore_above": 256, "fields": { "text": { "type": "text" } } },
         "zeek.http.version": { "type": "keyword" },
         "zeek.intel.cif_confidence": { "type": "float" },
-        "zeek.intel.cif_description": { "type": "keyword" },
-        "zeek.intel.cif_firstseen": { "type": "keyword" },
-        "zeek.intel.cif_lastseen": { "type": "keyword" },
-        "zeek.intel.cif_source": { "type": "keyword" },
-        "zeek.intel.cif_tags": { "type": "keyword" },
         "zeek.intel.file_description": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text", "norms": false } } },
         "zeek.intel.file_mime_type": { "type": "keyword" },
-        "zeek.intel.matched": { "type": "keyword" },
-        "zeek.intel.seen_indicator": { "type": "keyword" },
-        "zeek.intel.seen_indicator_type": { "type": "keyword" },
         "zeek.intel.seen_node": { "type": "keyword" },
         "zeek.intel.seen_where": { "type": "keyword" },
-        "zeek.intel.sources": { "type": "keyword" },
         "zeek.ipsec.certificates": { "type": "keyword" },
         "zeek.ipsec.doi": { "type": "integer" },
         "zeek.ipsec.exchange_type": { "type": "integer" },
diff --git a/logstash/maps/zeek_intel_indicator_types.yaml b/logstash/maps/zeek_intel_indicator_types.yaml
new file mode 100644
index 000000000..5b2819693
--- /dev/null
+++ b/logstash/maps/zeek_intel_indicator_types.yaml
@@ -0,0 +1,11 @@
+"Intel::DOMAIN": "domain-name"
+"Intel::EMAIL": "email-addr"
+"Intel::FILE_NAME": "file"
+"Intel::FILE_HASH": "file"
+"Intel::ADDR": "ipv4-addr"
+"Intel::ADDR4": "ipv4-addr"
+"Intel::ADDR6": "ipv6-addr"
+"Intel::SOFTWARE": "software"
+"Intel::URL": "url"
+"Intel::USER_NAME": "user-account"
+"Intel::CERT_HASH": "x509-certificate"
\ No newline at end of file
diff --git a/logstash/pipelines/zeek/1200_zeek_mutate.conf b/logstash/pipelines/zeek/1200_zeek_mutate.conf
index 434e62eae..1cc6d83b3 100644
--- a/logstash/pipelines/zeek/1200_zeek_mutate.conf
+++ b/logstash/pipelines/zeek/1200_zeek_mutate.conf
@@ -1033,10 +1033,18 @@ filter {
         code => "
           if (sources = event.get('[zeek][intel][sources]')) and (sources.length > 0) then
             sourcesAdj = Array.new
+            indicatorRef = Array.new unless (indicatorRef = event.get('[threat][indicator][reference]'))
+            indicatorRef = indicatorRef.is_a?(Array) ? indicatorRef : [indicatorRef]
             sources.each do |val|
-              sourcesAdj.push(val.gsub('\\x2c', ','))
+              adjVal = val.gsub('\\x2c', ',')
+              if adjVal.match?(/^http?s:/i)
+                indicatorRef.push(adjVal)
+              else
+                sourcesAdj.push(adjVal)
+              end
             end
-            event.set('[zeek][intel][sources]', sourcesAdj)
+            event.set('[zeek][intel][sources]', sourcesAdj) unless (sourcesAdj.length == 0)
+            event.set('[threat][indicator][reference]', indicatorRef) unless (indicatorRef.length == 0)
           end
           if (tags = event.get('[zeek][intel][cif_tags]')) and (tags.length > 0) then
             tagsAdj = Array.new
diff --git a/logstash/pipelines/zeek/1300_zeek_normalize.conf b/logstash/pipelines/zeek/1300_zeek_normalize.conf
index c7b1c8bc2..c9a091b18 100644
--- a/logstash/pipelines/zeek/1300_zeek_normalize.conf
+++ b/logstash/pipelines/zeek/1300_zeek_normalize.conf
@@ -2,70 +2,174 @@
 
 filter {
 
-  # Zeek intel indicators depending on indicator type #################################################################
-  if ([zeek][intel][seen_indicator]) and ([zeek][intel][seen_indicator_type]) {
-
-    if ([zeek][intel][seen_indicator_type] == "Intel::ADDR") {
-      # ECS - ADDR-type intel indicators to related.ip
-      mutate { id => "mutate_merge_field_related_ip_zeek_intel_addr"
-               merge => { "[related][ip]" => "[zeek][intel][seen_indicator]" } }
-
-    } else if ([zeek][intel][seen_indicator_type] == "Intel::URL") {
-      # ECS - URL-type intel indicators to url.original
-      if (![url][original]) {
-        mutate { id => "mutate_merge_field_related_ip_zeek_intel_url"
-                 merge => { "[url][original]" => "[zeek][intel][seen_indicator]" } }
+  # most Zeek intel stuff maps to ecs-threat
+  #   https://www.elastic.co/guide/en/ecs/current/ecs-threat.html
+  if ([zeek][intel]) {
+
+    # map confidence (0-1) to Low/Medium/High/None
+    if ([zeek][intel][cif_confidence]) {
+      ruby {
+        id => "ruby_map_zeek_intel_confidence"
+        code => "
+          if (confNumVal = event.get('[zeek][intel][cif_confidence]').to_f) then
+            confStrVal = case value
+              when 0...3
+                'Low'
+              when 3...7
+                'Medium'
+              when 7..10
+                'High'
+              else
+                'None'
+              end
+            event.set('[threat][indicator][confidence]', confStrVal)"
+          end"
+      }
+    }
+
+    # normalize indicator type and indicator value for ECS
+    if ([zeek][intel][seen_indicator_type]) {
+
+      # store indicator value in ECS-mapped fields
+      if ([zeek][intel][seen_indicator]) {
+
+        if ([zeek][intel][seen_indicator_type] == "Intel::ADDR") {
+          # ECS - ADDR-type intel indicators to related.ip
+          mutate { id => "mutate_merge_field_related_ip_zeek_intel_addr"
+                   merge => { "[related][ip]" => "[zeek][intel][seen_indicator]" } }
+          # determine if the seen indicator is ipv4 or ipv6
+          cidr {
+            id      => "cidr_detect_network_type_ipv4_source"
+            add_field => { "[@metadata][intel_indicator_ip_type]" => "Intel::ADDR4" }
+            address => [ "%{[zeek][intel][seen_indicator]}" ]
+            network => [ "0.0.0.0/0" ]
+          }
+          if (![@metadata][intel_indicator_ip_type]) {
+            mutate { id => "cidr_detect_network_type_ipv6_source"
+                     add_field => { "[@metadata][intel_indicator_ip_type]" => "Intel::ADDR6" } }
+          }
+          mutate { id => "mutate_replace_zeek_intel_addr_indicator_type"
+                   replace => { "[zeek][intel][seen_indicator_type]" => "%{[@metadata][intel_indicator_ip_type]}" } }
+
+        } else if ([zeek][intel][seen_indicator_type] == "Intel::URL") {
+          # ECS - URL-type intel indicators to url.original
+          if (![url][original]) {
+            mutate { id => "mutate_merge_field_related_ip_zeek_intel_url"
+                     merge => { "[url][original]" => "[zeek][intel][seen_indicator]" } }
+          }
+
+        } else if ([zeek][intel][seen_indicator_type] == "Intel::DOMAIN") {
+          # ECS - DOMAIN-type intel indicators to related.hosts
+          mutate { id => "mutate_merge_field_related_ip_zeek_intel_domain"
+                   merge => { "[related][hosts]" => "[zeek][intel][seen_indicator]" } }
+
+        } else if ([zeek][intel][seen_indicator_type] == "Intel::USER_NAME") {
+          # ECS - USER_NAME-type intel indicators to related.user
+          mutate { id => "mutate_merge_field_related_ip_zeek_intel_user"
+                   merge => { "[related][user]" => "[zeek][intel][seen_indicator]" } }
+
+        } else if ([zeek][intel][seen_indicator_type] == "Intel::FILE_HASH") {
+          # ECS - FILE_HASH-type intel indicators to related.hash
+          mutate { id => "mutate_merge_field_related_ip_zeek_intel_hash"
+                   merge => { "[related][hash]" => "[zeek][intel][seen_indicator]" } }
+
+        } else if ([zeek][intel][seen_indicator_type] == "Intel::FILE_NAME") {
+          # ECS - FILE_NAME-type intel indicators to file.path
+          if (![file][path]) {
+            mutate { id => "mutate_merge_field_related_ip_zeek_intel_file_name"
+                     merge => { "[file][path]" => "[zeek][intel][seen_indicator]" } }
+          }
+        }
+
+        mutate { id => "mutate_suricata_rename_zeek_intel_seen_indicator"
+                 rename => { "[zeek][intel][seen_indicator]" => "[threat][indicator][name]" } }
+      } # seen_indicator
+
+      # map zeek intel indicator type from seen.indicator_type to ECS-specified values
+      #   https://www.elastic.co/guide/en/ecs/current/ecs-threat.html#field-threat-enrichments-indicator-type
+      translate {
+        id => "translate_intel_seen_indicator_type_seen"
+        source => "[zeek][intel][seen_indicator_type]"
+        target => "[@metadata][intel_indicator_type_seen]"
+        dictionary_path => "/etc/zeek_intel_indicator_types.yaml"
+        remove_field => [ "[zeek][intel][seen_indicator_type]" ]
       }
 
-    } else if ([zeek][intel][seen_indicator_type] == "Intel::DOMAIN") {
-      # ECS - DOMAIN-type intel indicators to related.hosts
-      mutate { id => "mutate_merge_field_related_ip_zeek_intel_domain"
-               merge => { "[related][hosts]" => "[zeek][intel][seen_indicator]" } }
-
-    } else if ([zeek][intel][seen_indicator_type] == "Intel::USER_NAME") {
-      # ECS - USER_NAME-type intel indicators to related.user
-      mutate { id => "mutate_merge_field_related_ip_zeek_intel_user"
-               merge => { "[related][user]" => "[zeek][intel][seen_indicator]" } }
-
-    } else if ([zeek][intel][seen_indicator_type] == "Intel::FILE_HASH") {
-      # ECS - FILE_HASH-type intel indicators to related.hash
-      mutate { id => "mutate_merge_field_related_ip_zeek_intel_hash"
-               merge => { "[related][hash]" => "[zeek][intel][seen_indicator]" } }
-
-    } else if ([zeek][intel][seen_indicator_type] == "Intel::FILE_NAME") {
-      # ECS - FILE_NAME-type intel indicators to file.path
-      if (![file][path]) {
-        mutate { id => "mutate_merge_field_related_ip_zeek_intel_file_name"
-                 merge => { "[file][path]" => "[zeek][intel][seen_indicator]" } }
+    } # seen_indicator_type
+
+    if ([zeek][intel][matched]) {
+      # map zeek intel indicator type from seen.matched to ECS-specified values
+      #   https://www.elastic.co/guide/en/ecs/current/ecs-threat.html#field-threat-enrichments-indicator-type
+      translate {
+        id => "translate_intel_seen_indicator_types_seen"
+        iterate_on => "[zeek][intel][matched]"
+        source => "[zeek][intel][matched]"
+        target => "[@metadata][intel_indicator_types_matched]"
+        dictionary_path => "/etc/zeek_intel_indicator_types.yaml"
+        remove_field => [ "[zeek][intel][matched]" ]
       }
     }
-  }
 
-  if ([zeek][intel][seen_where] =~ /^(DNS|HTTP|SMTP|SSL|SSH|SMB|X509)::/) and (![network][protocol]) {
-    # use zeel.intel.seen_where to map to service
-    # https://docs.zeek.org/en/master/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Wheres
-    grok {
-      id => "grok_zeek_intel_seen_where"
-      match => { "[zeek][intel][seen_where]" => [ "^%{DATA:[@metadata][intel_seen_where]}::%{GREEDYDATA:[@metadata][intel_seen_where_sub]}$" ] }
+    # merge indicator types found above to threat.indicator.type
+    if ([@metadata][intel_indicator_type_seen]) {
+      mutate { id => "mutate_merge_zeek_intel_indicator_type_seen"
+               merge => { "[threat][indicator][type]" => "[@metadata][intel_indicator_type_seen]" } }
     }
-    if ([@metadata][intel_seen_where]) {
-      translate {
-        id => "translate_zeek_intel_seen_where"
-        source => "[@metadata][intel_seen_where]"
-        target => "[network][protocol]"
-        dictionary => {
-          "DNS" => "dns"
-          "HTTP" => "http"
-          "SMTP" => "smtp"
-          "SSL" => "tls"
-          "SSH" => "ssh"
-          "SMB" => "smb"
-          "X509" => "X.509"
+    if ([@metadata][intel_indicator_types_matched]) {
+      mutate { id => "mutate_merge_zeek_intel_indicator_types_matched"
+               merge => { "[threat][indicator][type]" => "[@metadata][intel_indicator_types_matched]" } }
+    }
+
+    # zeek.intel.sources -> threat.indicator.provider
+    if ([zeek][intel][sources]) {
+      mutate { id => "mutate_merge_zeek_intel_sources"
+               merge => { "[threat][indicator][provider]" => "[zeek][intel][sources]" }
+               remove_field => [ "[zeek][intel][sources]" ] }
+    }
+    # zeek.intel.cif_source -> threat.indicator.provider
+    if ([zeek][intel][cif_source]) {
+      mutate { id => "mutate_merge_zeek_intel_cif_source"
+               merge => { "[threat][indicator][provider]" => "[zeek][intel][cif_source]" }
+               remove_field => [ "[zeek][intel][cif_source]" ] }
+    }
+    # zeek.intel.cif_tags -> tags
+    if ([zeek][intel][cif_tags]) {
+      mutate { id => "mutate_merge_zeek_intel_cif_tags"
+               merge => { "[tags]" => "[zeek][intel][cif_tags]" }
+               remove_field => [ "[zeek][intel][cif_tags]" ] }
+    }
+    # zeek.intel.cif_description -> threat.indicator.description
+    if ([zeek][intel][cif_description]) {
+      mutate { id => "mutate_suricata_rename_zeek_intel_cif_description"
+               rename => { "[zeek][intel][cif_description]" => "[threat][indicator][description]" } }
+    }
+
+    if ([zeek][intel][seen_where] =~ /^(DNS|HTTP|SMTP|SSL|SSH|SMB|X509)::/) and (![network][protocol]) {
+      # use zeel.intel.seen_where to map to service
+      #   https://docs.zeek.org/en/master/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Wheres
+      grok {
+        id => "grok_zeek_intel_seen_where"
+        match => { "[zeek][intel][seen_where]" => [ "^%{DATA:[@metadata][intel_seen_where]}::%{GREEDYDATA:[@metadata][intel_seen_where_sub]}$" ] }
+      }
+      if ([@metadata][intel_seen_where]) {
+        translate {
+          id => "translate_zeek_intel_seen_where"
+          source => "[@metadata][intel_seen_where]"
+          target => "[network][protocol]"
+          dictionary => {
+            "DNS" => "dns"
+            "HTTP" => "http"
+            "SMTP" => "smtp"
+            "SSL" => "tls"
+            "SSH" => "ssh"
+            "SMB" => "smb"
+            "X509" => "X.509"
+          }
+          fallback => "%{[@metadata][intel_seen_where]}"
         }
-        fallback => "%{[@metadata][intel_seen_where]}"
       }
     }
-  }
+  } # zeek intel
 
   # Protocol/service version ##########################################################################################
   # collect protocol version under the parent network.protocol_version array
diff --git a/logstash/pipelines/zeek/1400_zeek_convert.conf b/logstash/pipelines/zeek/1400_zeek_convert.conf
index 7fc747619..424e82e29 100644
--- a/logstash/pipelines/zeek/1400_zeek_convert.conf
+++ b/logstash/pipelines/zeek/1400_zeek_convert.conf
@@ -464,4 +464,31 @@ filter {
     }
   }
 
+  if ([zeek][intel][cif_firstseen]) {
+    if ([zeek][intel][cif_firstseen] == "0.000000") {
+      mutate { id => "mutate_remove_field_zeek_intel_cif_firstseen_zero"
+               remove_field => [ "[zeek][intel][cif_firstseen]" ] }
+    } else {
+      date {
+        id => "date_zeek_intel_cif_firstseen"
+        match => [ "[zeek][intel][cif_firstseen]", "UNIX" ]
+        target => "[threat][indicator][first_seen]"
+        remove_field => [ "[zeek][intel][cif_firstseen]"
+      }
+    }
+  }
+  if ([zeek][intel][cif_lastseen]) {
+    if ([zeek][intel][cif_lastseen] == "0.000000") {
+      mutate { id => "mutate_remove_field_zeek_intel_cif_lastseen_zero"
+               remove_field => [ "[zeek][intel][cif_lastseen]" ] }
+    } else {
+      date {
+        id => "date_zeek_intel_cif_lastseen"
+        match => [ "[zeek][intel][cif_lastseen]", "UNIX" ]
+        target => "[threat][indicator][last_seen]"
+        remove_field => [ "[zeek][intel][cif_lastseen]"
+      }
+    }
+  }
+
 }
\ No newline at end of file
diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py
index 04a746c73..7b4279dd5 100644
--- a/shared/bin/zeek_threat_feed_utils.py
+++ b/shared/bin/zeek_threat_feed_utils.py
@@ -218,11 +218,13 @@ def map_mandiant_indicator_to_zeek(
 
         zeekItem = defaultdict(lambda: '-')
         tags = []
+        sources = []
 
         zeekItem[ZEEK_INTEL_INDICATOR_TYPE] = "Intel::" + zeek_type
 
         if hasattr(indicator, 'id'):
             zeekItem[ZEEK_INTEL_META_DESC] = indicator.id
+            zeekItem[ZEEK_INTEL_CIF_DESCRIPTION] = zeekItem[ZEEK_INTEL_META_DESC]
             zeekItem[ZEEK_INTEL_META_URL] = f'https://advantage.mandiant.com/indicator/{indicator.id}'
         if hasattr(indicator, 'mscore'):
             zeekItem[ZEEK_INTEL_CIF_CONFIDENCE] = str(round(indicator.mscore / 10))
@@ -231,15 +233,7 @@ def map_mandiant_indicator_to_zeek(
         if hasattr(indicator, 'last_seen'):
             zeekItem[ZEEK_INTEL_CIF_LASTSEEN] = str(mktime(indicator.last_seen.timetuple()))
         if hasattr(indicator, 'sources'):
-            zeekItem[ZEEK_INTEL_META_SOURCE] = '\\x7c'.join(
-                list(
-                    {
-                        entry['source_name'].replace(',', '\\x2c')
-                        for entry in indicator.sources
-                        if 'source_name' in entry
-                    }
-                )
-            )
+            sources.extend(list({entry['source_name'] for entry in indicator.sources if 'source_name' in entry}))
             if categories := list(
                 {
                     category
@@ -276,6 +270,10 @@ def map_mandiant_indicator_to_zeek(
                     tmpItem[ZEEK_INTEL_INDICATOR] = hashVal
                     if newId := hashish.get('id', None):
                         tmpItem[ZEEK_INTEL_META_URL] = f'https://advantage.mandiant.com/indicator/{newId}'
+                    if ZEEK_INTEL_META_URL in tmpItem:
+                        sources.append(tmpItem[ZEEK_INTEL_META_URL])
+                    if sources:
+                        tmpItem[ZEEK_INTEL_META_SOURCE] = '\\x7c'.join([x.replace(',', '\\x2c') for x in sources])
                     results.append(tmpItem)
                     if (logger is not None) and (LOGGING_DEBUG >= logger.root.level):
                         logger.debug(tmpItem)
@@ -283,6 +281,10 @@ def map_mandiant_indicator_to_zeek(
         elif hasattr(indicator, 'value') and (val := indicator.value):
             # handle other types besides the file hash
             zeekItem[ZEEK_INTEL_INDICATOR] = val
+            if ZEEK_INTEL_META_URL in zeekItem:
+                sources.append(zeekItem[ZEEK_INTEL_META_URL])
+            if sources:
+                zeekItem[ZEEK_INTEL_META_SOURCE] = '\\x7c'.join([x.replace(',', '\\x2c') for x in sources])
             results.append(zeekItem)
             if (logger is not None) and (LOGGING_DEBUG >= logger.root.level):
                 logger.debug(zeekItem)
@@ -465,6 +467,7 @@ def map_stix_indicator_to_zeek(
             zeekItem[ZEEK_INTEL_META_DESC] = '. '.join(
                 [x for x in [indicator.get('name', None), indicator.get('description', None)] if x is not None]
             )
+            zeekItem[ZEEK_INTEL_CIF_DESCRIPTION] = zeekItem[ZEEK_INTEL_META_DESC]
             # some of these are from CFM, what the heck...
             # if 'description' in indicator:
             #   "description": "severity level: Low\n\nCONFIDENCE: High",
@@ -542,6 +545,7 @@ def map_misp_attribute_to_zeek(
             zeekItem[ZEEK_INTEL_META_SOURCE] = '\\x7c'.join([x.replace(',', '\\x2c') for x in source])
         if description is not None:
             zeekItem[ZEEK_INTEL_META_DESC] = description
+            zeekItem[ZEEK_INTEL_CIF_DESCRIPTION] = zeekItem[ZEEK_INTEL_META_DESC]
         if url is not None:
             zeekItem[ZEEK_INTEL_META_URL] = url
         zeekItem[ZEEK_INTEL_INDICATOR] = attribute_value
@@ -553,7 +557,7 @@ def map_misp_attribute_to_zeek(
         else:
             zeekItem[ZEEK_INTEL_CIF_TAGS] = attribute.category.replace(',', '\\x2c')
         if confidence is not None:
-            zeekItem[ZEEK_INTEL_CIF_CONFIDENCE] = str(confidence)
+            zeekItem[ZEEK_INTEL_CIF_CONFIDENCE] = str(round(confidence / 10))
 
         results.append(zeekItem)
         if (logger is not None) and (LOGGING_DEBUG >= logger.root.level):

From 5aa50ef51a230e2256c88a52604af025ad333aaf Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Fri, 8 Nov 2024 15:15:31 -0700
Subject: [PATCH 25/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358

---
 logstash/pipelines/zeek/1300_zeek_normalize.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/logstash/pipelines/zeek/1300_zeek_normalize.conf b/logstash/pipelines/zeek/1300_zeek_normalize.conf
index c9a091b18..d81d1bc80 100644
--- a/logstash/pipelines/zeek/1300_zeek_normalize.conf
+++ b/logstash/pipelines/zeek/1300_zeek_normalize.conf
@@ -17,7 +17,7 @@ filter {
                 'Low'
               when 3...7
                 'Medium'
-              when 7..10
+              when 7...10
                 'High'
               else
                 'None'

From 7b44cc632527415ce1f1cd47a30334be7ab52b15 Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Tue, 12 Nov 2024 07:43:39 -0700
Subject: [PATCH 26/42] bump arkime to v5.5.0

---
 Dockerfiles/arkime.Dockerfile                                  | 2 +-
 hedgehog-iso/build.sh                                          | 2 +-
 hedgehog-iso/config/hooks/normal/0910-sensor-build.hook.chroot | 2 +-
 hedgehog-raspi/sensor_install.sh                               | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile
index 63fc30e43..60d9c0da3 100644
--- a/Dockerfiles/arkime.Dockerfile
+++ b/Dockerfiles/arkime.Dockerfile
@@ -33,7 +33,7 @@ ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
 
 ENV ARKIME_DIR "/opt/arkime"
-ENV ARKIME_VERSION "5.4.0"
+ENV ARKIME_VERSION "5.5.0"
 ENV ARKIME_DEB_URL "https://github.com/arkime/arkime/releases/download/v${ARKIME_VERSION}/arkime_${ARKIME_VERSION}-1.debian12_XXX.deb"
 ENV ARKIME_JA4_SO_URL "https://github.com/arkime/arkime/releases/download/v${ARKIME_VERSION}/ja4plus.XXX.so"
 ENV ARKIME_LOCALELASTICSEARCH no
diff --git a/hedgehog-iso/build.sh b/hedgehog-iso/build.sh
index a78b946be..862d50796 100755
--- a/hedgehog-iso/build.sh
+++ b/hedgehog-iso/build.sh
@@ -8,7 +8,7 @@ IMAGE_DISTRIBUTION=bookworm
 BEATS_VER="8.15.3"
 BEATS_OSS="-oss"
 
-ARKIME_VER="5.4.0"
+ARKIME_VER="5.5.0"
 
 BUILD_ERROR_CODE=1
 
diff --git a/hedgehog-iso/config/hooks/normal/0910-sensor-build.hook.chroot b/hedgehog-iso/config/hooks/normal/0910-sensor-build.hook.chroot
index 464898ffc..dea7d5351 100755
--- a/hedgehog-iso/config/hooks/normal/0910-sensor-build.hook.chroot
+++ b/hedgehog-iso/config/hooks/normal/0910-sensor-build.hook.chroot
@@ -64,7 +64,7 @@ rm -Rf zeek* spicy*
 ###
 
 # get Arkime JA4+ plugin
-ARKIME_VERSION="5.4.0"
+ARKIME_VERSION="5.5.0"
 curl "${GITHUB_API_CURL_ARGS[@]}" \
      -o "/opt/arkime/plugins/ja4plus.${ARCH}.so" \
      "https://github.com/arkime/arkime/releases/download/v${ARKIME_VERSION}/ja4plus.${ARCH}.so"
diff --git a/hedgehog-raspi/sensor_install.sh b/hedgehog-raspi/sensor_install.sh
index f1fa87af1..eb4122f29 100644
--- a/hedgehog-raspi/sensor_install.sh
+++ b/hedgehog-raspi/sensor_install.sh
@@ -32,7 +32,7 @@ SHARED_DIR='/opt/buildshared'
 WORK_DIR="$(mktemp -d -t hedgehog-XXXXXX)"
 SENSOR_DIR='/opt/sensor'
 
-ARKIME_VERSION="5.4.0"
+ARKIME_VERSION="5.5.0"
 
 BEATS_VER="8.15.3"
 BEATS_OSS="-oss"

From 0010e1a94c929dd2839e83e76944dcb5aad5a319 Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Tue, 12 Nov 2024 09:18:50 -0700
Subject: [PATCH 27/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358

---
 .../36ed695f-edcc-47c1-b0ec-50d20c93ce0f.json       |  4 ++--
 logstash/pipelines/zeek/1300_zeek_normalize.conf    | 13 +++++++++++--
 logstash/pipelines/zeek/1400_zeek_convert.conf      |  4 ++--
 3 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/dashboards/dashboards/36ed695f-edcc-47c1-b0ec-50d20c93ce0f.json b/dashboards/dashboards/36ed695f-edcc-47c1-b0ec-50d20c93ce0f.json
index dbe6f2015..765c7b0e3 100644
--- a/dashboards/dashboards/36ed695f-edcc-47c1-b0ec-50d20c93ce0f.json
+++ b/dashboards/dashboards/36ed695f-edcc-47c1-b0ec-50d20c93ce0f.json
@@ -329,9 +329,9 @@
       "updated_at": "2022-01-12T18:22:26.156Z",
       "version": "WzI0NCwxXQ==",
       "attributes": {
-        "visState": "{\"title\":\"Intel - Matched\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"threat.indicator.type\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Type Matched\"}}],\"listeners\":{}}",
+        "visState": "{\"title\":\"Intel - Indicator Type\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"threat.indicator.type\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Type Matched\"}}],\"listeners\":{}}",
         "description": "",
-        "title": "Intel - Matched",
+        "title": "Intel - Indicator Type",
         "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
         "version": 1,
         "kibanaSavedObjectMeta": {
diff --git a/logstash/pipelines/zeek/1300_zeek_normalize.conf b/logstash/pipelines/zeek/1300_zeek_normalize.conf
index d81d1bc80..5920ef8c4 100644
--- a/logstash/pipelines/zeek/1300_zeek_normalize.conf
+++ b/logstash/pipelines/zeek/1300_zeek_normalize.conf
@@ -12,7 +12,7 @@ filter {
         id => "ruby_map_zeek_intel_confidence"
         code => "
           if (confNumVal = event.get('[zeek][intel][cif_confidence]').to_f) then
-            confStrVal = case value
+            confStrVal = case confNumVal
               when 0...3
                 'Low'
               when 3...7
@@ -22,7 +22,7 @@ filter {
               else
                 'None'
               end
-            event.set('[threat][indicator][confidence]', confStrVal)"
+            event.set('[threat][indicator][confidence]', confStrVal)
           end"
       }
     }
@@ -119,6 +119,15 @@ filter {
       mutate { id => "mutate_merge_zeek_intel_indicator_types_matched"
                merge => { "[threat][indicator][type]" => "[@metadata][intel_indicator_types_matched]" } }
     }
+    if ([threat][indicator][type]) {
+      ruby {
+        id => "ruby_threat_indicator_type_uniq"
+        path => "/usr/share/logstash/malcolm-ruby/make_unique_array.rb"
+        script_params => {
+          "field" => "[threat][indicator][type]"
+        }
+      }
+    }
 
     # zeek.intel.sources -> threat.indicator.provider
     if ([zeek][intel][sources]) {
diff --git a/logstash/pipelines/zeek/1400_zeek_convert.conf b/logstash/pipelines/zeek/1400_zeek_convert.conf
index 424e82e29..4960e8022 100644
--- a/logstash/pipelines/zeek/1400_zeek_convert.conf
+++ b/logstash/pipelines/zeek/1400_zeek_convert.conf
@@ -473,7 +473,7 @@ filter {
         id => "date_zeek_intel_cif_firstseen"
         match => [ "[zeek][intel][cif_firstseen]", "UNIX" ]
         target => "[threat][indicator][first_seen]"
-        remove_field => [ "[zeek][intel][cif_firstseen]"
+        remove_field => [ "[zeek][intel][cif_firstseen]" ]
       }
     }
   }
@@ -486,7 +486,7 @@ filter {
         id => "date_zeek_intel_cif_lastseen"
         match => [ "[zeek][intel][cif_lastseen]", "UNIX" ]
         target => "[threat][indicator][last_seen]"
-        remove_field => [ "[zeek][intel][cif_lastseen]"
+        remove_field => [ "[zeek][intel][cif_lastseen]" ]
       }
     }
   }

From f95bb0563b3ae73c0a7e19a10b0ab18ecfce0bcf Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Tue, 12 Nov 2024 09:24:54 -0700
Subject: [PATCH 28/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358

---
 .../36ed695f-edcc-47c1-b0ec-50d20c93ce0f.json | 89 ++++++-------------
 1 file changed, 27 insertions(+), 62 deletions(-)

diff --git a/dashboards/dashboards/36ed695f-edcc-47c1-b0ec-50d20c93ce0f.json b/dashboards/dashboards/36ed695f-edcc-47c1-b0ec-50d20c93ce0f.json
index 765c7b0e3..cfd7012f1 100644
--- a/dashboards/dashboards/36ed695f-edcc-47c1-b0ec-50d20c93ce0f.json
+++ b/dashboards/dashboards/36ed695f-edcc-47c1-b0ec-50d20c93ce0f.json
@@ -1,5 +1,5 @@
 {
-  "version": "1.2.0",
+  "version": "2.18.0",
   "objects": [
     {
       "id": "36ed695f-edcc-47c1-b0ec-50d20c93ce0f",
@@ -7,13 +7,13 @@
       "namespaces": [
         "default"
       ],
-      "updated_at": "2024-04-29T15:49:16.000Z",
-      "version": "WzEwMjMsMV0=",
+      "updated_at": "2024-11-12T16:22:49.507Z",
+      "version": "WzEwNTIsMV0=",
       "attributes": {
         "title": "Zeek Intelligence",
         "hits": 0,
         "description": "",
-        "panelsJSON": "[{\"embeddableConfig\":{},\"gridData\":{\"h\":26,\"i\":\"2\",\"w\":8,\"x\":0,\"y\":0},\"panelIndex\":\"2\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{\"table\":null,\"vis\":{\"legendOpen\":false}},\"gridData\":{\"h\":8,\"i\":\"3\",\"w\":32,\"x\":16,\"y\":0},\"panelIndex\":\"3\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"5\",\"w\":16,\"x\":8,\"y\":8},\"panelIndex\":\"5\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_2\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":18,\"i\":\"6\",\"w\":15,\"x\":33,\"y\":26},\"panelIndex\":\"6\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_3\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":18,\"i\":\"7\",\"w\":13,\"x\":14,\"y\":44},\"panelIndex\":\"7\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_4\"},{\"embeddableConfig\":{\"table\":null,\"vis\":{\"params\":{\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":18,\"i\":\"8\",\"w\":21,\"x\":27,\"y\":44},\"panelIndex\":\"8\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_5\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"11\",\"w\":25,\"x\":8,\"y\":26},\"panelIndex\":\"11\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_6\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"12\",\"w\":14,\"x\":0,\"y\":44},\"panelIndex\":\"12\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_7\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":18,\"i\":\"13\",\"w\":8,\"x\":0,\"y\":26},\"panelIndex\":\"13\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_8\"},{\"embeddableConfig\":{\"columns\":[\"source.ip\",\"destination.ip\",\"destination.port\",\"threat.indicator.name\",\"threat.indicator.type\",\"threat.indicator.provider\",\"zeek.intel.seen_where\",\"event.id\"],\"sort\":[\"MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER\",\"desc\"]},\"gridData\":{\"h\":24,\"i\":\"14\",\"w\":48,\"x\":0,\"y\":62},\"panelIndex\":\"14\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_9\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"16\",\"w\":24,\"x\":24,\"y\":8},\"panelIndex\":\"16\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_10\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":8,\"i\":\"17\",\"w\":8,\"x\":8,\"y\":0},\"panelIndex\":\"17\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_11\"}]",
+        "panelsJSON": "[{\"version\":\"2.18.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":8,\"h\":26,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"2.18.0\",\"gridData\":{\"x\":16,\"y\":0,\"w\":32,\"h\":8,\"i\":\"3\"},\"panelIndex\":\"3\",\"embeddableConfig\":{\"table\":null,\"vis\":{\"legendOpen\":false}},\"panelRefName\":\"panel_1\"},{\"version\":\"2.18.0\",\"gridData\":{\"x\":16,\"y\":8,\"w\":10,\"h\":18,\"i\":\"5\"},\"panelIndex\":\"5\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"2.18.0\",\"gridData\":{\"x\":0,\"y\":26,\"w\":13,\"h\":18,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"panelRefName\":\"panel_3\"},{\"version\":\"2.18.0\",\"gridData\":{\"x\":13,\"y\":26,\"w\":11,\"h\":18,\"i\":\"7\"},\"panelIndex\":\"7\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"panelRefName\":\"panel_4\"},{\"version\":\"2.18.0\",\"gridData\":{\"x\":24,\"y\":26,\"w\":14,\"h\":18,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{\"table\":null,\"vis\":{\"params\":{\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"}}}},\"panelRefName\":\"panel_5\"},{\"version\":\"2.18.0\",\"gridData\":{\"x\":26,\"y\":8,\"w\":22,\"h\":18,\"i\":\"11\"},\"panelIndex\":\"11\",\"embeddableConfig\":{},\"panelRefName\":\"panel_6\"},{\"version\":\"2.18.0\",\"gridData\":{\"x\":38,\"y\":26,\"w\":10,\"h\":18,\"i\":\"12\"},\"panelIndex\":\"12\",\"embeddableConfig\":{},\"panelRefName\":\"panel_7\"},{\"version\":\"2.18.0\",\"gridData\":{\"x\":8,\"y\":8,\"w\":8,\"h\":18,\"i\":\"13\"},\"panelIndex\":\"13\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"panelRefName\":\"panel_8\"},{\"version\":\"2.18.0\",\"gridData\":{\"x\":0,\"y\":44,\"w\":48,\"h\":24,\"i\":\"14\"},\"panelIndex\":\"14\",\"embeddableConfig\":{\"columns\":[\"source.ip\",\"destination.ip\",\"destination.port\",\"threat.indicator.name\",\"threat.indicator.type\",\"threat.indicator.provider\",\"zeek.intel.seen_where\",\"event.id\"],\"sort\":[]},\"panelRefName\":\"panel_9\"},{\"version\":\"2.18.0\",\"gridData\":{\"x\":8,\"y\":0,\"w\":8,\"h\":8,\"i\":\"17\"},\"panelIndex\":\"17\",\"embeddableConfig\":{},\"panelRefName\":\"panel_10\"}]",
         "optionsJSON": "{\"useMargins\":true}",
         "version": 1,
         "timeRestore": false,
@@ -75,11 +75,6 @@
         {
           "name": "panel_10",
           "type": "visualization",
-          "id": "fa56cc7f-fb00-47fb-becb-1b1fdfea908e"
-        },
-        {
-          "name": "panel_11",
-          "type": "visualization",
           "id": "AWDG-Qf8xQT5EBNmq4G5"
         }
       ],
@@ -93,8 +88,8 @@
       "namespaces": [
         "default"
       ],
-      "updated_at": "2022-01-12T18:23:14.699Z",
-      "version": "Wzc2OSwxXQ==",
+      "updated_at": "2024-11-12T15:53:48.389Z",
+      "version": "WzkyOSwxXQ==",
       "attributes": {
         "title": "Navigation",
         "visState": "{\"title\":\"Navigation\",\"type\":\"markdown\",\"params\":{\"markdown\":\"### General Network Logs\\n[Overview](#/dashboard/0ad3d7c2-3441-485e-9dfe-dbb22e84e576)  \\n[Security Overview](#/dashboard/95479950-41f2-11ea-88fa-7151df485405)  \\n[ICS/IoT Security Overview](#/dashboard/4a4bde20-4760-11ea-949c-bbb5a9feecbf)  \\n[Severity](#/dashboard/d2dd0180-06b1-11ec-8c6b-353266ade330)  \\n[Connections](#/dashboard/abdd7550-2c7c-40dc-947e-f6d186a158c4)  \\n[Actions and Results](#/dashboard/a33e0a50-afcd-11ea-993f-b7d8522a8bed)  \\n[Files](#/dashboard/9ee51f94-3316-4fc5-bd89-93a52af69714)  \\n[Executables](#/dashboard/0a490422-0ce9-44bf-9a2d-19329ddde8c3)  \\n[Software](#/dashboard/87d990cc-9e0b-41e5-b8fe-b10ae1da0c85)  \\n[Zeek Known Summary](#/dashboard/89d1cc50-974c-11ed-bb6b-3fb06c879b11)  \\n[Zeek Intelligence](#/dashboard/36ed695f-edcc-47c1-b0ec-50d20c93ce0f)  \\n[Zeek Notices](#/dashboard/f1f09567-fc7f-450b-a341-19d2f2bb468b)  \\n[Zeek Weird](#/dashboard/1fff49f6-0199-4a0f-820b-721aff9ff1f1)  \\n[Signatures](#/dashboard/665d1610-523d-11e9-a30e-e3576242f3ed)  \\n[Suricata Alerts](#/dashboard/5694ca60-cbdf-11ec-a50a-5fedd672f5c5)  \\n[Asset Interaction Analysis](#/dashboard/677ee170-809e-11ed-8d5b-07069f823b6f)  \\n[↪ NetBox](/netbox/)  \\n[↪ Arkime](/arkime/)  \\n\\n### Common Protocols\\n[DCE/RPC](#/dashboard/432af556-c5c0-4cc3-8166-b274b4e3a406)   ●   [DHCP](#/dashboard/2d98bb8e-214c-4374-837b-20e1bcd63a5e)   ●   [DNS](#/dashboard/2cf94cd0-ecab-40a5-95a7-8419f3a39cd9)   ●   [FTP](#/dashboard/078b9aa5-9bd4-4f02-ae5e-cf80fa6f887b) / [TFTP](#/dashboard/bf5efbb0-60f1-11eb-9d60-dbf0411cfc48)   ●   [HTTP](#/dashboard/37041ee1-79c0-4684-a436-3173b0e89876) / [WebSocket](#/dashboard/b8cf5890-87ed-11ef-ae18-dbcd34795edb)   ●   [IRC](#/dashboard/76f2f912-80da-44cd-ab66-6a73c8344cc3)   ●   [Kerberos](#/dashboard/82da3101-2a9c-4ae2-bb61-d447a3fbe673)   ●   [LDAP](#/dashboard/05e3e000-f118-11e9-acda-83a8e29e1a24)   ●   [MQTT](#/dashboard/87a32f90-ef58-11e9-974e-9d600036d105)   ●   [MySQL](#/dashboard/50ced171-1b10-4c3f-8b67-2db9635661a6)   ●   [NTLM](#/dashboard/543118a9-02d7-43fe-b669-b8652177fc37)   ●   [NTP](#/dashboard/af5df620-eeb6-11e9-bdef-65a192b7f586)   ●   [OSPF](#/dashboard/1cc01ff0-5205-11ec-a62c-7bc80e88f3f0)   ●   [QUIC](#/dashboard/11ddd980-e388-11e9-b568-cf17de8e860c)   ●   [RADIUS](#/dashboard/ae79b7d1-4281-4095-b2f6-fa7eafda9970)   ●   [RDP](#/dashboard/7f41913f-cba8-43f5-82a8-241b7ead03e0)   ●   [RFB](#/dashboard/f77bf097-18a8-465c-b634-eb2acc7a4f26)   ●   [SIP](#/dashboard/0b2354ae-0fe9-4fd9-b156-1c3870e5c7aa)   ●   [SMB](#/dashboard/42e831b9-41a9-4f35-8b7d-e1566d368773)   ●   [SMTP](#/dashboard/bb827f8e-639e-468c-93c8-9f5bc132eb8f)   ●   [SNMP](#/dashboard/4e5f106e-c60a-4226-8f64-d534abb912ab)   ●   [SSH](#/dashboard/caef3ade-d289-4d05-a511-149f3e97f238)   ●   [SSL](#/dashboard/7f77b58a-df3e-4cc2-b782-fd7f8bad8ffb) / [X.509 Certificates](#/dashboard/024062a6-48d6-498f-a91a-3bf2da3a3cd3)   ●   [STUN](#/dashboard/fa477130-2b8a-11ec-a9f2-3911c8571bfd)   ●   [Syslog](#/dashboard/92985909-dc29-4533-9e80-d3182a0ecf1d)   ●   [TDS](#/dashboard/bed185a0-ef82-11e9-b38a-2db3ee640e88) / [TDS RPC](#/dashboard/32587740-ef88-11e9-b38a-2db3ee640e88) / [TDS SQL](#/dashboard/fa141950-ef89-11e9-b38a-2db3ee640e88)   ●   [Telnet / rlogin / rsh](#/dashboard/c2549e10-7f2e-11ea-9f8a-1fe1327e2cd2)   ●   [Tunnels](#/dashboard/11be6381-beef-40a7-bdce-88c5398392fc)\\n\\n### ICS/IoT Protocols\\n[BACnet](#/dashboard/2bec1490-eb94-11e9-a384-0fcf32210194)   ●   [BSAP](#/dashboard/ca5799a0-56b5-11eb-b749-576de068f8ad)   ●   [DNP3](#/dashboard/870a5862-6c26-4a08-99fd-0c06cda85ba3)   ●   [EtherCAT](#/dashboard/4a073440-b286-11eb-a4d4-09fa12a6ebd4)   ●   [EtherNet/IP](#/dashboard/29a1b290-eb98-11e9-a384-0fcf32210194)   ●   [GENISYS](#/dashboard/03207c00-d07e-11ec-b4a7-d1b4003706b7)   ●   [GE SRTP](#/dashboard/e233a570-45d9-11ef-96a6-432365601033)   ●   [HART-IP](#/dashboard/3a9e3440-75e2-11ef-8138-03748f839a49)   ●   [Modbus](#/dashboard/152f29dc-51a2-4f53-93e9-6e92765567b8)   ●   [OPCUA Binary](#/dashboard/dd87edd0-796a-11ec-9ce6-b395c1ff58f4)   ●   [PROFINET](#/dashboard/a7514350-eba6-11e9-a384-0fcf32210194)   ●   [S7comm](#/dashboard/e76d05c0-eb9f-11e9-a384-0fcf32210194)   ●   [Synchrophasor](#/dashboard/2cc56240-e460-11ed-a9d5-9f591c284cb4)   ●   [Best Guess](#/dashboard/12e3a130-d83b-11eb-a0b0-f328ce09b0b7)\\n\\n### Malcolm and Third-Party Logs\\n\\nResources: [System Overview](#/dashboard/Metricbeat-system-overview-ecs) / [Host Overview](#/dashboard/79ffd6e0-faa0-11e6-947f-177f697178b8-ecs)   ●   [Hardware Temperature](#/dashboard/0d4955f0-eb25-11ec-a6d4-b3526526c2c7)   ●   nginx [Overview](#/dashboard/55a9e6e0-a29e-11e7-928f-5dbe6f6f5519-ecs) / [Access and Error Logs](#/dashboard/046212a0-a2a1-11e7-928f-5dbe6f6f5519-ecs)   ●   Linux [Journald](#/dashboard/f6600310-9943-11ee-a029-e973f4774355) / [Kernel Messages](#/dashboard/3768ef70-d819-11ee-820d-dd9fd73a3921)   ●   [Windows Events](#/dashboard/79202ee0-d811-11ee-820d-dd9fd73a3921)   ●   [Malcolm Sensor File Integrity](#/dashboard/903f42c0-f634-11ec-828d-2fb7a4a26e1f)   ●   [Malcolm Sensor Audit Logs](#/dashboard/7a7e0a60-e8e8-11ec-b9d4-4569bb965430)   ●   [Packet Capture Statistics](#/dashboard/4ca94c70-d7da-11ee-9ed3-e7afff29e59a)\",\"type\":\"markdown\",\"fontSize\":10,\"openLinksInNewTab\":false},\"aggs\":[]}",
@@ -116,8 +111,8 @@
       "namespaces": [
         "default"
       ],
-      "updated_at": "2022-01-12T18:32:32.907Z",
-      "version": "Wzk5OCwxXQ==",
+      "updated_at": "2024-11-12T15:52:51.822Z",
+      "version": "WzI3MCwxXQ==",
       "attributes": {
         "title": "Intel - Log Count Over Time",
         "visState": "{\"title\":\"Intel - Log Count Over Time\",\"type\":\"line\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER\",\"timeRange\":{\"from\":\"now-25y\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\" \"},\"schema\":\"segment\"}],\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{\"text\":\"MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER per 12 hours\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"\"}}],\"seriesParams\":[{\"show\":true,\"mode\":\"normal\",\"type\":\"histogram\",\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"showCircles\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"type\":\"histogram\",\"labels\":{},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}",
@@ -146,8 +141,8 @@
       "namespaces": [
         "default"
       ],
-      "updated_at": "2022-01-12T18:22:26.156Z",
-      "version": "WzIzOCwxXQ==",
+      "updated_at": "2024-11-12T15:52:51.822Z",
+      "version": "WzI3MSwxXQ==",
       "attributes": {
         "title": "Intel - Seen",
         "visState": "{\"title\":\"Intel - Seen\",\"type\":\"pie\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.intel.seen_where\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":20,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Seen (Where)\"},\"schema\":\"segment\"}],\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":true,\"type\":\"pie\",\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}}}",
@@ -176,8 +171,8 @@
       "namespaces": [
         "default"
       ],
-      "updated_at": "2022-01-12T18:22:26.156Z",
-      "version": "WzIzOSwxXQ==",
+      "updated_at": "2024-11-12T15:52:51.822Z",
+      "version": "WzI3MiwxXQ==",
       "attributes": {
         "visState": "{\"title\":\"Intel - Source\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"threat.indicator.provider\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Source\"}}],\"listeners\":{}}",
         "description": "",
@@ -206,8 +201,8 @@
       "namespaces": [
         "default"
       ],
-      "updated_at": "2022-01-12T18:22:26.156Z",
-      "version": "WzI0MCwxXQ==",
+      "updated_at": "2024-11-12T15:52:51.822Z",
+      "version": "WzI3MywxXQ==",
       "attributes": {
         "visState": "{\"title\":\"Intel - Source IP Address\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source.ip\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"IP Address\"}}],\"listeners\":{}}",
         "description": "",
@@ -236,8 +231,8 @@
       "namespaces": [
         "default"
       ],
-      "updated_at": "2022-01-12T18:22:26.156Z",
-      "version": "WzI0MSwxXQ==",
+      "updated_at": "2024-11-12T15:52:51.822Z",
+      "version": "WzI3NCwxXQ==",
       "attributes": {
         "title": "Intel - Destination IP Address",
         "visState": "{\"title\":\"Intel - Destination IP Address\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"destination.ip\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"IP Address\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"destination.port\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Port\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false,\"percentageCol\":\"\"}}",
@@ -266,8 +261,8 @@
       "namespaces": [
         "default"
       ],
-      "updated_at": "2022-01-12T18:22:26.156Z",
-      "version": "WzI0MiwxXQ==",
+      "updated_at": "2024-11-12T15:52:51.822Z",
+      "version": "WzI3NSwxXQ==",
       "attributes": {
         "title": "Intel - Indicator",
         "visState": "{\"title\":\"Intel - Indicator\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"threat.indicator.type\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Type\"},\"schema\":\"bucket\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.intel.seen_where\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Seen Where\"},\"schema\":\"bucket\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"threat.indicator.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Indicator\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false,\"percentageCol\":\"\"}}",
@@ -296,8 +291,8 @@
       "namespaces": [
         "default"
       ],
-      "updated_at": "2022-01-12T18:22:26.156Z",
-      "version": "WzI0MywxXQ==",
+      "updated_at": "2024-11-12T15:52:51.822Z",
+      "version": "WzI3NiwxXQ==",
       "attributes": {
         "visState": "{\"title\":\"Intel - MIME Type\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"zeek.intel.file_mime_type\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"MIME Type\"}}],\"listeners\":{}}",
         "description": "",
@@ -326,8 +321,8 @@
       "namespaces": [
         "default"
       ],
-      "updated_at": "2022-01-12T18:22:26.156Z",
-      "version": "WzI0NCwxXQ==",
+      "updated_at": "2024-11-12T15:52:51.822Z",
+      "version": "WzI3NywxXQ==",
       "attributes": {
         "visState": "{\"title\":\"Intel - Indicator Type\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"threat.indicator.type\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Type Matched\"}}],\"listeners\":{}}",
         "description": "",
@@ -356,8 +351,8 @@
       "namespaces": [
         "default"
       ],
-      "updated_at": "2022-01-12T18:22:26.156Z",
-      "version": "WzI0NSwxXQ==",
+      "updated_at": "2024-11-12T15:52:51.822Z",
+      "version": "WzI3OCwxXQ==",
       "attributes": {
         "sort": [
           [
@@ -394,44 +389,14 @@
         "search": "7.9.3"
       }
     },
-    {
-      "id": "fa56cc7f-fb00-47fb-becb-1b1fdfea908e",
-      "type": "visualization",
-      "namespaces": [
-        "default"
-      ],
-      "updated_at": "2022-01-12T18:32:43.892Z",
-      "version": "WzEwMTIsMV0=",
-      "attributes": {
-        "title": "Intel - Indicator Type",
-        "visState": "{\"title\":\"Intel - Indicator Type\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"threat.indicator.type\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Indicator Type\"},\"schema\":\"segment\"}],\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100,\"rotate\":0},\"title\":{\"text\":\"Indicator Type\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"type\":\"histogram\",\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}",
-        "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}",
-        "description": "",
-        "version": 1,
-        "kibanaSavedObjectMeta": {
-          "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}"
-        },
-        "savedSearchRefName": "search_0"
-      },
-      "references": [
-        {
-          "name": "search_0",
-          "type": "search",
-          "id": "5154d8e9-c83e-4d42-bde3-33ad0c7d1798"
-        }
-      ],
-      "migrationVersion": {
-        "visualization": "7.10.0"
-      }
-    },
     {
       "id": "AWDG-Qf8xQT5EBNmq4G5",
       "type": "visualization",
       "namespaces": [
         "default"
       ],
-      "updated_at": "2022-01-12T18:22:26.156Z",
-      "version": "WzI0NywxXQ==",
+      "updated_at": "2024-11-12T15:52:51.822Z",
+      "version": "WzI4MCwxXQ==",
       "attributes": {
         "title": "Intel - Log Count",
         "visState": "{\"title\":\"Intel - Log Count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"30\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"bgFill\":\"#FB9E00\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}",
@@ -455,4 +420,4 @@
       }
     }
   ]
-}
+}
\ No newline at end of file

From b75d4a4b788b76c8b5977665501e96a3c9a9cf1f Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Tue, 12 Nov 2024 11:24:08 -0700
Subject: [PATCH 29/42] work in progress for mandiant threat intel integration,
 cisagov/Malcolm#358

---
 docs/zeek-intel.md                        |  48 +-
 shared/bin/zeek_intel_from_threat_feed.py |  13 +-
 shared/bin/zeek_intel_setup.sh            |  17 +-
 shared/bin/zeek_threat_feed_utils.py      | 802 ++++++++++++----------
 4 files changed, 483 insertions(+), 397 deletions(-)

diff --git a/docs/zeek-intel.md b/docs/zeek-intel.md
index bdcc16b67..84010e202 100644
--- a/docs/zeek-intel.md
+++ b/docs/zeek-intel.md
@@ -26,7 +26,23 @@ For a public example of Zeek intelligence files, see Critical Path Security's [r
 
 In addition to loading Zeek intelligence files on startup, Malcolm will [automatically generate]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/zeek_intel_from_threat_feed.py) a Zeek intelligence file for all [Structured Threat Information Expression (STIX™)](https://oasis-open.github.io/cti-documentation/stix/intro.html) [v2.0](https://docs.oasis-open.org/cti/stix/v2.0/stix-v2.0-part1-stix-core.html)/[v2.1](https://docs.oasis-open.org/cti/stix/v2.1/stix-v2.1.html) JSON files found under `./zeek/intel/STIX`.
 
-Additionally, if a special text file named `.stix_input.txt` is found in `./zeek/intel/STIX`, that file will be read and processed as a list of [TAXII™](https://oasis-open.github.io/cti-documentation/taxii/intro.html) [2.0](http://docs.oasis-open.org/cti/taxii/v2.0/cs01/taxii-v2.0-cs01.html)/[2.1](https://docs.oasis-open.org/cti/taxii/v2.1/csprd02/taxii-v2.1-csprd02.html) feeds, one per line, according to the following format (the username and password are optional):
+Additionally, if a [YAML](https://yaml.org/) file named `taxii.yaml` is found in `./zeek/intel/STIX`, that file will be read and processed as a list of [TAXII™](https://oasis-open.github.io/cti-documentation/taxii/intro.html) [2.0](http://docs.oasis-open.org/cti/taxii/v2.0/cs01/taxii-v2.0-cs01.html)/[2.1](https://docs.oasis-open.org/cti/taxii/v2.1/csprd02/taxii-v2.1-csprd02.html) feeds. This file should minimally include:
+
+```yaml
+- type: taxii
+  version: 2.1
+  url: https://example.com/taxii/api2/
+  collection: "*"
+```
+
+These other parameters can also optionally be provided:
+
+```yaml
+  username: guest
+  password: guest
+```
+
+Alternatively, if a text file named `.stix_input.txt` is found in `./zeek/intel/STIX`, that file will be read and processed as described above. The feeds are specified one per line, according to the following format (the username and password are optional):
 
 ```
 taxii|version|discovery_url|collection_name|username|password
@@ -50,10 +66,22 @@ Malcolm uses the [stix2](https://pypi.org/project/stix2/) and [taxii2-client](ht
 
 In addition to loading Zeek intelligence files on startup, Malcolm will [automatically generate]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/zeek_intel_from_threat_feed.py) a Zeek intelligence file for all [Malware Information Sharing Platform (MISP)](https://www.misp-project.org/datamodels/) JSON files found under `./zeek/intel/MISP`.
 
-Additionally, if a special text file named `.misp_input.txt` is found in `./zeek/intel/MISP`, that file will be read and processed as a list of [MISP feed](https://misp.gitbooks.io/misp-book/content/managing-feeds/#feeds) URLs, one per line, according to the following format:
+Additionally, if a [YAML](https://yaml.org/) file named `misp.yaml` is found in `./zeek/intel/MISP`, that file will be read and processed as a list of [MISP feed](https://misp.gitbooks.io/misp-book/content/managing-feeds/#feeds) URLs. This file should minimally include:
+
+```yaml
+- type: misp
+  url: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+```
+
+These other parameters can also optionally be provided:
+
+```yaml
+  auth_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+```
 
+Alternatively, if a special file named `.misp_input.txt` is found in `./zeek/intel/MISP`, that file will be read and processed as described above. The feeds are specified one per line, according to the following format (the authentication key is optional):
 ```
-misp|misp_url|auth_key (optional)
+misp|misp_url|auth_key
 ```
 
 For example:
@@ -85,18 +113,18 @@ Malcolm uses the [MISP/PyMISP](https://github.com/MISP/PyMISP) Python library to
 If a [YAML](https://yaml.org/) file named `mandiant.yaml` is found in `./zeek/intel/Mandiant`, that file will be read and processed as parameters for the [Mandiant Threat Intelligence](https://www.mandiant.com/threats) service. This file should minimally include:
 
 ```yaml
-type: mandiant
-api_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
-secret_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+- type: mandiant
+  api_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+  secret_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 ```
 
 These other parameters can also optionally be provided:
 
 ```yaml
-minimum_mscore: 60
-exclude_osint: False
-bearer_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
-api_base_url: https://api.intelligence.mandiant.com
+  minimum_mscore: 60
+  exclude_osint: False
+  bearer_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+  api_base_url: https://api.intelligence.mandiant.com
 ```
 
 Malcolm uses the [google/mandiant-ti-client](https://github.com/google/mandiant-ti-client) Python library to access Mandiant threat intelligence feeds.
diff --git a/shared/bin/zeek_intel_from_threat_feed.py b/shared/bin/zeek_intel_from_threat_feed.py
index 5319064a6..410ff19e4 100755
--- a/shared/bin/zeek_intel_from_threat_feed.py
+++ b/shared/bin/zeek_intel_from_threat_feed.py
@@ -174,7 +174,15 @@ def main():
                         if re.search(r"\.ya?ml$", infileParts[1], re.IGNORECASE):
                             with open(infileArg, 'r') as f:
                                 inputParams = yaml.safe_load(f)
-                            yamlInputs.append(inputParams)
+                            if inputParams:
+                                if isinstance(inputParams, dict):
+                                    yamlInputs.append(inputParams)
+                                elif isinstance(inputParams, list):
+                                    yamlInputs.extend(inputParams)
+                                else:
+                                    logging.error(
+                                        f"Connection parameters of type '{type(inputParams).__name__}' are not supported"
+                                    )
                         else:
                             with open(infileArg) as f:
                                 args.input.extend(f.read().splitlines())
@@ -194,8 +202,9 @@ def main():
 
                     else:
                         logging.warning(f"File '{infileArg}' not found")
+
                 except Exception as e:
-                    logging.warning(f"{type(e).__name__} for '{infileArg}': {e}")
+                    logging.error(f"{type(e).__name__} for '{infileArg}': {e}")
 
         # deduplicate input sources
         seenInput = {}
diff --git a/shared/bin/zeek_intel_setup.sh b/shared/bin/zeek_intel_setup.sh
index 1d81ca9cc..35b1476e6 100755
--- a/shared/bin/zeek_intel_setup.sh
+++ b/shared/bin/zeek_intel_setup.sh
@@ -75,35 +75,40 @@ EOF
                 # this directory contains STIX JSON files we'll need to convert to zeek intel files then load
                 while IFS= read -r line; do
                     THREAT_JSON_FILES+=( "$line" )
-                done < <( find "${INTEL_DIR}/${DIR}" -type f ! -name ".*" 2>/dev/null )
+                done < <( find "${INTEL_DIR}/${DIR}" \( -type f -a ! -name ".*" -a ! -name "taxii.yaml" \) 2>/dev/null )
 
             elif [[ "${DIR}" == "./MISP" ]]; then
                 # this directory contains MISP JSON files we'll need to convert to zeek intel files then load
                 while IFS= read -r line; do
                     THREAT_JSON_FILES+=( "$line" )
-                done < <( find "${INTEL_DIR}/${DIR}" -type f ! -name ".*" ! -name "manifest.json" ! -name "hashes.csv" 2>/dev/null )
+                done < <( find "${INTEL_DIR}/${DIR}" \( -type f -a ! -name ".*" -a ! -name "misp.yaml" -a ! -name "manifest.json" -a ! -name "hashes.csv" \) 2>/dev/null )
 
             elif [[ -f "${DIR}"/__load__.zeek ]]; then
                 # this intel feed has its own load directive and should take care of itself
                 echo "@load ${DIR}" >> ./__load__.zeek."${INSTANCE_UID}"
 
-            elif [[ "${DIR}" != "./Mandiant" ]]; then
+            else
                 # this custom directory contains "loose" intel files we'll need to load explicitly
                 while IFS= read -r line; do
                     LOOSE_INTEL_FILES+=( "$line" )
-                done < <( find "${INTEL_DIR}/${DIR}" -type f ! -name ".*" 2>/dev/null )
+                done < <( find "${INTEL_DIR}/${DIR}" \( -type f -a ! -name ".*" -a ! -name "*.yaml" \) 2>/dev/null )
             fi
         done
 
         # process STIX/MISP/Mandiant inputs by converting them to Zeek intel format
-        if ( (( ${#THREAT_JSON_FILES[@]} )) || [[ -r ./STIX/.stix_input.txt ]] || [[ -r ./MISP/.misp_input.txt ]] || [[ -r ./Mandiant/mandiant.yaml ]] ) && [[ -x "${THREAT_FEED_TO_ZEEK_SCRIPT}" ]]; then
+        if ( (( ${#THREAT_JSON_FILES[@]} )) || [[ -r ./STIX/.stix_input.txt ]] || [[ -r ./STIX/taxii.yaml ]] || [[ -r ./MISP/.misp_input.txt ]] || [[ -r ./MISP/misp.yaml ]] || [[ -r ./Mandiant/mandiant.yaml ]] ) && [[ -x "${THREAT_FEED_TO_ZEEK_SCRIPT}" ]]; then
             "${THREAT_FEED_TO_ZEEK_SCRIPT}" \
                 --ssl-verify ${ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION} \
                 --since "${ZEEK_INTEL_FEED_SINCE}" \
                 --threads ${ZEEK_INTEL_REFRESH_THREADS} \
                 --output ./.threat_autogen.zeek."${INSTANCE_UID}" \
                 --input "${THREAT_JSON_FILES[@]}" \
-                --input-file ./STIX/.stix_input.txt ./MISP/.misp_input.txt ./Mandiant/mandiant.yaml
+                --input-file \
+                    ./STIX/.stix_input.txt \
+                    ./STIX/taxii.yaml \
+                    ./MISP/.misp_input.txt \
+                    ./MISP/misp.yaml \
+                    ./Mandiant/mandiant.yaml
             if [[ $? -eq 0 ]]; then
                 rm -f ./.threat_autogen.zeek.old
                 mv --backup=simple --suffix=.old ./.threat_autogen.zeek."${INSTANCE_UID}" ./.threat_autogen.zeek
diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py
index 7b4279dd5..1077c1063 100644
--- a/shared/bin/zeek_threat_feed_utils.py
+++ b/shared/bin/zeek_threat_feed_utils.py
@@ -764,6 +764,356 @@ def ProcessMISP(
         return result
 
 
+def UpdateFromMISP(
+    connInfo,
+    since,
+    nowTime,
+    sslVerify,
+    zeekPrinter,
+    logger,
+    successCount,
+    workerId,
+):
+    with requests.Session() as mispSession:
+        mispSession.headers.update({'Accept': 'application/json;q=1.0,text/plain;q=0.9,text/html;q=0.9'})
+        if mispAuthKey := connInfo.get('auth_key', None):
+            mispSession.headers.update({'Authorization': mispAuthKey})
+
+        mispUrl = connInfo.get('url', None)
+
+        # download the URL and parse as JSON to figure out what it is. it could be:
+        # - a manifest JSON (https://www.circl.lu/doc/misp/feed-osint/manifest.json)
+        # - a directory listing *containing* a manifest.json (https://www.circl.lu/doc/misp/feed-osint/)
+        # - a directory listing of misc. JSON files without a manifest.json
+        # - an array of Attributes returned for a request via the MISP Automation API to an /attributes endpoint
+        # - an array of Events returned for a request via the MISP Automation API to an /events endpoint
+        mispResponse = mispSession.get(
+            mispUrl,
+            allow_redirects=True,
+            verify=sslVerify,
+        )
+        mispResponse.raise_for_status()
+        if mispJson := LoadStrIfJson(mispResponse.content):
+            # the contents are JSON. determine if this is:
+            #   - a single Event
+            #   - an array of Events
+            #   - an array of Attributes
+            #   - a manifest
+
+            if isinstance(mispJson, dict) and (len(mispJson.keys()) == 1) and ('Event' in mispJson):
+                # this is a single MISP Event, process it
+                if zeekPrinter.ProcessMISP(
+                    mispJson,
+                    url=mispUrl,
+                ):
+                    successCount.increment()
+
+            elif isinstance(mispJson, list) and (len(mispJson) > 0):
+                # are these Attributes or Events?
+                if isinstance(mispJson[0], dict) and ('id' in mispJson[0]) and ('type' in mispJson[0]):
+                    controllerType = 'attributes'
+                    resultKey = 'Attribute'
+                    pageSize = MISP_PAGE_SIZE_ATTRIBUTES
+                elif isinstance(mispJson[0], dict) and ('info' in mispJson[0]):
+                    controllerType = 'events'
+                    resultKey = 'Event'
+                    pageSize = MISP_PAGE_SIZE_EVENTS
+                else:
+                    controllerType = None
+                    resultKey = None
+                    pageSize = None
+
+                if controllerType:
+                    # this is an array of either Attributes or Events.
+                    #   rather than handling it via additional calls with request,
+                    #   let's use the MISP API to do the searching/pulling
+                    #   (yeah, we're duplicating the effort of pulling the
+                    #   first page, but meh, who cares?)
+                    if mispObject := PyMISP(
+                        mispUrl,
+                        mispAuthKey,
+                        sslVerify,
+                        debug=logger and (LOGGING_DEBUG >= logger.root.level),
+                    ):
+                        # search, looping over the pages pageSize at a time
+                        mispPage = 0
+                        while True:
+                            mispPage += 1
+                            resultCount = 0
+                            mispResults = mispObject.search(
+                                controller=controllerType,
+                                return_format='json',
+                                limit=pageSize,
+                                page=mispPage,
+                                type_attribute=list(MISP_ZEEK_INTEL_TYPE_MAP.keys()),
+                                timestamp=since,
+                            )
+                            if mispResults and isinstance(mispResults, dict) and (resultKey in mispResults):
+                                # Attributes results
+                                resultCount = len(mispResults[resultKey])
+                                for item in mispResults[resultKey]:
+                                    try:
+                                        if zeekPrinter.ProcessMISP(
+                                            item,
+                                            url=mispUrl,
+                                        ):
+                                            successCount.increment()
+                                    except Exception as e:
+                                        if logger is not None:
+                                            logger.warning(
+                                                f"[{workerId}]: {type(e).__name__} for MISP {resultKey}: {e}"
+                                            )
+
+                            elif mispResults and isinstance(mispResults, list):
+                                # Events results
+                                resultCount = len(mispResults)
+                                for item in mispResults:
+                                    if item and isinstance(item, dict) and (resultKey in item):
+                                        try:
+                                            if zeekPrinter.ProcessMISP(
+                                                item[resultKey],
+                                                url=mispUrl,
+                                            ):
+                                                successCount.increment()
+                                        except Exception as e:
+                                            if logger is not None:
+                                                logger.warning(
+                                                    f"[{workerId}]: {type(e).__name__} for MISP {resultKey}: {e}"
+                                                )
+
+                            else:
+                                # error or unrecognized results, set this to short circuit
+                                resultCount = 0
+
+                            if (logger is not None) and (LOGGING_DEBUG >= logger.root.level):
+                                logger.debug(f"[{workerId}]: MISP search page {mispPage} returned {resultCount}")
+                            if not mispResults or (resultCount < pageSize):
+                                break
+
+                else:
+                    # not an Event or an Attribute? what the heck are we even doing?
+                    raise Exception(f"Unknown MISP object '{json.dumps(mispJson)}'")
+
+            elif isinstance(mispJson, dict):
+                # this is a manifest, loop over, retrieve and process the MISP events it references
+                for uri in mispJson:
+                    try:
+                        newUrl = urljoin(mispUrl, f'{uri}.json')
+                        eventTime = (
+                            datetime.utcfromtimestamp(int(mispJson[uri]['timestamp'])).astimezone(UTCTimeZone)
+                            if 'timestamp' in mispJson[uri]
+                            else defaultNow
+                        )
+                        if (since is None) or (eventTime >= since):
+                            mispObjectReponse = mispSession.get(
+                                newUrl,
+                                allow_redirects=True,
+                                verify=sslVerify,
+                            )
+                            mispObjectReponse.raise_for_status()
+                            if zeekPrinter.ProcessMISP(
+                                mispObjectReponse.json(),
+                                url=newUrl,
+                            ):
+                                successCount.increment()
+                    except Exception as e:
+                        if logger is not None:
+                            logger.warning(f"[{workerId}]: {type(e).__name__} for MISP object at '{newUrl}': {e}")
+
+            else:
+                raise Exception(f"Unknown MISP format '{type(mispJson)}'")
+
+        else:
+            # the contents are NOT JSON, it's probably an HTML-formatted directory listing
+
+            # retrieve the links listed (non-recursive, all .json files in this directory)
+            paths = get_url_paths_from_response(mispResponse.text, parent_url=mispUrl, ext='.json')
+
+            # see if manifest.json exists in this directory
+            manifestPaths = [x for x in paths if x.endswith('/manifest.json')]
+            if len(manifestPaths) > 0:
+                # the manifest.json exists!
+                # retrieve it, then loop over it and retrieve and process the MISP events it references
+                for url in manifestPaths:
+                    try:
+                        mispManifestResponse = mispSession.get(
+                            url,
+                            allow_redirects=True,
+                            verify=sslVerify,
+                        )
+                        mispManifestResponse.raise_for_status()
+                        mispManifest = mispManifestResponse.json()
+                        for uri in mispManifest:
+                            try:
+                                eventTime = (
+                                    datetime.utcfromtimestamp(int(mispManifest[uri]['timestamp'])).astimezone(
+                                        UTCTimeZone
+                                    )
+                                    if 'timestamp' in mispManifest[uri]
+                                    else defaultNow
+                                )
+                                if (since is None) or (eventTime >= since):
+                                    newUrl = f'{mispUrl.strip("/")}/{uri}.json'
+                                    mispObjectReponse = mispSession.get(
+                                        newUrl,
+                                        allow_redirects=True,
+                                        verify=sslVerify,
+                                    )
+                                    mispObjectReponse.raise_for_status()
+                                    if zeekPrinter.ProcessMISP(
+                                        mispObjectReponse.json(),
+                                        url=newUrl,
+                                    ):
+                                        successCount.increment()
+                            except Exception as e:
+                                if logger is not None:
+                                    logger.warning(
+                                        f"[{workerId}]: {type(e).__name__} for MISP object at '{mispUrl}/{uri}.json': {e}"
+                                    )
+                    except Exception as e:
+                        if logger is not None:
+                            logger.warning(f"[{workerId}]: {type(e).__name__} for manifest at '{url}': {e}")
+
+            else:
+                # the manifest.json does not exist!
+                # just loop over, retrieve and process the .json files in this directory
+                for url in paths:
+                    try:
+                        mispObjectReponse = mispSession.get(
+                            url,
+                            allow_redirects=True,
+                            verify=sslVerify,
+                        )
+                        mispObjectReponse.raise_for_status()
+                        if zeekPrinter.ProcessMISP(
+                            mispObjectReponse.json(),
+                            url=url,
+                        ):
+                            successCount.increment()
+                    except Exception as e:
+                        if logger is not None:
+                            logger.warning(f"[{workerId}]: {type(e).__name__} for MISP object at '{url}': {e}")
+
+
+def UpdateFromTAXII(
+    connInfo,
+    since,
+    nowTime,
+    sslVerify,
+    zeekPrinter,
+    logger,
+    successCount,
+    workerId,
+):
+    # connect to the server     with the appropriate API for the TAXII version
+    taxiiUrl = connInfo.get('url', None)
+    taxiiCollection = connInfo.get('collection', None)
+    taxiiUsername = connInfo.get('username', None)
+    taxiiPassword = connInfo.get('password', None)
+    if taxiiVersion := connInfo.get('version', None):
+        if taxiiVersion == '2.0':
+            server = TaxiiServer_v20(taxiiUrl, user=taxiiUsername, password=taxiiPassword, verify=sslVerify)
+        elif taxiiVersion == '2.1':
+            server = TaxiiServer_v21(taxiiUrl, user=taxiiUsername, password=taxiiPassword, verify=sslVerify)
+        else:
+            raise Exception(f"Unsupported TAXII version '{taxiiVersion}'")
+    else:
+        raise Exception(f"TAXII version not specified")
+
+    # collect the collection URL(s) for the given collection name
+    collectionUrls = {}
+    for api_root in server.api_roots:
+        for collection in api_root.collections:
+            if (taxiiCollection == '*') or (collection.title.lower() == taxiiCollection.lower()):
+                collectionUrls[collection.title] = {
+                    'id': collection.id,
+                    'url': collection.url,
+                }
+
+    # connect to and retrieve indicator STIX objects from the collection URL(s)
+    for title, info in collectionUrls.items():
+        collection = (
+            TaxiiCollection_v21(info['url'], user=taxiiUsername, password=taxiiPassword, verify=sslVerify)
+            if taxiiVersion == '2.1'
+            else TaxiiCollection_v20(info['url'], user=taxiiUsername, password=taxiiPassword, verify=sslVerify)
+        )
+        try:
+            # loop over paginated results
+            for envelope in (
+                TaxiiAsPages_v21(
+                    collection.get_objects,
+                    per_request=TAXII_PAGE_SIZE,
+                    **TAXII_INDICATOR_FILTER,
+                )
+                if taxiiVersion == '2.1'
+                else TaxiiAsPages_v20(
+                    collection.get_objects,
+                    per_request=TAXII_PAGE_SIZE,
+                    **TAXII_INDICATOR_FILTER,
+                )
+            ):
+                if zeekPrinter.ProcessSTIX(
+                    envelope,
+                    source=[':'.join([x for x in [server.title, title] if x is not None])],
+                ):
+                    successCount.increment()
+
+        except Exception as e:
+            if logger is not None:
+                logger.warning(f"[{workerId}]: {type(e).__name__} for object of collection '{title}': {e}")
+
+
+def UpdateFromMandiant(
+    connInfo,
+    since,
+    nowTime,
+    sslVerify,
+    zeekPrinter,
+    logger,
+    successCount,
+    workerId,
+):
+    if mati_client := mandiant_threatintel.ThreatIntelClient(
+        api_key=connInfo.get('api_key', None),
+        secret_key=connInfo.get('secret_key', None),
+        bearer_token=connInfo.get('bearer_token', None),
+        api_base_url=connInfo.get('api_base_url', mandiant_threatintel.API_BASE_URL),
+        client_name=connInfo.get('client_name', mandiant_threatintel.CLIENT_APP_NAME),
+    ):
+        skip_attr_map = defaultdict(lambda: False)
+        skip_attr_map['campaigns'] = not bool(connInfo.get('include_campaigns', MANDIANT_INCLUDE_CAMPAIGNS_DEFAULT))
+        skip_attr_map['category'] = not bool(connInfo.get('include_category', MANDIANT_INCLUDE_CATEGORY_DEFAULT))
+        skip_attr_map['misp'] = not bool(connInfo.get('include_misp', MANDIANT_INCLUDE_MISP_DEFAULT))
+        skip_attr_map['reports'] = not bool(connInfo.get('include_reports', MANDIANT_INCLUDE_REPORTS_DEFAULT))
+        skip_attr_map['threat_rating'] = not bool(
+            connInfo.get('include_threat_rating', MANDIANT_INCLUDE_THREAT_RATING_DEFAULT)
+        )
+        skip_attr_map['attributed_associations'] = True
+        for indicator in mati_client.Indicators.get_list(
+            start_epoch=since if since else nowTime - relativedelta(hours=24),
+            end_epoch=nowTime,
+            page_size=connInfo.get('page_size', MANDIANT_PAGE_SIZE_DEFAULT),
+            minimum_mscore=connInfo.get('minimum_mscore', MANDIANT_MINIMUM_MSCORE_DEFAULT),
+            exclude_osint=connInfo.get('exclude_osint', MANDIANT_EXCLUDE_OSINT_DEFAULT),
+            include_campaigns=not skip_attr_map['campaigns'],
+            include_reports=not skip_attr_map['reports'],
+            include_threat_rating=not skip_attr_map['threat_rating'],
+            include_misp=not skip_attr_map['misp'],
+            include_category=skip_attr_map['category'],
+        ):
+            try:
+                if zeekPrinter.ProcessMandiant(indicator, skip_attr_map=skip_attr_map):
+                    successCount.increment()
+            except Exception as e:
+                if logger is not None:
+                    logger.warning(
+                        f"[{workerId}]: {type(e).__name__} for Mandiant indicator {indicator.id if isinstance(indicator, mandiant_threatintel.APIResponse) else ''}: {e}"
+                    )
+
+    else:
+        raise Exception("Could not connect to Mandiant threat intelligence service")
+
+
 def ProcessThreatInputWorker(threatInputWorkerArgs):
     inputQueue, zeekPrinter, since, sslVerify, defaultNow, workerThreadCount, successCount, logger = (
         threatInputWorkerArgs[0],
@@ -823,327 +1173,76 @@ def ProcessThreatInputWorker(threatInputWorkerArgs):
 
                         elif isinstance(inarg, dict):
                             ##################################################################################
-                            # Connection parameters specified in dict (e.g., Mandiant Threat Intel)
+                            # Connection parameters specified in dict (e.g., Mandiant Threat Intel) from a YAML file
                             if ('type' in inarg) and (threatFeedType := str(inarg['type'])):
-
-                                if threatFeedType.lower() == 'mandiant':
-                                    if mati_client := mandiant_threatintel.ThreatIntelClient(
-                                        api_key=inarg.get('api_key', None),
-                                        secret_key=inarg.get('secret_key', None),
-                                        bearer_token=inarg.get('bearer_token', None),
-                                        api_base_url=inarg.get('api_base_url', mandiant_threatintel.API_BASE_URL),
-                                        client_name=inarg.get('client_name', mandiant_threatintel.CLIENT_APP_NAME),
-                                    ):
-                                        skip_attr_map = defaultdict(lambda: False)
-                                        skip_attr_map['campaigns'] = not bool(
-                                            inarg.get('include_campaigns', MANDIANT_INCLUDE_CAMPAIGNS_DEFAULT)
-                                        )
-                                        skip_attr_map['category'] = not bool(
-                                            inarg.get('include_category', MANDIANT_INCLUDE_CATEGORY_DEFAULT)
-                                        )
-                                        skip_attr_map['misp'] = not bool(
-                                            inarg.get('include_misp', MANDIANT_INCLUDE_MISP_DEFAULT)
-                                        )
-                                        skip_attr_map['reports'] = not bool(
-                                            inarg.get('include_reports', MANDIANT_INCLUDE_REPORTS_DEFAULT)
-                                        )
-                                        skip_attr_map['threat_rating'] = not bool(
-                                            inarg.get('include_threat_rating', MANDIANT_INCLUDE_THREAT_RATING_DEFAULT)
-                                        )
-                                        skip_attr_map['attributed_associations'] = True
-                                        nowTime = datetime.now().astimezone(UTCTimeZone)
-                                        for indicator in mati_client.Indicators.get_list(
-                                            start_epoch=since if since else nowTime - relativedelta(hours=24),
-                                            end_epoch=nowTime,
-                                            page_size=inarg.get('page_size', MANDIANT_PAGE_SIZE_DEFAULT),
-                                            minimum_mscore=inarg.get('minimum_mscore', MANDIANT_MINIMUM_MSCORE_DEFAULT),
-                                            exclude_osint=inarg.get('exclude_osint', MANDIANT_EXCLUDE_OSINT_DEFAULT),
-                                            include_campaigns=not skip_attr_map['campaigns'],
-                                            include_reports=not skip_attr_map['reports'],
-                                            include_threat_rating=not skip_attr_map['threat_rating'],
-                                            include_misp=not skip_attr_map['misp'],
-                                            include_category=skip_attr_map['category'],
-                                        ):
-                                            try:
-                                                if zeekPrinter.ProcessMandiant(indicator, skip_attr_map=skip_attr_map):
-                                                    successCount.increment()
-                                            except Exception as e:
-                                                if logger is not None:
-                                                    logger.warning(
-                                                        f"[{workerId}]: {type(e).__name__} for Mandiant indicator {indicator.id if isinstance(indicator, mandiant_threatintel.APIResponse) else ''}: {e}"
-                                                    )
-
-                                    else:
-                                        raise Exception("Could not connect to Mandiant threat intelligence service")
+                                if threatFeedType.lower() == 'misp':
+                                    UpdateFromMISP(
+                                        inarg,
+                                        since,
+                                        defaultNow,
+                                        sslVerify,
+                                        zeekPrinter,
+                                        logger,
+                                        successCount,
+                                        workerId,
+                                    )
+                                elif threatFeedType.lower() == 'taxii':
+                                    UpdateFromTAXII(
+                                        inarg,
+                                        since,
+                                        defaultNow,
+                                        sslVerify,
+                                        zeekPrinter,
+                                        logger,
+                                        successCount,
+                                        workerId,
+                                    )
+                                elif threatFeedType.lower() == 'mandiant':
+                                    UpdateFromMandiant(
+                                        inarg,
+                                        since,
+                                        defaultNow,
+                                        sslVerify,
+                                        zeekPrinter,
+                                        logger,
+                                        successCount,
+                                        workerId,
+                                    )
                                 else:
                                     raise Exception(f"Could not handle identify threat feed type '{threatFeedType}'")
                             else:
                                 raise Exception(f"Could not identify threat feed type in '{inarg}'")
 
-                        elif inarg.lower().startswith('misp'):
+                        elif isinstance(inarg, str) and inarg.lower().startswith('misp'):
                             ##################################################################################
                             # MISP URL
+                            # this is a MISP URL, connect and retrieve MISP indicators from it
 
-                            # this is a MISP URL, connect and retrieve STIX indicators from it
+                            mispConnInfoDict = defaultdict(lambda: None)
+                            mispConnInfoDict['type'] = 'misp'
                             # misp|misp_url|auth_key
-
-                            mispConnInfo = [base64_decode_if_prefixed(x) for x in inarg.split('|')[1::]]
-                            mispUrl, mispAuthKey = (
-                                None,
-                                None,
+                            mispConnInfoParts = [base64_decode_if_prefixed(x) for x in inarg.split('|')[1::]]
+                            mispConnInfoDict['url'] = mispConnInfoParts[0]
+                            if len(mispConnInfoParts) >= 2:
+                                mispConnInfoDict['auth_key'] = mispConnInfoParts[1]
+                            UpdateFromMISP(
+                                mispConnInfoDict,
+                                since,
+                                defaultNow,
+                                sslVerify,
+                                zeekPrinter,
+                                logger,
+                                successCount,
+                                workerId,
                             )
-                            mispUrl = mispConnInfo[0]
-                            if len(mispConnInfo) >= 2:
-                                mispAuthKey = mispConnInfo[1]
 
-                            with requests.Session() as mispSession:
-                                mispSession.headers.update(
-                                    {'Accept': 'application/json;q=1.0,text/plain;q=0.9,text/html;q=0.9'}
-                                )
-                                if mispAuthKey is not None:
-                                    mispSession.headers.update({'Authorization': mispAuthKey})
-
-                                # download the URL and parse as JSON to figure out what it is. it could be:
-                                # - a manifest JSON (https://www.circl.lu/doc/misp/feed-osint/manifest.json)
-                                # - a directory listing *containing* a manifest.json (https://www.circl.lu/doc/misp/feed-osint/)
-                                # - a directory listing of misc. JSON files without a manifest.json
-                                # - an array of Attributes returned for a request via the MISP Automation API to an /attributes endpoint
-                                # - an array of Events returned for a request via the MISP Automation API to an /events endpoint
-                                mispResponse = mispSession.get(
-                                    mispUrl,
-                                    allow_redirects=True,
-                                    verify=sslVerify,
-                                )
-                                mispResponse.raise_for_status()
-                                if mispJson := LoadStrIfJson(mispResponse.content):
-                                    # the contents are JSON. determine if this is:
-                                    #   - a single Event
-                                    #   - an array of Events
-                                    #   - an array of Attributes
-                                    #   - a manifest
-
-                                    if (
-                                        isinstance(mispJson, dict)
-                                        and (len(mispJson.keys()) == 1)
-                                        and ('Event' in mispJson)
-                                    ):
-                                        # this is a single MISP Event, process it
-                                        if zeekPrinter.ProcessMISP(
-                                            mispJson,
-                                            url=mispUrl,
-                                        ):
-                                            successCount.increment()
-
-                                    elif isinstance(mispJson, list) and (len(mispJson) > 0):
-                                        # are these Attributes or Events?
-                                        if (
-                                            isinstance(mispJson[0], dict)
-                                            and ('id' in mispJson[0])
-                                            and ('type' in mispJson[0])
-                                        ):
-                                            controllerType = 'attributes'
-                                            resultKey = 'Attribute'
-                                            pageSize = MISP_PAGE_SIZE_ATTRIBUTES
-                                        elif isinstance(mispJson[0], dict) and ('info' in mispJson[0]):
-                                            controllerType = 'events'
-                                            resultKey = 'Event'
-                                            pageSize = MISP_PAGE_SIZE_EVENTS
-                                        else:
-                                            controllerType = None
-                                            resultKey = None
-                                            pageSize = None
-
-                                        if controllerType:
-                                            # this is an array of either Attributes or Events.
-                                            #   rather than handling it via additional calls with request,
-                                            #   let's use the MISP API to do the searching/pulling
-                                            #   (yeah, we're duplicating the effort of pulling the
-                                            #   first page, but meh, who cares?)
-                                            if mispObject := PyMISP(
-                                                mispUrl,
-                                                mispAuthKey,
-                                                sslVerify,
-                                                debug=logger and (LOGGING_DEBUG >= logger.root.level),
-                                            ):
-                                                # search, looping over the pages pageSize at a time
-                                                mispPage = 0
-                                                while True:
-                                                    mispPage += 1
-                                                    resultCount = 0
-                                                    mispResults = mispObject.search(
-                                                        controller=controllerType,
-                                                        return_format='json',
-                                                        limit=pageSize,
-                                                        page=mispPage,
-                                                        type_attribute=list(MISP_ZEEK_INTEL_TYPE_MAP.keys()),
-                                                        timestamp=since,
-                                                    )
-                                                    if (
-                                                        mispResults
-                                                        and isinstance(mispResults, dict)
-                                                        and (resultKey in mispResults)
-                                                    ):
-                                                        # Attributes results
-                                                        resultCount = len(mispResults[resultKey])
-                                                        for item in mispResults[resultKey]:
-                                                            try:
-                                                                if zeekPrinter.ProcessMISP(
-                                                                    item,
-                                                                    url=mispUrl,
-                                                                ):
-                                                                    successCount.increment()
-                                                            except Exception as e:
-                                                                if logger is not None:
-                                                                    logger.warning(
-                                                                        f"[{workerId}]: {type(e).__name__} for MISP {resultKey}: {e}"
-                                                                    )
-
-                                                    elif mispResults and isinstance(mispResults, list):
-                                                        # Events results
-                                                        resultCount = len(mispResults)
-                                                        for item in mispResults:
-                                                            if item and isinstance(item, dict) and (resultKey in item):
-                                                                try:
-                                                                    if zeekPrinter.ProcessMISP(
-                                                                        item[resultKey],
-                                                                        url=mispUrl,
-                                                                    ):
-                                                                        successCount.increment()
-                                                                except Exception as e:
-                                                                    if logger is not None:
-                                                                        logger.warning(
-                                                                            f"[{workerId}]: {type(e).__name__} for MISP {resultKey}: {e}"
-                                                                        )
-
-                                                    else:
-                                                        # error or unrecognized results, set this to short circuit
-                                                        resultCount = 0
-
-                                                    if (logger is not None) and (LOGGING_DEBUG >= logger.root.level):
-                                                        logger.debug(
-                                                            f"[{workerId}]: MISP search page {mispPage} returned {resultCount}"
-                                                        )
-                                                    if not mispResults or (resultCount < pageSize):
-                                                        break
-
-                                        else:
-                                            # not an Event or an Attribute? what the heck are we even doing?
-                                            raise Exception(f"Unknown MISP object '{json.dumps(mispJson)}'")
-
-                                    elif isinstance(mispJson, dict):
-                                        # this is a manifest, loop over, retrieve and process the MISP events it references
-                                        for uri in mispJson:
-                                            try:
-                                                newUrl = urljoin(mispUrl, f'{uri}.json')
-                                                eventTime = (
-                                                    datetime.utcfromtimestamp(
-                                                        int(mispJson[uri]['timestamp'])
-                                                    ).astimezone(UTCTimeZone)
-                                                    if 'timestamp' in mispJson[uri]
-                                                    else defaultNow
-                                                )
-                                                if (since is None) or (eventTime >= since):
-                                                    mispObjectReponse = mispSession.get(
-                                                        newUrl,
-                                                        allow_redirects=True,
-                                                        verify=sslVerify,
-                                                    )
-                                                    mispObjectReponse.raise_for_status()
-                                                    if zeekPrinter.ProcessMISP(
-                                                        mispObjectReponse.json(),
-                                                        url=newUrl,
-                                                    ):
-                                                        successCount.increment()
-                                            except Exception as e:
-                                                if logger is not None:
-                                                    logger.warning(
-                                                        f"[{workerId}]: {type(e).__name__} for MISP object at '{newUrl}': {e}"
-                                                    )
-
-                                    else:
-                                        raise Exception(f"Unknown MISP format '{type(mispJson)}'")
-
-                                else:
-                                    # the contents are NOT JSON, it's probably an HTML-formatted directory listing
-
-                                    # retrieve the links listed (non-recursive, all .json files in this directory)
-                                    paths = get_url_paths_from_response(
-                                        mispResponse.text, parent_url=mispUrl, ext='.json'
-                                    )
-
-                                    # see if manifest.json exists in this directory
-                                    manifestPaths = [x for x in paths if x.endswith('/manifest.json')]
-                                    if len(manifestPaths) > 0:
-                                        # the manifest.json exists!
-                                        # retrieve it, then loop over it and retrieve and process the MISP events it references
-                                        for url in manifestPaths:
-                                            try:
-                                                mispManifestResponse = mispSession.get(
-                                                    url,
-                                                    allow_redirects=True,
-                                                    verify=sslVerify,
-                                                )
-                                                mispManifestResponse.raise_for_status()
-                                                mispManifest = mispManifestResponse.json()
-                                                for uri in mispManifest:
-                                                    try:
-                                                        eventTime = (
-                                                            datetime.utcfromtimestamp(
-                                                                int(mispManifest[uri]['timestamp'])
-                                                            ).astimezone(UTCTimeZone)
-                                                            if 'timestamp' in mispManifest[uri]
-                                                            else defaultNow
-                                                        )
-                                                        if (since is None) or (eventTime >= since):
-                                                            newUrl = f'{mispUrl.strip("/")}/{uri}.json'
-                                                            mispObjectReponse = mispSession.get(
-                                                                newUrl,
-                                                                allow_redirects=True,
-                                                                verify=sslVerify,
-                                                            )
-                                                            mispObjectReponse.raise_for_status()
-                                                            if zeekPrinter.ProcessMISP(
-                                                                mispObjectReponse.json(),
-                                                                url=newUrl,
-                                                            ):
-                                                                successCount.increment()
-                                                    except Exception as e:
-                                                        if logger is not None:
-                                                            logger.warning(
-                                                                f"[{workerId}]: {type(e).__name__} for MISP object at '{mispUrl}/{uri}.json': {e}"
-                                                            )
-                                            except Exception as e:
-                                                if logger is not None:
-                                                    logger.warning(
-                                                        f"[{workerId}]: {type(e).__name__} for manifest at '{url}': {e}"
-                                                    )
-
-                                    else:
-                                        # the manifest.json does not exist!
-                                        # just loop over, retrieve and process the .json files in this directory
-                                        for url in paths:
-                                            try:
-                                                mispObjectReponse = mispSession.get(
-                                                    url,
-                                                    allow_redirects=True,
-                                                    verify=sslVerify,
-                                                )
-                                                mispObjectReponse.raise_for_status()
-                                                if zeekPrinter.ProcessMISP(
-                                                    mispObjectReponse.json(),
-                                                    url=url,
-                                                ):
-                                                    successCount.increment()
-                                            except Exception as e:
-                                                if logger is not None:
-                                                    logger.warning(
-                                                        f"[{workerId}]: {type(e).__name__} for MISP object at '{url}': {e}"
-                                                    )
-
-                        elif inarg.lower().startswith('taxii'):
+                        elif isinstance(inarg, str) and inarg.lower().startswith('taxii'):
                             ##################################################################################
                             # TAXI (STIX) URL
 
+                            taxiiConnInfoDict = defaultdict(lambda: None)
+                            taxiiConnInfoDict['type'] = 'taxii'
+
                             # this is a TAXII URL, connect and retrieve STIX indicators from it
                             # taxii|2.0|discovery_url|collection_name|username|password
                             #
@@ -1152,83 +1251,28 @@ def ProcessThreatInputWorker(threatInputWorkerArgs):
                             # - "taxii|2.0|https://limo.anomali.com/api/v1/taxii2/taxii/|CyberCrime|guest|guest"
                             #
                             # collection_name can be specified as * to retrieve all collections (careful!)
-
                             taxiiConnInfo = [base64_decode_if_prefixed(x) for x in inarg.split('|')[1::]]
-                            taxiiVersion, taxiiDisoveryURL, taxiiCollectionName, taxiiUsername, taxiiPassword = (
-                                None,
-                                None,
-                                None,
-                                None,
-                                None,
-                            )
                             if len(taxiiConnInfo) >= 3:
-                                taxiiVersion, taxiiDisoveryURL, taxiiCollectionName = taxiiConnInfo[0:3]
+                                (
+                                    taxiiConnInfoDict['version'],
+                                    taxiiConnInfoDict['url'],
+                                    taxiiConnInfoDict['collection'],
+                                ) = taxiiConnInfo[0:3]
                             if len(taxiiConnInfo) >= 4:
-                                taxiiUsername = taxiiConnInfo[3]
+                                taxiiConnInfoDict['username'] = taxiiConnInfo[3]
                             if len(taxiiConnInfo) >= 5:
-                                taxiiPassword = taxiiConnInfo[4]
-
-                            # connect to the server with the appropriate API for the TAXII version
-                            if taxiiVersion == '2.0':
-                                server = TaxiiServer_v20(
-                                    taxiiDisoveryURL, user=taxiiUsername, password=taxiiPassword, verify=sslVerify
-                                )
-                            elif taxiiVersion == '2.1':
-                                server = TaxiiServer_v21(
-                                    taxiiDisoveryURL, user=taxiiUsername, password=taxiiPassword, verify=sslVerify
-                                )
-                            else:
-                                raise Exception(f"Unsupported TAXII version '{taxiiVersion}'")
-
-                            # collect the collection URL(s) for the given collection name
-                            collectionUrls = {}
-                            for api_root in server.api_roots:
-                                for collection in api_root.collections:
-                                    if (taxiiCollectionName == '*') or (
-                                        collection.title.lower() == taxiiCollectionName.lower()
-                                    ):
-                                        collectionUrls[collection.title] = {
-                                            'id': collection.id,
-                                            'url': collection.url,
-                                        }
-
-                            # connect to and retrieve indicator STIX objects from the collection URL(s)
-                            for title, info in collectionUrls.items():
-                                collection = (
-                                    TaxiiCollection_v21(
-                                        info['url'], user=taxiiUsername, password=taxiiPassword, verify=sslVerify
-                                    )
-                                    if taxiiVersion == '2.1'
-                                    else TaxiiCollection_v20(
-                                        info['url'], user=taxiiUsername, password=taxiiPassword, verify=sslVerify
-                                    )
-                                )
-                                try:
-                                    # loop over paginated results
-                                    for envelope in (
-                                        TaxiiAsPages_v21(
-                                            collection.get_objects,
-                                            per_request=TAXII_PAGE_SIZE,
-                                            **TAXII_INDICATOR_FILTER,
-                                        )
-                                        if taxiiVersion == '2.1'
-                                        else TaxiiAsPages_v20(
-                                            collection.get_objects,
-                                            per_request=TAXII_PAGE_SIZE,
-                                            **TAXII_INDICATOR_FILTER,
-                                        )
-                                    ):
-                                        if zeekPrinter.ProcessSTIX(
-                                            envelope,
-                                            source=[':'.join([x for x in [server.title, title] if x is not None])],
-                                        ):
-                                            successCount.increment()
-
-                                except Exception as e:
-                                    if logger is not None:
-                                        logger.warning(
-                                            f"[{workerId}]: {type(e).__name__} for object of collection '{title}': {e}"
-                                        )
+                                taxiiConnInfoDict['password'] = taxiiConnInfo[4]
+
+                            UpdateFromTAXII(
+                                taxiiConnInfoDict,
+                                since,
+                                defaultNow,
+                                sslVerify,
+                                zeekPrinter,
+                                logger,
+                                successCount,
+                                workerId,
+                            )
 
                 except Exception as e:
                     if logger is not None:

From 940eaf7893f2b94b08c32e39ad99d5b42b5fc6ca Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Tue, 12 Nov 2024 11:38:59 -0700
Subject: [PATCH 30/42] bump beats and logstas to v8.16.0

---
 Dockerfiles/filebeat.Dockerfile  | 2 +-
 Dockerfiles/logstash.Dockerfile  | 2 +-
 config/logstash.env.example      | 2 +-
 hedgehog-iso/build.sh            | 2 +-
 hedgehog-raspi/sensor_install.sh | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile
index 07879d8c4..8e1239942 100644
--- a/Dockerfiles/filebeat.Dockerfile
+++ b/Dockerfiles/filebeat.Dockerfile
@@ -1,4 +1,4 @@
-FROM docker.elastic.co/beats/filebeat-oss:8.15.3
+FROM docker.elastic.co/beats/filebeat-oss:8.16.0
 
 # Copyright (c) 2024 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="malcolm@inl.gov"
diff --git a/Dockerfiles/logstash.Dockerfile b/Dockerfiles/logstash.Dockerfile
index 69348bb05..c2ef0f626 100644
--- a/Dockerfiles/logstash.Dockerfile
+++ b/Dockerfiles/logstash.Dockerfile
@@ -1,4 +1,4 @@
-FROM docker.elastic.co/logstash/logstash-oss:8.15.3
+FROM docker.elastic.co/logstash/logstash-oss:8.16.0
 
 LABEL maintainer="malcolm@inl.gov"
 LABEL org.opencontainers.image.authors='malcolm@inl.gov'
diff --git a/config/logstash.env.example b/config/logstash.env.example
index 3db494a47..7f2682043 100644
--- a/config/logstash.env.example
+++ b/config/logstash.env.example
@@ -14,4 +14,4 @@ LOGSTASH_NETBOX_ENRICHMENT_DATASETS=suricata.alert,zeek.conn,zeek.dhcp,zeek.dns,
 # Zeek log types that will be ignored (dropped) by LogStash
 LOGSTASH_ZEEK_IGNORED_LOGS=analyzer,broker,cluster,config,loaded_scripts,packet_filter,png,print,prof,reporter,stderr,stdout
 # Logstash memory allowance and other Java options
-LS_JAVA_OPTS=-server -Xmx2500m -Xms2500m -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true
\ No newline at end of file
+LS_JAVA_OPTS=-server -Xmx2500m -Xms2500m -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true -Dlogstash.pipelinebus.implementation=v1
\ No newline at end of file
diff --git a/hedgehog-iso/build.sh b/hedgehog-iso/build.sh
index 862d50796..8430380df 100755
--- a/hedgehog-iso/build.sh
+++ b/hedgehog-iso/build.sh
@@ -5,7 +5,7 @@ IMAGE_PUBLISHER=idaholab
 IMAGE_VERSION=1.0.0
 IMAGE_DISTRIBUTION=bookworm
 
-BEATS_VER="8.15.3"
+BEATS_VER="8.16.0"
 BEATS_OSS="-oss"
 
 ARKIME_VER="5.5.0"
diff --git a/hedgehog-raspi/sensor_install.sh b/hedgehog-raspi/sensor_install.sh
index eb4122f29..c146edd73 100644
--- a/hedgehog-raspi/sensor_install.sh
+++ b/hedgehog-raspi/sensor_install.sh
@@ -34,7 +34,7 @@ SENSOR_DIR='/opt/sensor'
 
 ARKIME_VERSION="5.5.0"
 
-BEATS_VER="8.15.3"
+BEATS_VER="8.16.0"
 BEATS_OSS="-oss"
 
 # Option to build from sources if desired

From 29197f91c3038bc122fa315359413c4eaf556843 Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Tue, 12 Nov 2024 11:47:15 -0700
Subject: [PATCH 31/42] bump beats and logstas to v8.16.0

---
 Dockerfiles/filebeat.Dockerfile | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile
index 8e1239942..588657b9d 100644
--- a/Dockerfiles/filebeat.Dockerfile
+++ b/Dockerfiles/filebeat.Dockerfile
@@ -100,15 +100,14 @@ RUN export EVTXARCH=$(uname -m | sed 's/arm64/aarch64/') && \
         psmisc \
         python3-pip \
         python3-setuptools \
-        python3.9 \
+        python3 \
         rsync \
         tar \
         tini \
         unar \
         unzip \
         xz-utils && \
-    ln -s -f -r /usr/bin/python3.9 /usr/bin/python3 && \
-    python3.9 -m pip install --no-compile --no-cache-dir patool entrypoint2 pyunpack python-magic ordered-set supervisor watchdog==6.0.0 && \
+    python3 -m pip install --no-compile --no-cache-dir patool entrypoint2 pyunpack python-magic ordered-set supervisor watchdog==6.0.0 && \
     curl -fsSL -o /usr/local/bin/supercronic "${SUPERCRONIC_URL}${BINARCH}" && \
       chmod +x /usr/local/bin/supercronic && \
     curl -fsSL -o /usr/local/bin/yq "${YQ_URL}${BINARCH}" && \

From 27c6afd0988a409cdcc790fa73137ea258decfea Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Tue, 12 Nov 2024 11:48:34 -0700
Subject: [PATCH 32/42] bump beats and logstas to v8.16.0

---
 Dockerfiles/filebeat.Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile
index 588657b9d..18932f286 100644
--- a/Dockerfiles/filebeat.Dockerfile
+++ b/Dockerfiles/filebeat.Dockerfile
@@ -107,7 +107,7 @@ RUN export EVTXARCH=$(uname -m | sed 's/arm64/aarch64/') && \
         unar \
         unzip \
         xz-utils && \
-    python3 -m pip install --no-compile --no-cache-dir patool entrypoint2 pyunpack python-magic ordered-set supervisor watchdog==6.0.0 && \
+    python3 -m pip install --no-compile --no-cache-dir --break-system-packages patool entrypoint2 pyunpack python-magic ordered-set supervisor watchdog==6.0.0 && \
     curl -fsSL -o /usr/local/bin/supercronic "${SUPERCRONIC_URL}${BINARCH}" && \
       chmod +x /usr/local/bin/supercronic && \
     curl -fsSL -o /usr/local/bin/yq "${YQ_URL}${BINARCH}" && \

From c6e037eb777576c460e8c769d3353eb4291cb725 Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Tue, 12 Nov 2024 12:08:55 -0700
Subject: [PATCH 33/42] bump beats and logstas to v8.16.0

---
 filebeat/scripts/filebeat-process-zeek-folder.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/filebeat/scripts/filebeat-process-zeek-folder.sh b/filebeat/scripts/filebeat-process-zeek-folder.sh
index 450baf0e2..9aae1798e 100755
--- a/filebeat/scripts/filebeat-process-zeek-folder.sh
+++ b/filebeat/scripts/filebeat-process-zeek-folder.sh
@@ -39,7 +39,7 @@ if mkdir $LOCKDIR; then
   # get new logs ready for processing
   cd "$ZEEK_LOGS_DIR"
   find . -path ./processed -prune -o -path ./current -prune -o -path ./upload -prune -o -path ./extract_files -prune -o -path ./live -prune -o -type f -exec file --separator '|' --mime-type "{}" \; | grep -P "(application/gzip|application/x-gzip|application/x-7z-compressed|application/x-bzip2|application/x-cpio|application/x-lzip|application/x-lzma|application/x-rar-compressed|application/x-tar|application/x-xz|application/zip|application/x-ms-evtx|application/octet-stream)" | sort -V | \
-    xargs -n 1 -P $FILEBEAT_PREPARE_PROCESS_COUNT -I '{}' bash -c '
+    xargs -P $FILEBEAT_PREPARE_PROCESS_COUNT -I '{}' bash -c '
 
     # separate filename and mime type
     FILENAME="$( echo "{}" | awk -F"|" "{print \$1}" )"

From 28b6d655f9ca23761d7dd1f1b089d26929789678 Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Tue, 12 Nov 2024 12:42:12 -0700
Subject: [PATCH 34/42] for idaholab/Malcolm#491; added a logstash health check

---
 api/project/__init__.py              | 22 +++++-----------------
 docker-compose-dev.yml               |  2 +-
 docker-compose.yml                   |  2 +-
 kubernetes/14-logstash.yml           |  7 +++----
 logstash/scripts/container_health.sh | 12 ++++++++++++
 5 files changed, 22 insertions(+), 23 deletions(-)
 create mode 100755 logstash/scripts/container_health.sh

diff --git a/api/project/__init__.py b/api/project/__init__.py
index ff7e7cfb1..46a7c0111 100644
--- a/api/project/__init__.py
+++ b/api/project/__init__.py
@@ -169,15 +169,6 @@
 missing_field_map['ip'] = '0.0.0.0'
 missing_field_map['long'] = 0
 
-logstash_default_pipelines = [
-    "malcolm-beats",
-    "malcolm-enrichment",
-    "malcolm-input",
-    "malcolm-output",
-    "malcolm-suricata",
-    "malcolm-zeek",
-]
-
 urllib3.disable_warnings()
 warnings.filterwarnings(
     "ignore",
@@ -937,7 +928,7 @@ def ready():
     logstash_lumberjack
         true or false, the ready status of Logstash's lumberjack protocol listener
     logstash_pipelines
-        true or false, the ready status of Logstash's default pipelines
+        true or false, the ready status of Logstash's pipelines
     netbox
         true or false, the ready status of NetBox
     opensearch
@@ -998,9 +989,9 @@ def ready():
             print(f"{type(e).__name__}: {str(e)} getting freq status")
 
     try:
-        logstashStats = requests.get(f'{logstashUrl}/_node').json()
+        logstashHealth = requests.get(f'{logstashUrl}/_health_report').json()
     except Exception as e:
-        logstashStats = {}
+        logstashHealth = {}
         if debugApi:
             print(f"{type(e).__name__}: {str(e)} getting Logstash node status")
 
@@ -1057,11 +1048,8 @@ def ready():
         filebeat_tcp=filebeatTcpJsonStatus,
         freq=freqStatus,
         logstash_lumberjack=logstashLJStatus,
-        logstash_pipelines=(malcolm_utils.deep_get(logstashStats, ["status"]) == "green")
-        and all(
-            pipeline in malcolm_utils.deep_get(logstashStats, ["pipelines"], {})
-            for pipeline in logstash_default_pipelines
-        ),
+        logstash_pipelines=(malcolm_utils.deep_get(logstashHealth, ["status"]) == "green")
+        and (malcolm_utils.deep_get(logstashHealth, ["indicators", "pipelines", "status"]) == "green"),
         netbox=bool(
             isinstance(netboxStatus, dict)
             and netboxStatus
diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
index f671b0379..8ae8694e0 100644
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -290,7 +290,7 @@ services:
       target: /usr/share/logstash/malcolm-ruby
       read_only: true
     healthcheck:
-      test: ["CMD", "curl", "--silent", "--fail", "http://localhost:9600"]
+      test: ["CMD", "/usr/local/bin/container_health.sh"]
       interval: 30s
       timeout: 15s
       retries: 3
diff --git a/docker-compose.yml b/docker-compose.yml
index 770cd05ce..6128cc55d 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -254,7 +254,7 @@ services:
       target: /certs/server.key
       read_only: true
     healthcheck:
-      test: ["CMD", "curl", "--silent", "--fail", "http://localhost:9600"]
+      test: ["CMD", "/usr/local/bin/container_health.sh"]
       interval: 30s
       timeout: 15s
       retries: 3
diff --git a/kubernetes/14-logstash.yml b/kubernetes/14-logstash.yml
index 30b9b0f82..1239914e8 100644
--- a/kubernetes/14-logstash.yml
+++ b/kubernetes/14-logstash.yml
@@ -90,10 +90,9 @@ spec:
           - configMapRef:
               name: logstash-env
         livenessProbe:
-          httpGet:
-            path: /
-            port: 9600
-            scheme: HTTP
+          exec:
+            command:
+            - /usr/local/bin/container_health.sh
           initialDelaySeconds: 600
           periodSeconds: 30
           timeoutSeconds: 15
diff --git a/logstash/scripts/container_health.sh b/logstash/scripts/container_health.sh
new file mode 100755
index 000000000..de11eb407
--- /dev/null
+++ b/logstash/scripts/container_health.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -euo pipefail
+
+JQ_EVAL=$(
+    curl --fail --silent -XGET http://localhost:9600/_health_report | \
+        jq '(.status == "green") and
+            (.indicators.pipelines.status == "green") and
+            (all(.indicators.pipelines.indicators[]; .status == "green"))' 2>/dev/null
+)
+
+[[ "$JQ_EVAL" == "true" ]] && exit 0 && exit 1
\ No newline at end of file

From 429a45e7eca20d526f0a691ff177e914cf7a1d5a Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Tue, 12 Nov 2024 14:02:57 -0700
Subject: [PATCH 35/42] for idaholab/Malcolm#361, some tweaks to the nginx conf
 to make sure query parameters get passed in

---
 nginx/nginx.conf                          |  3 ++-
 shared/bin/extracted_files_http_server.py | 22 +++++-----------------
 2 files changed, 7 insertions(+), 18 deletions(-)

diff --git a/nginx/nginx.conf b/nginx/nginx.conf
index d837369dc..2de0a8b0f 100644
--- a/nginx/nginx.conf
+++ b/nginx/nginx.conf
@@ -194,7 +194,7 @@ http {
     # extracted file download
     location ~* ^/extracted-files\b(.*) {
       include /etc/nginx/nginx_auth_rt.conf;
-      proxy_pass http://extracted-file-http-server$1;
+      proxy_pass http://extracted-file-http-server$1$is_args$args;
       proxy_redirect off;
       proxy_set_header Host file-monitor.malcolm.local;
     }
@@ -204,6 +204,7 @@ http {
       include /etc/nginx/nginx_auth_rt.conf;
       include /etc/nginx/nginx_system_resolver.conf;
       set $upstream $1:8006;
+      # TODO: check, do i need is_args/args here?
       rewrite ^/hh-extracted-files/([a-zA-Z0-9-\.]+)(.*)$ $2 break;
       proxy_pass https://$upstream;
       proxy_ssl_verify off;
diff --git a/shared/bin/extracted_files_http_server.py b/shared/bin/extracted_files_http_server.py
index ca42fdf25..6c8d6a2a8 100755
--- a/shared/bin/extracted_files_http_server.py
+++ b/shared/bin/extracted_files_http_server.py
@@ -50,12 +50,6 @@
 malcolm_forward_header = 'X-Malcolm-Forward'
 
 
-###################################################################################################
-# a function for performing "natural" (case insensitive) sort
-def natural_sort_key(s, _nsre=re.compile('([0-9]+)')):
-    return [int(text) if text.isdigit() else text.lower() for text in _nsre.split(s)]
-
-
 ###################################################################################################
 # return the names and flags for Zipping a list of files
 def LocalFilesForZip(names):
@@ -146,7 +140,7 @@ def do_GET(self):
 
                             for dirpath, dirnames, filenames in os.walk(fullpath):
                                 # list directories first
-                                for dirname in sorted(dirnames, key=natural_sort_key):
+                                for dirname in sorted(dirnames, key=str.casefold):
                                     try:
                                         child = os.path.join(dirpath, dirname)
                                         if args.links or (not os.path.islink(child)):
@@ -154,7 +148,7 @@ def do_GET(self):
                                     except Exception as e:
                                         eprint(f'Error with directory "{dirname}"": {e}')
                                 # list files
-                                for filename in sorted(filenames, key=natural_sort_key):
+                                for filename in sorted(filenames, key=str.casefold):
                                     try:
                                         child = os.path.join(dirpath, filename)
                                         if args.links or (not os.path.islink(child)):
@@ -236,9 +230,7 @@ def do_GET(self):
                                                         timestamp = datetime.strptime(timestampStr, '%Y%m%d%H%M%S')
                                                         timestampStr = timestamp.isoformat()
                                                         timestampStartFilterStr = (
-                                                            (timestamp - timedelta(days=1))
-                                                            .isoformat()
-                                                            .split('.')[0]
+                                                            (timestamp - timedelta(days=1)).isoformat().split('.')[0]
                                                         )
                                                     except Exception as te:
                                                         if timestampStr:
@@ -296,7 +288,7 @@ def do_GET(self):
                                                 ),
                                                 td(sizeof_fmt(os.path.getsize(child)), style="text-align: right"),
                                             )
-                                            
+
                                             # show special malcolm columns if requested
                                             if showMalcolmCols:
                                                 if fmatch is not None:
@@ -348,11 +340,7 @@ def do_GET(self):
                                         li(span('Previous', cls='page-link disabled'), cls='page-item')
 
                                     # add a space between text
-                                    li(
-                                        ' ',
-                                        cls='page-item spacer',
-                                        style='width: 10px;'
-                                    )
+                                    li(' ', cls='page-item spacer', style='width: 10px;')
 
                                     # next page link
                                     if page < totalPages:

From a077875429a771f728f42613c0054ba55ea67363 Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Wed, 13 Nov 2024 06:54:45 -0700
Subject: [PATCH 36/42] bump elasticsearch and elasticsearch-dsl to 8.16.0

---
 api/requirements.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/api/requirements.txt b/api/requirements.txt
index aa497c6ab..3a87cbcf6 100644
--- a/api/requirements.txt
+++ b/api/requirements.txt
@@ -5,6 +5,6 @@ opensearch-py==2.6.0
 requests==2.32.0
 regex==2022.3.2
 dateparser==1.1.1
-elasticsearch==8.15.1
-elasticsearch-dsl==8.15.4
+elasticsearch==8.16.0
+elasticsearch-dsl==8.16.0
 psutil==5.9.8
\ No newline at end of file

From 301268005893a81f2ba2894a93021b898c4b2e4e Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Wed, 13 Nov 2024 07:41:41 -0700
Subject: [PATCH 37/42] decomplicate taxii server stuff

---
 shared/bin/zeek_threat_feed_utils.py | 45 +++++++++++++---------------
 1 file changed, 21 insertions(+), 24 deletions(-)

diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py
index 1077c1063..e9f9ab7cc 100644
--- a/shared/bin/zeek_threat_feed_utils.py
+++ b/shared/bin/zeek_threat_feed_utils.py
@@ -1010,15 +1010,19 @@ def UpdateFromTAXII(
     taxiiCollection = connInfo.get('collection', None)
     taxiiUsername = connInfo.get('username', None)
     taxiiPassword = connInfo.get('password', None)
-    if taxiiVersion := connInfo.get('version', None):
-        if taxiiVersion == '2.0':
-            server = TaxiiServer_v20(taxiiUrl, user=taxiiUsername, password=taxiiPassword, verify=sslVerify)
-        elif taxiiVersion == '2.1':
-            server = TaxiiServer_v21(taxiiUrl, user=taxiiUsername, password=taxiiPassword, verify=sslVerify)
-        else:
-            raise Exception(f"Unsupported TAXII version '{taxiiVersion}'")
+    taxiiVersion = str(connInfo.get('version', None))
+    if taxiiVersion == '2.0':
+        TaxiiServerClass = TaxiiServer_v20
+        TaxiiCollectionClass = TaxiiCollection_v20
+        TaxiiAsPagesClass = TaxiiAsPages_v20
+    elif taxiiVersion == '2.1':
+        TaxiiServerClass = TaxiiServer_v21
+        TaxiiCollectionClass = TaxiiCollection_v21
+        TaxiiAsPagesClass = TaxiiAsPages_v21
     else:
-        raise Exception(f"TAXII version not specified")
+        raise Exception(f"Unsupported TAXII version '{taxiiVersion}'")
+
+    server = TaxiiServerClass(taxiiUrl, user=taxiiUsername, password=taxiiPassword, verify=sslVerify)
 
     # collect the collection URL(s) for the given collection name
     collectionUrls = {}
@@ -1032,25 +1036,18 @@ def UpdateFromTAXII(
 
     # connect to and retrieve indicator STIX objects from the collection URL(s)
     for title, info in collectionUrls.items():
-        collection = (
-            TaxiiCollection_v21(info['url'], user=taxiiUsername, password=taxiiPassword, verify=sslVerify)
-            if taxiiVersion == '2.1'
-            else TaxiiCollection_v20(info['url'], user=taxiiUsername, password=taxiiPassword, verify=sslVerify)
+        collection = TaxiiCollectionClass(
+            info['url'],
+            user=taxiiUsername,
+            password=taxiiPassword,
+            verify=sslVerify,
         )
         try:
             # loop over paginated results
-            for envelope in (
-                TaxiiAsPages_v21(
-                    collection.get_objects,
-                    per_request=TAXII_PAGE_SIZE,
-                    **TAXII_INDICATOR_FILTER,
-                )
-                if taxiiVersion == '2.1'
-                else TaxiiAsPages_v20(
-                    collection.get_objects,
-                    per_request=TAXII_PAGE_SIZE,
-                    **TAXII_INDICATOR_FILTER,
-                )
+            for envelope in TaxiiAsPagesClass(
+                collection.get_objects,
+                per_request=TAXII_PAGE_SIZE,
+                **TAXII_INDICATOR_FILTER,
             ):
                 if zeekPrinter.ProcessSTIX(
                     envelope,

From 61895f738428a3eab5eccb50a14b2b0b9cf52eda Mon Sep 17 00:00:00 2001
From: Robert Ruidisch <robert.ruidisch@gmail.com>
Date: Wed, 13 Nov 2024 15:53:54 +0100
Subject: [PATCH 38/42] fixed errors when running appliance packager on macOS

---
 scripts/malcolm_appliance_packager.sh | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/scripts/malcolm_appliance_packager.sh b/scripts/malcolm_appliance_packager.sh
index 7bd1f26c3..8d39f2467 100755
--- a/scripts/malcolm_appliance_packager.sh
+++ b/scripts/malcolm_appliance_packager.sh
@@ -35,7 +35,11 @@ CURRENT_REV_SHA="$(git rev-parse --short --verify HEAD)"
 if [ -z "$CURRENT_REV_SHA" ]; then
   CURRENT_REV_TAG="$(date +%Y.%m.%d_%H:%M:%S)"
 else
-  CURRENT_REV_DATE="$(git log -1 --format="%at" | xargs -I{} date -d @{} +%Y%m%d_%H%M%S)"
+  if [[ "$(uname -s)" == 'Darwin' ]]; then
+    CURRENT_REV_DATE="$(git log -1 --format="%at" | xargs -I{} date -r {} +%Y%m%d_%H%M%S)"
+  else
+    CURRENT_REV_DATE="$(git log -1 --format="%at" | xargs -I{} date -d @{} +%Y%m%d_%H%M%S)"
+  fi
   if [ -z "$CURRENT_REV_DATE" ]; then
     CURRENT_REV_TAG="$(date +%Y.%m.%d_%H:%M:%S)"
   fi
@@ -145,7 +149,11 @@ if mkdir "$DESTDIR"; then
   cp $VERBOSE "$SCRIPT_PATH/malcolm_kubernetes.py" "$RUN_PATH/"
   cp $VERBOSE "$SCRIPT_PATH/malcolm_utils.py" "$RUN_PATH/"
 
-  tar $VERBOSE --numeric-owner --owner=0 --group=0 -czf "$DESTNAME" "./$(basename $DESTDIR)/"
+  if [[ "$(uname -s)" == "Darwin" ]]; then
+      tar $VERBOSE -czf "$DESTNAME" "./$(basename $DESTDIR)/"
+  else
+      tar $VERBOSE --numeric-owner --owner=0 --group=0 -czf "$DESTNAME" "./$(basename $DESTDIR)/"
+  fi
   echo "Packaged Malcolm to \"$DESTNAME\""
 
   unset CONFIRMATION

From 79958a1328dbca1d783a6c822249b7cd21f81065 Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Wed, 13 Nov 2024 12:07:09 -0700
Subject: [PATCH 39/42] minor fix for taxii

---
 shared/bin/zeek_threat_feed_utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py
index e9f9ab7cc..81d1e0697 100644
--- a/shared/bin/zeek_threat_feed_utils.py
+++ b/shared/bin/zeek_threat_feed_utils.py
@@ -638,12 +638,13 @@ def ProcessMandiant(self, indicator, skip_attr_map={}):
     def ProcessSTIX(
         self,
         toParse,
+        version=None,
         source: Union[Tuple[str], None] = None,
     ):
         result = False
         try:
             # parse the STIX and process all "Indicator" objects
-            for obj in STIXParse(toParse, allow_custom=True).objects:
+            for obj in STIXParse(toParse, allow_custom=True, version=version).objects:
                 if type(obj).__name__ == "Indicator":
                     if not result:
                         result = True
@@ -1051,6 +1052,7 @@ def UpdateFromTAXII(
             ):
                 if zeekPrinter.ProcessSTIX(
                     envelope,
+                    version=taxiiVersion,
                     source=[':'.join([x for x in [server.title, title] if x is not None])],
                 ):
                     successCount.increment()

From 9e142a630d780abb681d6539c1fcd364d35e9432 Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Wed, 13 Nov 2024 13:36:00 -0700
Subject: [PATCH 40/42] put page numbers in page labels for extracted files

---
 shared/bin/extracted_files_http_server.py | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/shared/bin/extracted_files_http_server.py b/shared/bin/extracted_files_http_server.py
index 6c8d6a2a8..8842add80 100755
--- a/shared/bin/extracted_files_http_server.py
+++ b/shared/bin/extracted_files_http_server.py
@@ -327,7 +327,11 @@ def do_GET(self):
                                         eprint(f'Error with file "{filename}": {e}')
 
                             # pagination controls
-                            with div(cls='pagination'):
+                            br()
+                            with div(
+                                cls='pagination',
+                                style='text-align: center; display: flex; justify-content: center; padding: 0;',
+                            ):
                                 with ul(
                                     cls='pagination-list',
                                     style='display: flex; list-style: none; justify-content: center; padding: 0;',
@@ -335,7 +339,14 @@ def do_GET(self):
                                     # previous page link
                                     if page > 1:
                                         prevPageUrl = f'?page={page - 1}&elements={elements}'
-                                        li(a('Previous', href=prevPageUrl, cls='page-link'), cls='page-item')
+                                        li(
+                                            a(
+                                                f'Previous ({page - 1})',
+                                                href=prevPageUrl,
+                                                cls='page-link',
+                                            ),
+                                            cls='page-item',
+                                        )
                                     else:
                                         li(span('Previous', cls='page-link disabled'), cls='page-item')
 
@@ -345,7 +356,10 @@ def do_GET(self):
                                     # next page link
                                     if page < totalPages:
                                         nextPageUrl = f'?page={page + 1}&elements={elements}'
-                                        li(a('Next', href=nextPageUrl, cls='page-link'), cls='page-item')
+                                        li(
+                                            a(f'Next ({page + 1} of {totalPages})', href=nextPageUrl, cls='page-link'),
+                                            cls='page-item',
+                                        )
                                     else:
                                         li(span('Next', cls='page-link disabled'), cls='page-item')
 

From 4a77400f4c80a972000db53fd61db4248170e774 Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Wed, 13 Nov 2024 14:01:02 -0700
Subject: [PATCH 41/42] add new opcua log type to malcolm

---
 arkime/etc/config.ini                         | 19 +++++++++-
 arkime/wise/source.zeeklogs.js                | 14 ++++++++
 .../composable/component/zeek_ot.json         | 15 ++++++++
 .../zeek/1043_zeek_opcua_binary.conf          | 35 +++++++++++++++++++
 logstash/pipelines/zeek/1200_zeek_mutate.conf | 13 +++++++
 .../pipelines/zeek/1300_zeek_normalize.conf   |  3 ++
 .../pipelines/zeek/1400_zeek_convert.conf     | 25 +++++++++++++
 7 files changed, 123 insertions(+), 1 deletion(-)

diff --git a/arkime/etc/config.ini b/arkime/etc/config.ini
index 3af5c6c5e..4aabd2f3a 100644
--- a/arkime/etc/config.ini
+++ b/arkime/etc/config.ini
@@ -2065,6 +2065,23 @@ zeek.opcua_binary_variant_metadata.built_in_data_type=db:zeek.opcua_binary_varia
 zeek.opcua_binary_variant_metadata.built_in_data_type_str=db:zeek.opcua_binary_variant_metadata.built_in_data_type_str;group:zeek_opcua;kind:termfield;viewerOnly:true;friendly:built_in_data_type_str;help:built_in_data_type_str
 zeek.opcua_binary_variant_metadata.variant_data_array_dim=db:zeek.opcua_binary_variant_metadata.variant_data_array_dim;group:zeek_opcua;kind:integer;viewerOnly:true;friendly:variant_data_array_dim;help:variant_data_array_dim
 
+# opcua_binary_write.log
+# https://github.com/cisagov/icsnpp-opcua-binary
+zeek.opcua_binary_write.node_id_encoding_mask=db:zeek.opcua_binary_write.node_id_encoding_mask;group:zeek_opcua_binary_write;kind:termfield;friendly:node_id_encoding_mask;help:node_id_encoding_mask
+zeek.opcua_binary_write.node_id_namespace_idx=db:zeek.opcua_binary_write.node_id_namespace_idx;group:zeek_opcua_binary_write;kind:integer;friendly:node_id_namespace_idx;help:node_id_namespace_idx
+zeek.opcua_binary_write.node_id_numeric=db:zeek.opcua_binary_write.node_id_numeric;group:zeek_opcua_binary_write;kind:integer;friendly:node_id_numeric;help:node_id_numeric
+zeek.opcua_binary_write.node_id_string=db:zeek.opcua_binary_write.node_id_string;group:zeek_opcua_binary_write;kind:termfield;friendly:node_id_string;help:node_id_string
+zeek.opcua_binary_write.node_id_guid=db:zeek.opcua_binary_write.node_id_guid;group:zeek_opcua_binary_write;kind:termfield;friendly:node_id_guid;help:node_id_guid
+zeek.opcua_binary_write.node_id_opaque=db:zeek.opcua_binary_write.node_id_opaque;group:zeek_opcua_binary_write;kind:termfield;friendly:node_id_opaque;help:node_id_opaque
+zeek.opcua_binary_write.attribute_id=db:zeek.opcua_binary_write.attribute_id;group:zeek_opcua_binary_write;kind:integer;friendly:attribute_id;help:attribute_id
+zeek.opcua_binary_write.attribute_id_str=db:zeek.opcua_binary_write.attribute_id_str;group:zeek_opcua_binary_write;kind:termfield;friendly:attribute_id_str;help:attribute_id_str
+zeek.opcua_binary_write.index_range=db:zeek.opcua_binary_write.index_range;group:zeek_opcua_binary_write;kind:termfield;friendly:index_range;help:index_range
+zeek.opcua_binary_write.data_value_encoding_mask=db:zeek.opcua_binary_write.data_value_encoding_mask;group:zeek_opcua_binary_write;kind:termfield;friendly:data_value_encoding_mask;help:data_value_encoding_mask
+zeek.opcua_binary_write.source_timestamp=db:zeek.opcua_binary_write.source_timestamp;group:zeek_opcua_binary_write;kind:termfield;friendly:source_timestamp;help:source_timestamp
+zeek.opcua_binary_write.source_pico_sec=db:zeek.opcua_binary_write.source_pico_sec;group:zeek_opcua_binary_write;kind:integer;friendly:source_pico_sec;help:source_pico_sec
+zeek.opcua_binary_write.server_timestamp=db:zeek.opcua_binary_write.server_timestamp;group:zeek_opcua_binary_write;kind:termfield;friendly:server_timestamp;help:server_timestamp
+zeek.opcua_binary_write.server_pico_sec=db:zeek.opcua_binary_write.server_pico_sec;group:zeek_opcua_binary_write;kind:integer;friendly:server_pico_sec;help:server_pico_sec
+
 # ocsp.log
 # https://docs.zeek.org/en/stable/scripts/policy/files/x509/log-ocsp.zeek.html#type-OCSP::Info
 zeek.ocsp.hashAlgorithm=db:zeek.ocsp.hashAlgorithm;group:zeek_ocsp;kind:termfield;viewerOnly:true;friendly:Issuer Name and Key Hash Algorithm;help:Issuer Name and Key Hash Algorithm
@@ -3358,7 +3375,7 @@ o_zeek_notice=require:zeek.notice;title:Zeek notice.log;fields:rule.category,rul
 o_zeek_ntlm=require:zeek.ntlm;title:Zeek ntlm.log;fields:zeek.ntlm.host,zeek.ntlm.domain,zeek.ntlm.success,zeek.ntlm.status,zeek.ntlm.server_nb_computer,zeek.ntlm.server_dns_computer,zeek.ntlm.server_tree
 o_zeek_ntp=require:zeek.ntp;title:Zeek ntp.log;fields:zeek.ntp.version,zeek.ntp.mode,zeek.ntp.mode_str,zeek.ntp.stratum,zeek.ntp.poll,zeek.ntp.precision,zeek.ntp.root_delay,zeek.ntp.root_disp,zeek.ntp.ref_id,zeek.ntp.ref_time,zeek.ntp.org_time,zeek.ntp.rec_time,zeek.ntp.xmt_time,zeek.ntp.num_exts
 o_zeek_ocsp=require:zeek.ocsp;title:Zeek ocsp.log;fields:zeek.ocsp.hashAlgorithm,zeek.ocsp.issuerNameHash,zeek.ocsp.issuerKeyHash,zeek.ocsp.serialNumber,zeek.ocsp.certStatus,zeek.ocsp.revoketime,zeek.ocsp.revokereason,zeek.ocsp.thisUpdate,zeek.ocsp.nextUpdate
-o_zeek_opcua=require:zeek.opcua_binary;title:Zeek OPC UA Binary logs;fields:zeek.opcua_binary.filter_source_link_id,zeek.opcua_binary.operand_source_link_id,zeek.opcua_binary.variant_source_link_id,zeek.opcua_binary.encoding_mask,zeek.opcua_binary.endpoint_url,zeek.opcua_binary.error,zeek.opcua_binary.identifier,zeek.opcua_binary.identifier_str,zeek.opcua_binary.is_final,zeek.opcua_binary.max_chunk_cnt,zeek.opcua_binary.max_msg_size,zeek.opcua_binary.msg_size,zeek.opcua_binary.msg_type,zeek.opcua_binary.namespace_idx,zeek.opcua_binary.opcua_link_id,zeek.opcua_binary.rcv_buf_size,zeek.opcua_binary.rcv_cert,zeek.opcua_binary.rcv_cert_len,zeek.opcua_binary.reason,zeek.opcua_binary.req_hdr_add_hdr_enc_mask,zeek.opcua_binary.req_hdr_add_hdr_type_id,zeek.opcua_binary.req_hdr_audit_entry_id,zeek.opcua_binary.req_hdr_node_id_guid,zeek.opcua_binary.req_hdr_node_id_namespace_idx,zeek.opcua_binary.req_hdr_node_id_numeric,zeek.opcua_binary.req_hdr_node_id_opaque,zeek.opcua_binary.req_hdr_node_id_string,zeek.opcua_binary.req_hdr_node_id_type,zeek.opcua_binary.req_hdr_request_handle,zeek.opcua_binary.req_hdr_return_diag,zeek.opcua_binary.req_hdr_timeout_hint,zeek.opcua_binary.req_hdr_timestamp,zeek.opcua_binary.request_id,zeek.opcua_binary.res_hdr_add_hdr_enc_mask,zeek.opcua_binary.res_hdr_add_hdr_type_id,zeek.opcua_binary.res_hdr_request_handle,zeek.opcua_binary.res_hdr_service_diag_encoding,zeek.opcua_binary.res_hdr_timestamp,zeek.opcua_binary.sec_channel_id,zeek.opcua_binary.sec_policy_uri,zeek.opcua_binary.sec_policy_uri_len,zeek.opcua_binary.seq_number,zeek.opcua_binary.snd_buf_size,zeek.opcua_binary.snd_cert,zeek.opcua_binary.snd_cert_len,zeek.opcua_binary.version,zeek.opcua_binary_activate_session.client_algorithm,zeek.opcua_binary_activate_session.client_signature,zeek.opcua_binary_activate_session.ext_obj_certificate_data,zeek.opcua_binary_activate_session.ext_obj_encoding,zeek.opcua_binary_activate_session.ext_obj_encryption_algorithom,zeek.opcua_binary_activate_session.ext_obj_password,zeek.opcua_binary_activate_session.ext_obj_policy_id,zeek.opcua_binary_activate_session.ext_obj_token_data,zeek.opcua_binary_activate_session.ext_obj_type_id_encoding_mask,zeek.opcua_binary_activate_session.ext_obj_type_id_guid,zeek.opcua_binary_activate_session.ext_obj_type_id_namespace_idx,zeek.opcua_binary_activate_session.ext_obj_type_id_numeric,zeek.opcua_binary_activate_session.ext_obj_type_id_opaque,zeek.opcua_binary_activate_session.ext_obj_type_id_str,zeek.opcua_binary_activate_session.ext_obj_type_id_string,zeek.opcua_binary_activate_session.ext_obj_user_name,zeek.opcua_binary_activate_session.server_nonce,zeek.opcua_binary_activate_session.user_token_algorithm,zeek.opcua_binary_activate_session.user_token_signature,zeek.opcua_binary_activate_session_client_software_cert.cert_data,zeek.opcua_binary_activate_session_client_software_cert.cert_signature,zeek.opcua_binary_activate_session_client_software_cert.client_software_cert_link_id,zeek.opcua_binary_activate_session_locale_id.local_id,zeek.opcua_binary_activate_session_locale_id.opcua_locale_link_id,zeek.opcua_binary_aggregate_filter.aggregate_type_encoding_mask,zeek.opcua_binary_aggregate_filter.aggregate_type_guid,zeek.opcua_binary_aggregate_filter.aggregate_type_namespace_idx,zeek.opcua_binary_aggregate_filter.aggregate_type_numeric,zeek.opcua_binary_aggregate_filter.aggregate_type_opaque,zeek.opcua_binary_aggregate_filter.aggregate_type_string,zeek.opcua_binary_aggregate_filter.percent_data_bad,zeek.opcua_binary_aggregate_filter.percent_data_good,zeek.opcua_binary_aggregate_filter.processing_interval,zeek.opcua_binary_aggregate_filter.revised_percent_data_bad,zeek.opcua_binary_aggregate_filter.revised_percent_data_good,zeek.opcua_binary_aggregate_filter.revised_processing_interval,zeek.opcua_binary_aggregate_filter.revised_start_time,zeek.opcua_binary_aggregate_filter.revised_start_time_str,zeek.opcua_binary_aggregate_filter.revised_treat_uncertain_as_bad,zeek.opcua_binary_aggregate_filter.revised_use_server_capabilities_default,zeek.opcua_binary_aggregate_filter.revised_use_slopped_extrapolation,zeek.opcua_binary_aggregate_filter.start_time,zeek.opcua_binary_aggregate_filter.start_time_str,zeek.opcua_binary_aggregate_filter.treat_uncertain_as_bad,zeek.opcua_binary_aggregate_filter.use_server_capabilities_default,zeek.opcua_binary_aggregate_filter.use_slopped_extrapolation,zeek.opcua_binary_browse.browse_next_release_continuation_point,zeek.opcua_binary_browse.browse_service_type,zeek.opcua_binary_browse.browse_view_description_timestamp,zeek.opcua_binary_browse.browse_view_description_view_version,zeek.opcua_binary_browse.browse_view_id_encoding_mask,zeek.opcua_binary_browse.browse_view_id_guid,zeek.opcua_binary_browse.browse_view_id_namespace_idx,zeek.opcua_binary_browse.browse_view_id_numeric,zeek.opcua_binary_browse.browse_view_id_opaque,zeek.opcua_binary_browse.browse_view_id_string,zeek.opcua_binary_browse.req_max_ref_nodes,zeek.opcua_binary_browse_description.browse_description_encoding_mask,zeek.opcua_binary_browse_description.browse_description_guid,zeek.opcua_binary_browse_description.browse_description_include_subtypes,zeek.opcua_binary_browse_description.browse_description_link_id,zeek.opcua_binary_browse_description.browse_description_namespace_idx,zeek.opcua_binary_browse_description.browse_description_numeric,zeek.opcua_binary_browse_description.browse_description_opaque,zeek.opcua_binary_browse_description.browse_description_ref_encoding_mask,zeek.opcua_binary_browse_description.browse_description_ref_guid,zeek.opcua_binary_browse_description.browse_description_ref_namespace_idx,zeek.opcua_binary_browse_description.browse_description_ref_numeric,zeek.opcua_binary_browse_description.browse_description_ref_opaque,zeek.opcua_binary_browse_description.browse_description_ref_string,zeek.opcua_binary_browse_description.browse_description_string,zeek.opcua_binary_browse_description.browse_direction,zeek.opcua_binary_browse_description.browse_node_class_mask,zeek.opcua_binary_browse_description.browse_result_mask,zeek.opcua_binary_browse_request_continuation_point.browse_next_link_id,zeek.opcua_binary_browse_request_continuation_point.continuation_point,zeek.opcua_binary_browse_response_references.browse_reference_link_id,zeek.opcua_binary_browse_response_references.browse_response_display_name_locale,zeek.opcua_binary_browse_response_references.browse_response_display_name_mask,zeek.opcua_binary_browse_response_references.browse_response_display_name_text,zeek.opcua_binary_browse_response_references.browse_response_is_forward,zeek.opcua_binary_browse_response_references.browse_response_node_class,zeek.opcua_binary_browse_response_references.browse_response_ref_encoding_mask,zeek.opcua_binary_browse_response_references.browse_response_ref_guid,zeek.opcua_binary_browse_response_references.browse_response_ref_name,zeek.opcua_binary_browse_response_references.browse_response_ref_name_idx,zeek.opcua_binary_browse_response_references.browse_response_ref_namespace_idx,zeek.opcua_binary_browse_response_references.browse_response_ref_numeric,zeek.opcua_binary_browse_response_references.browse_response_ref_opaque,zeek.opcua_binary_browse_response_references.browse_response_ref_string,zeek.opcua_binary_browse_response_references.browse_response_ref_type_encoding_mask,zeek.opcua_binary_browse_response_references.browse_response_ref_type_guid,zeek.opcua_binary_browse_response_references.browse_response_ref_type_namespace_idx,zeek.opcua_binary_browse_response_references.browse_response_ref_type_namespace_uri,zeek.opcua_binary_browse_response_references.browse_response_ref_type_numeric,zeek.opcua_binary_browse_response_references.browse_response_ref_type_opaque,zeek.opcua_binary_browse_response_references.browse_response_ref_type_server_idx,zeek.opcua_binary_browse_response_references.browse_response_ref_type_string,zeek.opcua_binary_browse_response_references.browse_response_type_def_encoding_mask,zeek.opcua_binary_browse_response_references.browse_response_type_def_guid,zeek.opcua_binary_browse_response_references.browse_response_type_def_namespace_idx,zeek.opcua_binary_browse_response_references.browse_response_type_def_namespace_uri,zeek.opcua_binary_browse_response_references.browse_response_type_def_numeric,zeek.opcua_binary_browse_response_references.browse_response_type_def_opaque,zeek.opcua_binary_browse_response_references.browse_response_type_def_server_idx,zeek.opcua_binary_browse_response_references.browse_response_type_def_string,zeek.opcua_binary_browse_result.browse_response_link_id,zeek.opcua_binary_browse_result.browse_result_continuation_point,zeek.opcua_binary_close_session.del_subscriptions,zeek.opcua_binary_create_monitored_items.subscription_id,zeek.opcua_binary_create_monitored_items.timestamps_to_return,zeek.opcua_binary_create_monitored_items.timestamps_to_return_str,zeek.opcua_binary_create_monitored_items_create_item.create_item_link_id,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_attribute_id,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_index_range,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_name,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_namespace_idx,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_node_id_encoding_mask,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_node_id_guid,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_node_id_namespace_idx,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_node_id_numeric,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_node_id_opaque,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_node_id_string,zeek.opcua_binary_create_monitored_items_create_item.monitored_item_index_id,zeek.opcua_binary_create_monitored_items_create_item.monitoring_mode,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_client_handle,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_discard_oldest,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_encoding,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_node_id_encoding_mask,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_node_id_guid,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_node_id_namespace_idx,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_node_id_numeric,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_node_id_opaque,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_node_id_string,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_string,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_queue_size,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_revised_queue_size,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_revised_sampling_interval,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_sampling_interval,zeek.opcua_binary_create_session.algorithm,zeek.opcua_binary_create_session.application_type,zeek.opcua_binary_create_session.application_uri,zeek.opcua_binary_create_session.auth_token_encoding_mask,zeek.opcua_binary_create_session.auth_token_guid,zeek.opcua_binary_create_session.auth_token_namespace_idx,zeek.opcua_binary_create_session.auth_token_numeric,zeek.opcua_binary_create_session.auth_token_opaque,zeek.opcua_binary_create_session.auth_token_string,zeek.opcua_binary_create_session.client_cert,zeek.opcua_binary_create_session.client_cert_size,zeek.opcua_binary_create_session.client_nonce,zeek.opcua_binary_create_session.discovery_profile_uri,zeek.opcua_binary_create_session.encoding_mask,zeek.opcua_binary_create_session.endpoint_url,zeek.opcua_binary_create_session.gateway_server_uri,zeek.opcua_binary_create_session.locale,zeek.opcua_binary_create_session.max_req_msg_size,zeek.opcua_binary_create_session.max_res_msg_size,zeek.opcua_binary_create_session.product_uri,zeek.opcua_binary_create_session.req_session_timeout,zeek.opcua_binary_create_session.revised_session_timeout,zeek.opcua_binary_create_session.server_cert,zeek.opcua_binary_create_session.server_cert_size,zeek.opcua_binary_create_session.server_nonce,zeek.opcua_binary_create_session.server_uri,zeek.opcua_binary_create_session.session_id_encoding_mask,zeek.opcua_binary_create_session.session_id_guid,zeek.opcua_binary_create_session.session_id_namespace_idx,zeek.opcua_binary_create_session.session_id_numeric,zeek.opcua_binary_create_session.session_id_opaque,zeek.opcua_binary_create_session.session_id_string,zeek.opcua_binary_create_session.session_name,zeek.opcua_binary_create_session.signature,zeek.opcua_binary_create_session.text,zeek.opcua_binary_create_session_discovery.discovery_profile_link_id,zeek.opcua_binary_create_session_discovery.discovery_profile_uri,zeek.opcua_binary_create_session_discovery.discovery_profile_url,zeek.opcua_binary_create_session_endpoints.application_type,zeek.opcua_binary_create_session_endpoints.application_uri,zeek.opcua_binary_create_session_endpoints.cert_size,zeek.opcua_binary_create_session_endpoints.discovery_profile_uri,zeek.opcua_binary_create_session_endpoints.encoding_mask,zeek.opcua_binary_create_session_endpoints.endpoint_link_id,zeek.opcua_binary_create_session_endpoints.endpoint_url,zeek.opcua_binary_create_session_endpoints.gateway_server_uri,zeek.opcua_binary_create_session_endpoints.locale,zeek.opcua_binary_create_session_endpoints.message_security_mode,zeek.opcua_binary_create_session_endpoints.product_uri,zeek.opcua_binary_create_session_endpoints.security_level,zeek.opcua_binary_create_session_endpoints.security_policy_uri,zeek.opcua_binary_create_session_endpoints.server_cert,zeek.opcua_binary_create_session_endpoints.text,zeek.opcua_binary_create_session_endpoints.transport_profile_uri,zeek.opcua_binary_create_session_user_token.user_token_endpoint_url,zeek.opcua_binary_create_session_user_token.user_token_issued_type,zeek.opcua_binary_create_session_user_token.user_token_link_id,zeek.opcua_binary_create_session_user_token.user_token_policy_id,zeek.opcua_binary_create_session_user_token.user_token_sec_policy_uri,zeek.opcua_binary_create_session_user_token.user_token_type,zeek.opcua_binary_create_subscription.max_notifications_per_publish,zeek.opcua_binary_create_subscription.priority,zeek.opcua_binary_create_subscription.publishing_enabled,zeek.opcua_binary_create_subscription.requested_lifetime_count,zeek.opcua_binary_create_subscription.requested_max_keep_alive_count,zeek.opcua_binary_create_subscription.requested_publishing_interval,zeek.opcua_binary_create_subscription.revised_lifetime_count,zeek.opcua_binary_create_subscription.revised_max_keep_alive_count,zeek.opcua_binary_create_subscription.revised_publishing_interval,zeek.opcua_binary_create_subscription.subscription_id,zeek.opcua_binary_data_change_filter.deadband_type,zeek.opcua_binary_data_change_filter.deadband_value,zeek.opcua_binary_data_change_filter.trigger,zeek.opcua_binary_diag_info_detail.addl_info,zeek.opcua_binary_diag_info_detail.diag_info_link_id,zeek.opcua_binary_diag_info_detail.has_addl_info,zeek.opcua_binary_diag_info_detail.has_inner_diag_info,zeek.opcua_binary_diag_info_detail.has_inner_stat_code,zeek.opcua_binary_diag_info_detail.has_locale,zeek.opcua_binary_diag_info_detail.has_locale_txt,zeek.opcua_binary_diag_info_detail.has_namespace_uri,zeek.opcua_binary_diag_info_detail.has_symbolic_id,zeek.opcua_binary_diag_info_detail.inner_diag_level,zeek.opcua_binary_diag_info_detail.inner_stat_code,zeek.opcua_binary_diag_info_detail.locale,zeek.opcua_binary_diag_info_detail.locale_str,zeek.opcua_binary_diag_info_detail.locale_txt,zeek.opcua_binary_diag_info_detail.locale_txt_str,zeek.opcua_binary_diag_info_detail.namespace_uri,zeek.opcua_binary_diag_info_detail.namespace_uri_str,zeek.opcua_binary_diag_info_detail.root_object_id,zeek.opcua_binary_diag_info_detail.source,zeek.opcua_binary_diag_info_detail.source_str,zeek.opcua_binary_diag_info_detail.symbolic_id,zeek.opcua_binary_diag_info_detail.symbolic_id_str,zeek.opcua_binary_event_filter_attribute_operand.alias,zeek.opcua_binary_event_filter_attribute_operand.attribute,zeek.opcua_binary_event_filter_attribute_operand.index_range,zeek.opcua_binary_event_filter_attribute_operand.node_id_encoding_mask,zeek.opcua_binary_event_filter_attribute_operand.node_id_guid,zeek.opcua_binary_event_filter_attribute_operand.node_id_namespace_idx,zeek.opcua_binary_event_filter_attribute_operand.node_id_numeric,zeek.opcua_binary_event_filter_attribute_operand.node_id_opaque,zeek.opcua_binary_event_filter_attribute_operand.node_id_string,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.browse_path_element_link_id,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.include_subtypes,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.is_inverse,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.target_name,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.target_name_namespace_idx,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.type_id_encoding_mask,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.type_id_guid,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.type_id_namespace_idx,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.type_id_numeric,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.type_id_opaque,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.type_id_string,zeek.opcua_binary_event_filter_element_operand.element_index,zeek.opcua_binary_event_filter_select_clause.attribute_id,zeek.opcua_binary_event_filter_select_clause.index_range,zeek.opcua_binary_event_filter_select_clause.select_clause_link_id,zeek.opcua_binary_event_filter_select_clause.type_id_encoding_mask,zeek.opcua_binary_event_filter_select_clause.type_id_guid,zeek.opcua_binary_event_filter_select_clause.type_id_namespace_idx,zeek.opcua_binary_event_filter_select_clause.type_id_numeric,zeek.opcua_binary_event_filter_select_clause.type_id_opaque,zeek.opcua_binary_event_filter_select_clause.type_id_string,zeek.opcua_binary_event_filter_simple_attribute_operand.attribute_id,zeek.opcua_binary_event_filter_simple_attribute_operand.index_range,zeek.opcua_binary_event_filter_simple_attribute_operand.type_id_encoding_mask,zeek.opcua_binary_event_filter_simple_attribute_operand.type_id_guid,zeek.opcua_binary_event_filter_simple_attribute_operand.type_id_namespace_idx,zeek.opcua_binary_event_filter_simple_attribute_operand.type_id_numeric,zeek.opcua_binary_event_filter_simple_attribute_operand.type_id_opaque,zeek.opcua_binary_event_filter_simple_attribute_operand.type_id_string,zeek.opcua_binary_event_filter_simple_attribute_operand_browse_paths.browse_path_src,zeek.opcua_binary_event_filter_simple_attribute_operand_browse_paths.name,zeek.opcua_binary_event_filter_simple_attribute_operand_browse_paths.namespace_index,zeek.opcua_binary_event_filter_simple_attribute_operand_browse_paths.simple_attribute_operand_browse_path_link_id,zeek.opcua_binary_event_filter_where_clause.where_clause_link_id,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_element_link_id,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_encoding,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_node_id_encoding_mask,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_node_id_guid,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_node_id_namespace_idx,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_node_id_numeric,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_node_id_opaque,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_node_id_string,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_string,zeek.opcua_binary_event_filter_where_clause_elements.filter_operator,zeek.opcua_binary_get_endpoints.endpoint_url,zeek.opcua_binary_get_endpoints_description.application_type,zeek.opcua_binary_get_endpoints_description.application_uri,zeek.opcua_binary_get_endpoints_description.cert_size,zeek.opcua_binary_get_endpoints_description.discovery_profile_uri,zeek.opcua_binary_get_endpoints_description.encoding_mask,zeek.opcua_binary_get_endpoints_description.endpoint_description_link_id,zeek.opcua_binary_get_endpoints_description.endpoint_uri,zeek.opcua_binary_get_endpoints_description.gateway_server_uri,zeek.opcua_binary_get_endpoints_description.locale,zeek.opcua_binary_get_endpoints_description.message_security_mode,zeek.opcua_binary_get_endpoints_description.product_uri,zeek.opcua_binary_get_endpoints_description.security_level,zeek.opcua_binary_get_endpoints_description.security_policy_uri,zeek.opcua_binary_get_endpoints_description.server_cert,zeek.opcua_binary_get_endpoints_description.text,zeek.opcua_binary_get_endpoints_description.transport_profile_uri,zeek.opcua_binary_get_endpoints_discovery.discovery_profile_link_id,zeek.opcua_binary_get_endpoints_discovery.discovery_profile_url,zeek.opcua_binary_get_endpoints_locale_id.locale_id,zeek.opcua_binary_get_endpoints_locale_id.locale_link_id,zeek.opcua_binary_get_endpoints_profile_uri.profile_uri,zeek.opcua_binary_get_endpoints_profile_uri.profile_uri_link_id,zeek.opcua_binary_get_endpoints_user_token.user_token_endpoint_url,zeek.opcua_binary_get_endpoints_user_token.user_token_issued_type,zeek.opcua_binary_get_endpoints_user_token.user_token_link_id,zeek.opcua_binary_get_endpoints_user_token.user_token_policy_id,zeek.opcua_binary_get_endpoints_user_token.user_token_sec_policy_uri,zeek.opcua_binary_get_endpoints_user_token.user_token_type,zeek.opcua_binary_opensecure_channel.client_nonce,zeek.opcua_binary_opensecure_channel.client_proto_ver,zeek.opcua_binary_opensecure_channel.message_security_mode,zeek.opcua_binary_opensecure_channel.req_lifetime,zeek.opcua_binary_opensecure_channel.sec_token_created_at,zeek.opcua_binary_opensecure_channel.sec_token_id,zeek.opcua_binary_opensecure_channel.sec_token_request_type,zeek.opcua_binary_opensecure_channel.sec_token_revised_time,zeek.opcua_binary_opensecure_channel.sec_token_sec_channel_id,zeek.opcua_binary_opensecure_channel.server_nonce,zeek.opcua_binary_opensecure_channel.server_proto_ver,zeek.opcua_binary_read.max_age,zeek.opcua_binary_read.timestamps_to_return,zeek.opcua_binary_read.timestamps_to_return_str,zeek.opcua_binary_read_nodes_to_read.attribute_id,zeek.opcua_binary_read_nodes_to_read.attribute_id_str,zeek.opcua_binary_read_nodes_to_read.data_encoding_name,zeek.opcua_binary_read_nodes_to_read.data_encoding_name_idx,zeek.opcua_binary_read_nodes_to_read.index_range,zeek.opcua_binary_read_nodes_to_read.node_id_encoding_mask,zeek.opcua_binary_read_nodes_to_read.node_id_guid,zeek.opcua_binary_read_nodes_to_read.node_id_namespace_idx,zeek.opcua_binary_read_nodes_to_read.node_id_numeric,zeek.opcua_binary_read_nodes_to_read.node_id_opaque,zeek.opcua_binary_read_nodes_to_read.node_id_string,zeek.opcua_binary_read_nodes_to_read.nodes_to_read_link_id,zeek.opcua_binary_read_results.data_value_encoding_mask,zeek.opcua_binary_read_results.level,zeek.opcua_binary_read_results.results_link_id,zeek.opcua_binary_read_results.server_pico_sec,zeek.opcua_binary_read_results.server_timestamp,zeek.opcua_binary_read_results.source_pico_sec,zeek.opcua_binary_read_results.source_timestamp,zeek.opcua_binary_status_code_detail.historian_bits,zeek.opcua_binary_status_code_detail.historian_bits_str,zeek.opcua_binary_status_code_detail.historianextradata,zeek.opcua_binary_status_code_detail.historianmultivalue,zeek.opcua_binary_status_code_detail.historianpartial,zeek.opcua_binary_status_code_detail.info_type,zeek.opcua_binary_status_code_detail.info_type_str,zeek.opcua_binary_status_code_detail.limit_bits,zeek.opcua_binary_status_code_detail.limit_bits_str,zeek.opcua_binary_status_code_detail.overflow,zeek.opcua_binary_status_code_detail.semantics_changed,zeek.opcua_binary_status_code_detail.severity,zeek.opcua_binary_status_code_detail.severity_str,zeek.opcua_binary_status_code_detail.source,zeek.opcua_binary_status_code_detail.source_level,zeek.opcua_binary_status_code_detail.source_str,zeek.opcua_binary_status_code_detail.status_code,zeek.opcua_binary_status_code_detail.status_code_link_id,zeek.opcua_binary_status_code_detail.structure_changed,zeek.opcua_binary_status_code_detail.sub_code,zeek.opcua_binary_status_code_detail.sub_code_str,zeek.opcua_binary_variant_array_dims.array_dim_link_id,zeek.opcua_binary_variant_array_dims.dimension,zeek.opcua_binary_variant_data.variant_data_encoding_name,zeek.opcua_binary_variant_data.variant_data_encoding_name_idx,zeek.opcua_binary_variant_data.variant_data_link_id,zeek.opcua_binary_variant_data.variant_data_locale,zeek.opcua_binary_variant_data.variant_data_mask,zeek.opcua_binary_variant_data.variant_data_node_id_encoding_mask,zeek.opcua_binary_variant_data.variant_data_node_id_guid,zeek.opcua_binary_variant_data.variant_data_node_id_namespace_idx,zeek.opcua_binary_variant_data.variant_data_node_id_namespace_uri,zeek.opcua_binary_variant_data.variant_data_node_id_numeric,zeek.opcua_binary_variant_data.variant_data_node_id_opaque,zeek.opcua_binary_variant_data.variant_data_node_id_server_idx,zeek.opcua_binary_variant_data.variant_data_node_id_string,zeek.opcua_binary_variant_data.variant_data_text,zeek.opcua_binary_variant_data.variant_data_value_decimal,zeek.opcua_binary_variant_data.variant_data_value_signed_numeric,zeek.opcua_binary_variant_data.variant_data_value_string,zeek.opcua_binary_variant_data.variant_data_value_time,zeek.opcua_binary_variant_data.variant_data_value_unsigned_numeric,zeek.opcua_binary_variant_data_value.data_value_encoding_mask,zeek.opcua_binary_variant_data_value.server_pico_sec,zeek.opcua_binary_variant_data_value.server_timestamp,zeek.opcua_binary_variant_data_value.source_pico_sec,zeek.opcua_binary_variant_data_value.source_timestamp,zeek.opcua_binary_variant_data_value.variant_data_value_source_link,zeek.opcua_binary_variant_extension_object.ext_obj_encoding,zeek.opcua_binary_variant_extension_object.ext_obj_link_id,zeek.opcua_binary_variant_extension_object.ext_obj_node_id_encoding_mask,zeek.opcua_binary_variant_extension_object.ext_obj_node_id_guid,zeek.opcua_binary_variant_extension_object.ext_obj_node_id_namespace_idx,zeek.opcua_binary_variant_extension_object.ext_obj_node_id_numeric,zeek.opcua_binary_variant_extension_object.ext_obj_node_id_opaque,zeek.opcua_binary_variant_extension_object.ext_obj_node_id_string,zeek.opcua_binary_variant_extension_object.ext_obj_type_id_str,zeek.opcua_binary_variant_metadata.built_in_data_type,zeek.opcua_binary_variant_metadata.built_in_data_type_str,zeek.opcua_binary_variant_metadata.dara_variant_encoding_mask,zeek.opcua_binary_variant_metadata.data_variant_data_type,zeek.opcua_binary_variant_metadata.data_variant_data_type_str,zeek.opcua_binary_variant_metadata.variant_data_array_dim,zeek.opcua_binary_variant_metadata.variant_data_source,zeek.opcua_binary_variant_metadata.variant_data_source_str
+o_zeek_opcua=require:zeek.opcua_binary;title:Zeek OPC UA Binary logs;fields:zeek.opcua_binary.filter_source_link_id,zeek.opcua_binary.operand_source_link_id,zeek.opcua_binary.variant_source_link_id,zeek.opcua_binary.encoding_mask,zeek.opcua_binary.endpoint_url,zeek.opcua_binary.error,zeek.opcua_binary.identifier,zeek.opcua_binary.identifier_str,zeek.opcua_binary.is_final,zeek.opcua_binary.max_chunk_cnt,zeek.opcua_binary.max_msg_size,zeek.opcua_binary.msg_size,zeek.opcua_binary.msg_type,zeek.opcua_binary.namespace_idx,zeek.opcua_binary.opcua_link_id,zeek.opcua_binary.rcv_buf_size,zeek.opcua_binary.rcv_cert,zeek.opcua_binary.rcv_cert_len,zeek.opcua_binary.reason,zeek.opcua_binary.req_hdr_add_hdr_enc_mask,zeek.opcua_binary.req_hdr_add_hdr_type_id,zeek.opcua_binary.req_hdr_audit_entry_id,zeek.opcua_binary.req_hdr_node_id_guid,zeek.opcua_binary.req_hdr_node_id_namespace_idx,zeek.opcua_binary.req_hdr_node_id_numeric,zeek.opcua_binary.req_hdr_node_id_opaque,zeek.opcua_binary.req_hdr_node_id_string,zeek.opcua_binary.req_hdr_node_id_type,zeek.opcua_binary.req_hdr_request_handle,zeek.opcua_binary.req_hdr_return_diag,zeek.opcua_binary.req_hdr_timeout_hint,zeek.opcua_binary.req_hdr_timestamp,zeek.opcua_binary.request_id,zeek.opcua_binary.res_hdr_add_hdr_enc_mask,zeek.opcua_binary.res_hdr_add_hdr_type_id,zeek.opcua_binary.res_hdr_request_handle,zeek.opcua_binary.res_hdr_service_diag_encoding,zeek.opcua_binary.res_hdr_timestamp,zeek.opcua_binary.sec_channel_id,zeek.opcua_binary.sec_policy_uri,zeek.opcua_binary.sec_policy_uri_len,zeek.opcua_binary.seq_number,zeek.opcua_binary.snd_buf_size,zeek.opcua_binary.snd_cert,zeek.opcua_binary.snd_cert_len,zeek.opcua_binary.version,zeek.opcua_binary_activate_session.client_algorithm,zeek.opcua_binary_activate_session.client_signature,zeek.opcua_binary_activate_session.ext_obj_certificate_data,zeek.opcua_binary_activate_session.ext_obj_encoding,zeek.opcua_binary_activate_session.ext_obj_encryption_algorithom,zeek.opcua_binary_activate_session.ext_obj_password,zeek.opcua_binary_activate_session.ext_obj_policy_id,zeek.opcua_binary_activate_session.ext_obj_token_data,zeek.opcua_binary_activate_session.ext_obj_type_id_encoding_mask,zeek.opcua_binary_activate_session.ext_obj_type_id_guid,zeek.opcua_binary_activate_session.ext_obj_type_id_namespace_idx,zeek.opcua_binary_activate_session.ext_obj_type_id_numeric,zeek.opcua_binary_activate_session.ext_obj_type_id_opaque,zeek.opcua_binary_activate_session.ext_obj_type_id_str,zeek.opcua_binary_activate_session.ext_obj_type_id_string,zeek.opcua_binary_activate_session.ext_obj_user_name,zeek.opcua_binary_activate_session.server_nonce,zeek.opcua_binary_activate_session.user_token_algorithm,zeek.opcua_binary_activate_session.user_token_signature,zeek.opcua_binary_activate_session_client_software_cert.cert_data,zeek.opcua_binary_activate_session_client_software_cert.cert_signature,zeek.opcua_binary_activate_session_client_software_cert.client_software_cert_link_id,zeek.opcua_binary_activate_session_locale_id.local_id,zeek.opcua_binary_activate_session_locale_id.opcua_locale_link_id,zeek.opcua_binary_aggregate_filter.aggregate_type_encoding_mask,zeek.opcua_binary_aggregate_filter.aggregate_type_guid,zeek.opcua_binary_aggregate_filter.aggregate_type_namespace_idx,zeek.opcua_binary_aggregate_filter.aggregate_type_numeric,zeek.opcua_binary_aggregate_filter.aggregate_type_opaque,zeek.opcua_binary_aggregate_filter.aggregate_type_string,zeek.opcua_binary_aggregate_filter.percent_data_bad,zeek.opcua_binary_aggregate_filter.percent_data_good,zeek.opcua_binary_aggregate_filter.processing_interval,zeek.opcua_binary_aggregate_filter.revised_percent_data_bad,zeek.opcua_binary_aggregate_filter.revised_percent_data_good,zeek.opcua_binary_aggregate_filter.revised_processing_interval,zeek.opcua_binary_aggregate_filter.revised_start_time,zeek.opcua_binary_aggregate_filter.revised_start_time_str,zeek.opcua_binary_aggregate_filter.revised_treat_uncertain_as_bad,zeek.opcua_binary_aggregate_filter.revised_use_server_capabilities_default,zeek.opcua_binary_aggregate_filter.revised_use_slopped_extrapolation,zeek.opcua_binary_aggregate_filter.start_time,zeek.opcua_binary_aggregate_filter.start_time_str,zeek.opcua_binary_aggregate_filter.treat_uncertain_as_bad,zeek.opcua_binary_aggregate_filter.use_server_capabilities_default,zeek.opcua_binary_aggregate_filter.use_slopped_extrapolation,zeek.opcua_binary_browse.browse_next_release_continuation_point,zeek.opcua_binary_browse.browse_service_type,zeek.opcua_binary_browse.browse_view_description_timestamp,zeek.opcua_binary_browse.browse_view_description_view_version,zeek.opcua_binary_browse.browse_view_id_encoding_mask,zeek.opcua_binary_browse.browse_view_id_guid,zeek.opcua_binary_browse.browse_view_id_namespace_idx,zeek.opcua_binary_browse.browse_view_id_numeric,zeek.opcua_binary_browse.browse_view_id_opaque,zeek.opcua_binary_browse.browse_view_id_string,zeek.opcua_binary_browse.req_max_ref_nodes,zeek.opcua_binary_browse_description.browse_description_encoding_mask,zeek.opcua_binary_browse_description.browse_description_guid,zeek.opcua_binary_browse_description.browse_description_include_subtypes,zeek.opcua_binary_browse_description.browse_description_link_id,zeek.opcua_binary_browse_description.browse_description_namespace_idx,zeek.opcua_binary_browse_description.browse_description_numeric,zeek.opcua_binary_browse_description.browse_description_opaque,zeek.opcua_binary_browse_description.browse_description_ref_encoding_mask,zeek.opcua_binary_browse_description.browse_description_ref_guid,zeek.opcua_binary_browse_description.browse_description_ref_namespace_idx,zeek.opcua_binary_browse_description.browse_description_ref_numeric,zeek.opcua_binary_browse_description.browse_description_ref_opaque,zeek.opcua_binary_browse_description.browse_description_ref_string,zeek.opcua_binary_browse_description.browse_description_string,zeek.opcua_binary_browse_description.browse_direction,zeek.opcua_binary_browse_description.browse_node_class_mask,zeek.opcua_binary_browse_description.browse_result_mask,zeek.opcua_binary_browse_request_continuation_point.browse_next_link_id,zeek.opcua_binary_browse_request_continuation_point.continuation_point,zeek.opcua_binary_browse_response_references.browse_reference_link_id,zeek.opcua_binary_browse_response_references.browse_response_display_name_locale,zeek.opcua_binary_browse_response_references.browse_response_display_name_mask,zeek.opcua_binary_browse_response_references.browse_response_display_name_text,zeek.opcua_binary_browse_response_references.browse_response_is_forward,zeek.opcua_binary_browse_response_references.browse_response_node_class,zeek.opcua_binary_browse_response_references.browse_response_ref_encoding_mask,zeek.opcua_binary_browse_response_references.browse_response_ref_guid,zeek.opcua_binary_browse_response_references.browse_response_ref_name,zeek.opcua_binary_browse_response_references.browse_response_ref_name_idx,zeek.opcua_binary_browse_response_references.browse_response_ref_namespace_idx,zeek.opcua_binary_browse_response_references.browse_response_ref_numeric,zeek.opcua_binary_browse_response_references.browse_response_ref_opaque,zeek.opcua_binary_browse_response_references.browse_response_ref_string,zeek.opcua_binary_browse_response_references.browse_response_ref_type_encoding_mask,zeek.opcua_binary_browse_response_references.browse_response_ref_type_guid,zeek.opcua_binary_browse_response_references.browse_response_ref_type_namespace_idx,zeek.opcua_binary_browse_response_references.browse_response_ref_type_namespace_uri,zeek.opcua_binary_browse_response_references.browse_response_ref_type_numeric,zeek.opcua_binary_browse_response_references.browse_response_ref_type_opaque,zeek.opcua_binary_browse_response_references.browse_response_ref_type_server_idx,zeek.opcua_binary_browse_response_references.browse_response_ref_type_string,zeek.opcua_binary_browse_response_references.browse_response_type_def_encoding_mask,zeek.opcua_binary_browse_response_references.browse_response_type_def_guid,zeek.opcua_binary_browse_response_references.browse_response_type_def_namespace_idx,zeek.opcua_binary_browse_response_references.browse_response_type_def_namespace_uri,zeek.opcua_binary_browse_response_references.browse_response_type_def_numeric,zeek.opcua_binary_browse_response_references.browse_response_type_def_opaque,zeek.opcua_binary_browse_response_references.browse_response_type_def_server_idx,zeek.opcua_binary_browse_response_references.browse_response_type_def_string,zeek.opcua_binary_browse_result.browse_response_link_id,zeek.opcua_binary_browse_result.browse_result_continuation_point,zeek.opcua_binary_close_session.del_subscriptions,zeek.opcua_binary_create_monitored_items.subscription_id,zeek.opcua_binary_create_monitored_items.timestamps_to_return,zeek.opcua_binary_create_monitored_items.timestamps_to_return_str,zeek.opcua_binary_create_monitored_items_create_item.create_item_link_id,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_attribute_id,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_index_range,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_name,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_namespace_idx,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_node_id_encoding_mask,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_node_id_guid,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_node_id_namespace_idx,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_node_id_numeric,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_node_id_opaque,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_node_id_string,zeek.opcua_binary_create_monitored_items_create_item.monitored_item_index_id,zeek.opcua_binary_create_monitored_items_create_item.monitoring_mode,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_client_handle,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_discard_oldest,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_encoding,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_node_id_encoding_mask,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_node_id_guid,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_node_id_namespace_idx,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_node_id_numeric,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_node_id_opaque,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_node_id_string,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_string,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_queue_size,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_revised_queue_size,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_revised_sampling_interval,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_sampling_interval,zeek.opcua_binary_create_session.algorithm,zeek.opcua_binary_create_session.application_type,zeek.opcua_binary_create_session.application_uri,zeek.opcua_binary_create_session.auth_token_encoding_mask,zeek.opcua_binary_create_session.auth_token_guid,zeek.opcua_binary_create_session.auth_token_namespace_idx,zeek.opcua_binary_create_session.auth_token_numeric,zeek.opcua_binary_create_session.auth_token_opaque,zeek.opcua_binary_create_session.auth_token_string,zeek.opcua_binary_create_session.client_cert,zeek.opcua_binary_create_session.client_cert_size,zeek.opcua_binary_create_session.client_nonce,zeek.opcua_binary_create_session.discovery_profile_uri,zeek.opcua_binary_create_session.encoding_mask,zeek.opcua_binary_create_session.endpoint_url,zeek.opcua_binary_create_session.gateway_server_uri,zeek.opcua_binary_create_session.locale,zeek.opcua_binary_create_session.max_req_msg_size,zeek.opcua_binary_create_session.max_res_msg_size,zeek.opcua_binary_create_session.product_uri,zeek.opcua_binary_create_session.req_session_timeout,zeek.opcua_binary_create_session.revised_session_timeout,zeek.opcua_binary_create_session.server_cert,zeek.opcua_binary_create_session.server_cert_size,zeek.opcua_binary_create_session.server_nonce,zeek.opcua_binary_create_session.server_uri,zeek.opcua_binary_create_session.session_id_encoding_mask,zeek.opcua_binary_create_session.session_id_guid,zeek.opcua_binary_create_session.session_id_namespace_idx,zeek.opcua_binary_create_session.session_id_numeric,zeek.opcua_binary_create_session.session_id_opaque,zeek.opcua_binary_create_session.session_id_string,zeek.opcua_binary_create_session.session_name,zeek.opcua_binary_create_session.signature,zeek.opcua_binary_create_session.text,zeek.opcua_binary_create_session_discovery.discovery_profile_link_id,zeek.opcua_binary_create_session_discovery.discovery_profile_uri,zeek.opcua_binary_create_session_discovery.discovery_profile_url,zeek.opcua_binary_create_session_endpoints.application_type,zeek.opcua_binary_create_session_endpoints.application_uri,zeek.opcua_binary_create_session_endpoints.cert_size,zeek.opcua_binary_create_session_endpoints.discovery_profile_uri,zeek.opcua_binary_create_session_endpoints.encoding_mask,zeek.opcua_binary_create_session_endpoints.endpoint_link_id,zeek.opcua_binary_create_session_endpoints.endpoint_url,zeek.opcua_binary_create_session_endpoints.gateway_server_uri,zeek.opcua_binary_create_session_endpoints.locale,zeek.opcua_binary_create_session_endpoints.message_security_mode,zeek.opcua_binary_create_session_endpoints.product_uri,zeek.opcua_binary_create_session_endpoints.security_level,zeek.opcua_binary_create_session_endpoints.security_policy_uri,zeek.opcua_binary_create_session_endpoints.server_cert,zeek.opcua_binary_create_session_endpoints.text,zeek.opcua_binary_create_session_endpoints.transport_profile_uri,zeek.opcua_binary_create_session_user_token.user_token_endpoint_url,zeek.opcua_binary_create_session_user_token.user_token_issued_type,zeek.opcua_binary_create_session_user_token.user_token_link_id,zeek.opcua_binary_create_session_user_token.user_token_policy_id,zeek.opcua_binary_create_session_user_token.user_token_sec_policy_uri,zeek.opcua_binary_create_session_user_token.user_token_type,zeek.opcua_binary_create_subscription.max_notifications_per_publish,zeek.opcua_binary_create_subscription.priority,zeek.opcua_binary_create_subscription.publishing_enabled,zeek.opcua_binary_create_subscription.requested_lifetime_count,zeek.opcua_binary_create_subscription.requested_max_keep_alive_count,zeek.opcua_binary_create_subscription.requested_publishing_interval,zeek.opcua_binary_create_subscription.revised_lifetime_count,zeek.opcua_binary_create_subscription.revised_max_keep_alive_count,zeek.opcua_binary_create_subscription.revised_publishing_interval,zeek.opcua_binary_create_subscription.subscription_id,zeek.opcua_binary_data_change_filter.deadband_type,zeek.opcua_binary_data_change_filter.deadband_value,zeek.opcua_binary_data_change_filter.trigger,zeek.opcua_binary_diag_info_detail.addl_info,zeek.opcua_binary_diag_info_detail.diag_info_link_id,zeek.opcua_binary_diag_info_detail.has_addl_info,zeek.opcua_binary_diag_info_detail.has_inner_diag_info,zeek.opcua_binary_diag_info_detail.has_inner_stat_code,zeek.opcua_binary_diag_info_detail.has_locale,zeek.opcua_binary_diag_info_detail.has_locale_txt,zeek.opcua_binary_diag_info_detail.has_namespace_uri,zeek.opcua_binary_diag_info_detail.has_symbolic_id,zeek.opcua_binary_diag_info_detail.inner_diag_level,zeek.opcua_binary_diag_info_detail.inner_stat_code,zeek.opcua_binary_diag_info_detail.locale,zeek.opcua_binary_diag_info_detail.locale_str,zeek.opcua_binary_diag_info_detail.locale_txt,zeek.opcua_binary_diag_info_detail.locale_txt_str,zeek.opcua_binary_diag_info_detail.namespace_uri,zeek.opcua_binary_diag_info_detail.namespace_uri_str,zeek.opcua_binary_diag_info_detail.root_object_id,zeek.opcua_binary_diag_info_detail.source,zeek.opcua_binary_diag_info_detail.source_str,zeek.opcua_binary_diag_info_detail.symbolic_id,zeek.opcua_binary_diag_info_detail.symbolic_id_str,zeek.opcua_binary_event_filter_attribute_operand.alias,zeek.opcua_binary_event_filter_attribute_operand.attribute,zeek.opcua_binary_event_filter_attribute_operand.index_range,zeek.opcua_binary_event_filter_attribute_operand.node_id_encoding_mask,zeek.opcua_binary_event_filter_attribute_operand.node_id_guid,zeek.opcua_binary_event_filter_attribute_operand.node_id_namespace_idx,zeek.opcua_binary_event_filter_attribute_operand.node_id_numeric,zeek.opcua_binary_event_filter_attribute_operand.node_id_opaque,zeek.opcua_binary_event_filter_attribute_operand.node_id_string,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.browse_path_element_link_id,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.include_subtypes,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.is_inverse,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.target_name,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.target_name_namespace_idx,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.type_id_encoding_mask,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.type_id_guid,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.type_id_namespace_idx,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.type_id_numeric,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.type_id_opaque,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.type_id_string,zeek.opcua_binary_event_filter_element_operand.element_index,zeek.opcua_binary_event_filter_select_clause.attribute_id,zeek.opcua_binary_event_filter_select_clause.index_range,zeek.opcua_binary_event_filter_select_clause.select_clause_link_id,zeek.opcua_binary_event_filter_select_clause.type_id_encoding_mask,zeek.opcua_binary_event_filter_select_clause.type_id_guid,zeek.opcua_binary_event_filter_select_clause.type_id_namespace_idx,zeek.opcua_binary_event_filter_select_clause.type_id_numeric,zeek.opcua_binary_event_filter_select_clause.type_id_opaque,zeek.opcua_binary_event_filter_select_clause.type_id_string,zeek.opcua_binary_event_filter_simple_attribute_operand.attribute_id,zeek.opcua_binary_event_filter_simple_attribute_operand.index_range,zeek.opcua_binary_event_filter_simple_attribute_operand.type_id_encoding_mask,zeek.opcua_binary_event_filter_simple_attribute_operand.type_id_guid,zeek.opcua_binary_event_filter_simple_attribute_operand.type_id_namespace_idx,zeek.opcua_binary_event_filter_simple_attribute_operand.type_id_numeric,zeek.opcua_binary_event_filter_simple_attribute_operand.type_id_opaque,zeek.opcua_binary_event_filter_simple_attribute_operand.type_id_string,zeek.opcua_binary_event_filter_simple_attribute_operand_browse_paths.browse_path_src,zeek.opcua_binary_event_filter_simple_attribute_operand_browse_paths.name,zeek.opcua_binary_event_filter_simple_attribute_operand_browse_paths.namespace_index,zeek.opcua_binary_event_filter_simple_attribute_operand_browse_paths.simple_attribute_operand_browse_path_link_id,zeek.opcua_binary_event_filter_where_clause.where_clause_link_id,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_element_link_id,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_encoding,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_node_id_encoding_mask,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_node_id_guid,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_node_id_namespace_idx,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_node_id_numeric,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_node_id_opaque,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_node_id_string,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_string,zeek.opcua_binary_event_filter_where_clause_elements.filter_operator,zeek.opcua_binary_get_endpoints.endpoint_url,zeek.opcua_binary_get_endpoints_description.application_type,zeek.opcua_binary_get_endpoints_description.application_uri,zeek.opcua_binary_get_endpoints_description.cert_size,zeek.opcua_binary_get_endpoints_description.discovery_profile_uri,zeek.opcua_binary_get_endpoints_description.encoding_mask,zeek.opcua_binary_get_endpoints_description.endpoint_description_link_id,zeek.opcua_binary_get_endpoints_description.endpoint_uri,zeek.opcua_binary_get_endpoints_description.gateway_server_uri,zeek.opcua_binary_get_endpoints_description.locale,zeek.opcua_binary_get_endpoints_description.message_security_mode,zeek.opcua_binary_get_endpoints_description.product_uri,zeek.opcua_binary_get_endpoints_description.security_level,zeek.opcua_binary_get_endpoints_description.security_policy_uri,zeek.opcua_binary_get_endpoints_description.server_cert,zeek.opcua_binary_get_endpoints_description.text,zeek.opcua_binary_get_endpoints_description.transport_profile_uri,zeek.opcua_binary_get_endpoints_discovery.discovery_profile_link_id,zeek.opcua_binary_get_endpoints_discovery.discovery_profile_url,zeek.opcua_binary_get_endpoints_locale_id.locale_id,zeek.opcua_binary_get_endpoints_locale_id.locale_link_id,zeek.opcua_binary_get_endpoints_profile_uri.profile_uri,zeek.opcua_binary_get_endpoints_profile_uri.profile_uri_link_id,zeek.opcua_binary_get_endpoints_user_token.user_token_endpoint_url,zeek.opcua_binary_get_endpoints_user_token.user_token_issued_type,zeek.opcua_binary_get_endpoints_user_token.user_token_link_id,zeek.opcua_binary_get_endpoints_user_token.user_token_policy_id,zeek.opcua_binary_get_endpoints_user_token.user_token_sec_policy_uri,zeek.opcua_binary_get_endpoints_user_token.user_token_type,zeek.opcua_binary_opensecure_channel.client_nonce,zeek.opcua_binary_opensecure_channel.client_proto_ver,zeek.opcua_binary_opensecure_channel.message_security_mode,zeek.opcua_binary_opensecure_channel.req_lifetime,zeek.opcua_binary_opensecure_channel.sec_token_created_at,zeek.opcua_binary_opensecure_channel.sec_token_id,zeek.opcua_binary_opensecure_channel.sec_token_request_type,zeek.opcua_binary_opensecure_channel.sec_token_revised_time,zeek.opcua_binary_opensecure_channel.sec_token_sec_channel_id,zeek.opcua_binary_opensecure_channel.server_nonce,zeek.opcua_binary_opensecure_channel.server_proto_ver,zeek.opcua_binary_read.max_age,zeek.opcua_binary_read.timestamps_to_return,zeek.opcua_binary_read.timestamps_to_return_str,zeek.opcua_binary_read_nodes_to_read.attribute_id,zeek.opcua_binary_read_nodes_to_read.attribute_id_str,zeek.opcua_binary_read_nodes_to_read.data_encoding_name,zeek.opcua_binary_read_nodes_to_read.data_encoding_name_idx,zeek.opcua_binary_read_nodes_to_read.index_range,zeek.opcua_binary_read_nodes_to_read.node_id_encoding_mask,zeek.opcua_binary_read_nodes_to_read.node_id_guid,zeek.opcua_binary_read_nodes_to_read.node_id_namespace_idx,zeek.opcua_binary_read_nodes_to_read.node_id_numeric,zeek.opcua_binary_read_nodes_to_read.node_id_opaque,zeek.opcua_binary_read_nodes_to_read.node_id_string,zeek.opcua_binary_read_nodes_to_read.nodes_to_read_link_id,zeek.opcua_binary_read_results.data_value_encoding_mask,zeek.opcua_binary_read_results.level,zeek.opcua_binary_read_results.results_link_id,zeek.opcua_binary_read_results.server_pico_sec,zeek.opcua_binary_read_results.server_timestamp,zeek.opcua_binary_read_results.source_pico_sec,zeek.opcua_binary_read_results.source_timestamp,zeek.opcua_binary_status_code_detail.historian_bits,zeek.opcua_binary_status_code_detail.historian_bits_str,zeek.opcua_binary_status_code_detail.historianextradata,zeek.opcua_binary_status_code_detail.historianmultivalue,zeek.opcua_binary_status_code_detail.historianpartial,zeek.opcua_binary_status_code_detail.info_type,zeek.opcua_binary_status_code_detail.info_type_str,zeek.opcua_binary_status_code_detail.limit_bits,zeek.opcua_binary_status_code_detail.limit_bits_str,zeek.opcua_binary_status_code_detail.overflow,zeek.opcua_binary_status_code_detail.semantics_changed,zeek.opcua_binary_status_code_detail.severity,zeek.opcua_binary_status_code_detail.severity_str,zeek.opcua_binary_status_code_detail.source,zeek.opcua_binary_status_code_detail.source_level,zeek.opcua_binary_status_code_detail.source_str,zeek.opcua_binary_status_code_detail.status_code,zeek.opcua_binary_status_code_detail.status_code_link_id,zeek.opcua_binary_status_code_detail.structure_changed,zeek.opcua_binary_status_code_detail.sub_code,zeek.opcua_binary_status_code_detail.sub_code_str,zeek.opcua_binary_variant_array_dims.array_dim_link_id,zeek.opcua_binary_variant_array_dims.dimension,zeek.opcua_binary_variant_data.variant_data_encoding_name,zeek.opcua_binary_variant_data.variant_data_encoding_name_idx,zeek.opcua_binary_variant_data.variant_data_link_id,zeek.opcua_binary_variant_data.variant_data_locale,zeek.opcua_binary_variant_data.variant_data_mask,zeek.opcua_binary_variant_data.variant_data_node_id_encoding_mask,zeek.opcua_binary_variant_data.variant_data_node_id_guid,zeek.opcua_binary_variant_data.variant_data_node_id_namespace_idx,zeek.opcua_binary_variant_data.variant_data_node_id_namespace_uri,zeek.opcua_binary_variant_data.variant_data_node_id_numeric,zeek.opcua_binary_variant_data.variant_data_node_id_opaque,zeek.opcua_binary_variant_data.variant_data_node_id_server_idx,zeek.opcua_binary_variant_data.variant_data_node_id_string,zeek.opcua_binary_variant_data.variant_data_text,zeek.opcua_binary_variant_data.variant_data_value_decimal,zeek.opcua_binary_variant_data.variant_data_value_signed_numeric,zeek.opcua_binary_variant_data.variant_data_value_string,zeek.opcua_binary_variant_data.variant_data_value_time,zeek.opcua_binary_variant_data.variant_data_value_unsigned_numeric,zeek.opcua_binary_variant_data_value.data_value_encoding_mask,zeek.opcua_binary_variant_data_value.server_pico_sec,zeek.opcua_binary_variant_data_value.server_timestamp,zeek.opcua_binary_variant_data_value.source_pico_sec,zeek.opcua_binary_variant_data_value.source_timestamp,zeek.opcua_binary_variant_data_value.variant_data_value_source_link,zeek.opcua_binary_variant_extension_object.ext_obj_encoding,zeek.opcua_binary_variant_extension_object.ext_obj_link_id,zeek.opcua_binary_variant_extension_object.ext_obj_node_id_encoding_mask,zeek.opcua_binary_variant_extension_object.ext_obj_node_id_guid,zeek.opcua_binary_variant_extension_object.ext_obj_node_id_namespace_idx,zeek.opcua_binary_variant_extension_object.ext_obj_node_id_numeric,zeek.opcua_binary_variant_extension_object.ext_obj_node_id_opaque,zeek.opcua_binary_variant_extension_object.ext_obj_node_id_string,zeek.opcua_binary_variant_extension_object.ext_obj_type_id_str,zeek.opcua_binary_variant_metadata.built_in_data_type,zeek.opcua_binary_variant_metadata.built_in_data_type_str,zeek.opcua_binary_variant_metadata.dara_variant_encoding_mask,zeek.opcua_binary_variant_metadata.data_variant_data_type,zeek.opcua_binary_variant_metadata.data_variant_data_type_str,zeek.opcua_binary_variant_metadata.variant_data_array_dim,zeek.opcua_binary_variant_metadata.variant_data_source,zeek.opcua_binary_variant_metadata.variant_data_source_str,zeek_opcua_binary_write=require:zeek.opcua_binary_write;title:Zeek opcua_binary_write.log;fields:zeek.opcua_binary_write.source_h,zeek.opcua_binary_write.source_p,zeek.opcua_binary_write.destination_h,zeek.opcua_binary_write.destination_p,zeek.opcua_binary_write.node_id_encoding_mask,zeek.opcua_binary_write.node_id_namespace_idx,zeek.opcua_binary_write.node_id_numeric,zeek.opcua_binary_write.node_id_string,zeek.opcua_binary_write.node_id_guid,zeek.opcua_binary_write.node_id_opaque,zeek.opcua_binary_write.attribute_id,zeek.opcua_binary_write.attribute_id_str,zeek.opcua_binary_write.index_range,zeek.opcua_binary_write.data_value_encoding_mask,zeek.opcua_binary_write.source_timestamp,zeek.opcua_binary_write.source_pico_sec,zeek.opcua_binary_write.server_timestamp,zeek.opcua_binary_write.server_pico_sec
 o_zeek_ospf=require:zeek.ospf;title:Zeek ospf.log;fields:zeek.ospf.ospf_type,zeek.ospf.version,zeek.ospf.router_id,zeek.ospf.area_id,zeek.ospf.interface_id,zeek.ospf.netmask,zeek.ospf.desig_router,zeek.ospf.backup_router,zeek.ospf.neighbors,zeek.ospf.lsa_type,zeek.ospf.link_state_id,zeek.ospf.advert_router,zeek.ospf.routers,zeek.ospf.link_id,zeek.ospf.link_data,zeek.ospf.link_type,zeek.ospf.neighbor_router_id,zeek.ospf.metrics,zeek.ospf.fwd_addrs,zeek.ospf.route_tags,zeek.ospf.neighbor_interface_id,zeek.ospf.prefix,zeek.ospf.metric,zeek.ospf.dest_router_id,zeek.ospf.link_prefixes,zeek.ospf.intra_prefixes
 o_zeek_pe=require:zeek.pe;title:Zeek pe.log;fields:zeek.pe.machine,zeek.pe.compile_ts,zeek.pe.os,zeek.pe.subsystem,zeek.pe.is_exe,zeek.pe.is_64bit,zeek.pe.uses_aslr,zeek.pe.uses_dep,zeek.pe.uses_code_integrity,zeek.pe.uses_seh,zeek.pe.has_import_table,zeek.pe.has_export_table,zeek.pe.has_cert_table,zeek.pe.has_debug_data,zeek.pe.section_names
 o_zeek_profinet=require:zeek.profinet;title:Zeek profinet.log;fields:zeek.profinet.operation_type,zeek.profinet.block_version,zeek.profinet.slot_number,zeek.profinet.subslot_number,zeek.profinet.index
diff --git a/arkime/wise/source.zeeklogs.js b/arkime/wise/source.zeeklogs.js
index 134af896a..2bc468f3f 100644
--- a/arkime/wise/source.zeeklogs.js
+++ b/arkime/wise/source.zeeklogs.js
@@ -2295,6 +2295,20 @@ class MalcolmSource extends WISESource {
       "zeek.opcua_binary_variant_metadata.variant_data_array_dim",
       "zeek.opcua_binary_variant_metadata.variant_data_source",
       "zeek.opcua_binary_variant_metadata.variant_data_source_str",
+      "zeek.opcua_binary_write.node_id_encoding_mask",
+      "zeek.opcua_binary_write.node_id_namespace_idx",
+      "zeek.opcua_binary_write.node_id_numeric",
+      "zeek.opcua_binary_write.node_id_string",
+      "zeek.opcua_binary_write.node_id_guid",
+      "zeek.opcua_binary_write.node_id_opaque",
+      "zeek.opcua_binary_write.attribute_id",
+      "zeek.opcua_binary_write.attribute_id_str",
+      "zeek.opcua_binary_write.index_range",
+      "zeek.opcua_binary_write.data_value_encoding_mask",
+      "zeek.opcua_binary_write.source_timestamp",
+      "zeek.opcua_binary_write.source_pico_sec",
+      "zeek.opcua_binary_write.server_timestamp",
+      "zeek.opcua_binary_write.server_pico_sec",
       "zeek.ospf.advert_router",
       "zeek.ospf.area_id",
       "zeek.ospf.backup_router",
diff --git a/dashboards/templates/composable/component/zeek_ot.json b/dashboards/templates/composable/component/zeek_ot.json
index 93877c03b..51a6bfea7 100644
--- a/dashboards/templates/composable/component/zeek_ot.json
+++ b/dashboards/templates/composable/component/zeek_ot.json
@@ -863,6 +863,7 @@
         "zeek.opcua_binary.identifier": { "type": "long" },
         "zeek.opcua_binary.identifier_str": { "type": "keyword" },
         "zeek.opcua_binary.is_final": { "type": "keyword" },
+        "zeek.opcua_binary.log_types": { "type": "integer" },
         "zeek.opcua_binary.max_chunk_cnt": { "type": "long" },
         "zeek.opcua_binary.max_msg_size": { "type": "long" },
         "zeek.opcua_binary.msg_size": { "type": "long" },
@@ -1316,6 +1317,20 @@
         "zeek.opcua_binary_variant_metadata.variant_data_array_dim": { "type": "long" },
         "zeek.opcua_binary_variant_metadata.variant_data_source": { "type": "long" },
         "zeek.opcua_binary_variant_metadata.variant_data_source_str": { "type": "keyword" },
+        "zeek.opcua_binary_write.node_id_encoding_mask": { "type": "keyword" },
+        "zeek.opcua_binary_write.node_id_namespace_idx": { "type": "long" },
+        "zeek.opcua_binary_write.node_id_numeric": { "type": "long" },
+        "zeek.opcua_binary_write.node_id_string": { "type": "keyword" },
+        "zeek.opcua_binary_write.node_id_guid": { "type": "keyword" },
+        "zeek.opcua_binary_write.node_id_opaque": { "type": "keyword" },
+        "zeek.opcua_binary_write.attribute_id": { "type": "long" },
+        "zeek.opcua_binary_write.attribute_id_str": { "type": "keyword" },
+        "zeek.opcua_binary_write.index_range": { "type": "keyword" },
+        "zeek.opcua_binary_write.data_value_encoding_mask": { "type": "keyword" },
+        "zeek.opcua_binary_write.source_timestamp": { "type": "date" },
+        "zeek.opcua_binary_write.source_pico_sec": { "type": "long" },
+        "zeek.opcua_binary_write.server_timestamp": { "type": "date" },
+        "zeek.opcua_binary_write.server_pico_sec": { "type": "long" },
         "zeek.profinet.block_version": { "type": "keyword" },
         "zeek.profinet.index": { "type": "keyword" },
         "zeek.profinet.operation_type": { "type": "keyword" },
diff --git a/logstash/pipelines/zeek/1043_zeek_opcua_binary.conf b/logstash/pipelines/zeek/1043_zeek_opcua_binary.conf
index b9431e3d2..59e70689f 100644
--- a/logstash/pipelines/zeek/1043_zeek_opcua_binary.conf
+++ b/logstash/pipelines/zeek/1043_zeek_opcua_binary.conf
@@ -1569,6 +1569,41 @@ filter {
         add_tag => [ "ics" ]
       }
 
+    } else if ([log_source] == "opcua_binary_write") {
+      #############################################################################################################################
+      # opcua_binary_write.log
+      # write-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary)
+
+      if ("_jsonparsesuccess" not in [tags]) {
+        dissect {
+          id => "dissect_zeek_opcua_binary_write"
+          mapping => {
+            "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][node_id_encoding_mask]} %{[zeek_cols][node_id_namespace_idx]} %{[zeek_cols][node_id_numeric]} %{[zeek_cols][node_id_string]}  %{[zeek_cols][node_id_guid]}  %{[zeek_cols][node_id_opaque]}  %{[zeek_cols][attribute_id]}  %{[zeek_cols][attribute_id_str]}  %{[zeek_cols][index_range]} %{[zeek_cols][data_value_encoding_mask]}  %{[zeek_cols][req_status_code_link_id]} %{[zeek_cols][source_timestamp]}  %{[zeek_cols][source_pico_sec]} %{[zeek_cols][server_timestamp]}  %{[zeek_cols][server_pico_sec]} %{[zeek_cols][write_results_variant_metadata_link_id]}  %{[zeek_cols][res_status_code_link_id]} %{[zeek_cols][diag_info_link_id]}"
+          }
+        }
+
+        if ("_dissectfailure" in [tags]) {
+          mutate {
+            id => "mutate_split_zeek_opcua_binary_write"
+            split => { "[message]" => " " }
+          }
+          ruby {
+            id => "ruby_zip_zeek_opcua_binary_write"
+            init => "$zeek_opcua_binary_write_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'node_id_encoding_mask', 'node_id_namespace_idx', 'node_id_numeric', 'node_id_string', 'node_id_guid', 'node_id_opaque', 'attribute_id', 'attribute_id_str', 'index_range', 'data_value_encoding_mask', 'req_status_code_link_id', 'source_timestamp', 'source_pico_sec', 'server_timestamp', 'server_pico_sec', 'write_results_variant_metadata_link_id', 'res_status_code_link_id', 'diag_info_link_id' ]"
+            code => "event.set('[zeek_cols]', $zeek_opcua_binary_write_field_names.zip(event.get('[message]')).to_h)"
+          }
+        }
+      }
+
+      mutate {
+        id => "mutate_add_fields_zeek_opcua_binary_write"
+        add_field => {
+          "[zeek_cols][proto]" => "tcp"
+          "[zeek_cols][service]" => "opcua-binary"
+        }
+        add_tag => [ "ics" ]
+      }
+
     } else {
       # some other unknown zeek opcua- log file. should start with ts at least!
 
diff --git a/logstash/pipelines/zeek/1200_zeek_mutate.conf b/logstash/pipelines/zeek/1200_zeek_mutate.conf
index 1cc6d83b3..e3f7ae622 100644
--- a/logstash/pipelines/zeek/1200_zeek_mutate.conf
+++ b/logstash/pipelines/zeek/1200_zeek_mutate.conf
@@ -1602,6 +1602,19 @@ filter {
       rename => { "[zeek][opcua_binary_event_filter_attribute_operand][browse_path_element_link_id]" => "[zeek][opcua_binary_event_filter_attribute_operand_browse_paths][browse_path_element_link_id]" }
       rename => { "[zeek][opcua_binary_event_filter_element_operand][content_filter_filter_operand_link_id]" => "[zeek][opcua_binary][operand_source_link_id]" }
       rename => { "[zeek][opcua_binary_event_filter_literal_operand][content_filter_filter_operand_link_id]" => "[zeek][opcua_binary][operand_source_link_id]" }
+      rename => { "[zeek][opcua_binary_write][opcua_link_id]" => "[zeek][opcua_binary][opcua_link_id]" }
+      rename => { "[zeek][opcua_binary_write][diag_info_link_id]" => "[zeek][opcua_binary_diag_info_detail][diag_info_link_id]" }
+      rename => { "[zeek][opcua_binary_write][write_results_variant_metadata_link_id]" => "[zeek][opcua_binary][variant_source_link_id]" }
+    }
+    if ([zeek][opcua_binary_write][req_status_code_link_id]) {
+      mutate { id => "mutate_merge_zeek_opcua_binary_write_req_status_code_link_id"
+               merge => { "[zeek][opcua_binary_status_code_detail][status_code_link_id]" => "[zeek][opcua_binary_write][req_status_code_link_id]" }
+               remove_field => [ "[zeek][opcua_binary_write][req_status_code_link_id]" ] }
+    }
+    if ([zeek][opcua_binary_write][res_status_code_link_id]) {
+      mutate { id => "mutate_merge_zeek_opcua_binary_write_res_status_code_link_id"
+               merge => { "[zeek][opcua_binary_status_code_detail][status_code_link_id]" => "[zeek][opcua_binary_write][res_status_code_link_id]" }
+               remove_field => [ "[zeek][opcua_binary_write][res_status_code_link_id]" ] }
     }
 
     # count the number of contributing "log types" after we've renamed stuff
diff --git a/logstash/pipelines/zeek/1300_zeek_normalize.conf b/logstash/pipelines/zeek/1300_zeek_normalize.conf
index 5920ef8c4..8deb42be8 100644
--- a/logstash/pipelines/zeek/1300_zeek_normalize.conf
+++ b/logstash/pipelines/zeek/1300_zeek_normalize.conf
@@ -580,6 +580,9 @@ filter {
       } else if ([event][dataset] =~ /^opcua_binary_read/) {
         mutate { id => "mutate_add_field_metadata_opcua_read"
                  add_field => { "[@metadata][opcua_action_from_dataset]" => "Read" } }
+      } else if ([event][dataset] =~ /^opcua_binary_write/) {
+        mutate { id => "mutate_add_field_metadata_opcua_write"
+                 add_field => { "[@metadata][opcua_action_from_dataset]" => "Write" } }
       }
       if ([@metadata][opcua_action_from_dataset]) {
         mutate { id => "mutate_merge_zeek_opcua_action_from_dataset"
diff --git a/logstash/pipelines/zeek/1400_zeek_convert.conf b/logstash/pipelines/zeek/1400_zeek_convert.conf
index 4960e8022..28d2c6cb4 100644
--- a/logstash/pipelines/zeek/1400_zeek_convert.conf
+++ b/logstash/pipelines/zeek/1400_zeek_convert.conf
@@ -281,6 +281,31 @@ filter {
     }
   }
 
+  if ([zeek][opcua_binary_write][server_timestamp]) {
+    if ([zeek][opcua_binary_write][server_timestamp] == "0.000000") {
+      mutate { id => "mutate_remove_zeek_opcua_binary_write_server_timestamp"
+               remove_field => [ "[zeek][opcua_binary_write][server_timestamp]" ] }
+    } else {
+      date {
+        id => "date_zeek_zeek_opcua_binary_write_server_timestamp"
+        match => [ "[zeek][opcua_binary_write][server_timestamp]", "UNIX" ]
+        target => "[zeek][opcua_binary_write][server_timestamp]"
+      }
+    }
+  }
+
+  if ([zeek][opcua_binary_write][source_timestamp]) {
+    if ([zeek][opcua_binary_write][source_timestamp] == "0.000000") {
+      mutate { id => "mutate_remove_zeek_opcua_binary_write_source_timestamp"
+               remove_field => [ "[zeek][opcua_binary_write][source_timestamp]" ] }
+    } else {
+      date {
+        id => "date_zeek_zeek_opcua_binary_write_source_timestamp"
+        match => [ "[zeek][opcua_binary_write][source_timestamp]", "UNIX" ]
+        target => "[zeek][opcua_binary_write][source_timestamp]"
+      }
+    }
+  }
 
   if ([zeek][pe][compile_ts]) {
     if ([zeek][pe][compile_ts] == "0.000000") {

From 0aecb4503377440222a0b9e3037faa42336644c3 Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Thu, 14 Nov 2024 09:02:45 -0700
Subject: [PATCH 42/42] for cisagov/Malcolm#401, create an API for exporting
 dashboards

---
 api/project/__init__.py         | 90 +++++++++++++++++++++++++++++++++
 docs/api-dashboard-export.md    | 35 +++++++++++++
 docs/api.md                     |  1 +
 docs/contributing-dashboards.md | 82 ++++++++++++++++++++----------
 4 files changed, 180 insertions(+), 28 deletions(-)
 create mode 100644 docs/api-dashboard-export.md

diff --git a/api/project/__init__.py b/api/project/__init__.py
index 46a7c0111..3e1d45f65 100644
--- a/api/project/__init__.py
+++ b/api/project/__init__.py
@@ -1062,6 +1062,96 @@ def ready():
     )
 
 
+@app.route(
+    f"{('/' + app.config['MALCOLM_API_PREFIX']) if app.config['MALCOLM_API_PREFIX'] else ''}/dashboard-export/<dashid>",
+    methods=['GET', 'POST'],
+)
+def dashboard_export(dashid):
+    """Uses the opensearch dashboards API to export a dashboard. Also handles the _REPLACER strings
+    as described in "Adding new visualizations and dashboards" at
+    https://idaholab.github.io/Malcolm/docs/contributing-dashboards.html#DashboardsNewViz
+
+    Parameters
+    ----------
+    dashid : string
+        the ID of the dashboard to export
+    request : Request
+        Uses 'replace' from requests arguments, true (default) or false; indicates whether or not to do
+        MALCOLM_NETWORK_INDEX_PATTERN_REPLACER, MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER,
+        MALCOLM_OTHER_INDEX_PATTERN_REPLACER
+
+    Returns
+    -------
+    content
+        The JSON of the exported dashboard
+    """
+
+    args = get_request_arguments(request)
+    try:
+        # call the API to get the dashboard JSON
+        response = requests.get(
+            f"{dashboardsUrl}/api/{'kibana' if (databaseMode == malcolm_utils.DatabaseMode.ElasticsearchRemote) else 'opensearch-dashboards'}/dashboards/export",
+            params={
+                'dashboard': dashid,
+            },
+            auth=opensearchReqHttpAuth,
+            verify=opensearchSslVerify,
+        )
+        response.raise_for_status()
+
+        if doReplacers := malcolm_utils.str2bool(args.get('replace', 'true')):
+            # replace references to index pattern names with the _REPLACER strings, which will allow other Malcolm
+            #   instances that use different index pattern names to import them and substitute their own names
+            replacements = {
+                app.config['MALCOLM_NETWORK_INDEX_PATTERN']: 'MALCOLM_NETWORK_INDEX_PATTERN_REPLACER',
+                app.config['MALCOLM_NETWORK_INDEX_TIME_FIELD']: 'MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER',
+                app.config['MALCOLM_OTHER_INDEX_PATTERN']: 'MALCOLM_OTHER_INDEX_PATTERN_REPLACER',
+            }
+            pattern = re.compile('|'.join(re.escape(key) for key in replacements))
+            responseText = pattern.sub(lambda match: replacements[match.group(0)], response.text)
+        else:
+            # ... or just return it as-is
+            responseText = response.text
+
+        # remove index pattern definition from exported dashboard as they get created programatically
+        #   on Malcolm startup and we don't want them to come in with imported dashboards
+        if responseParsed := malcolm_utils.LoadStrIfJson(responseText):
+            if 'objects' in responseParsed and isinstance(responseParsed['objects'], list):
+                responseParsed['objects'] = [
+                    o
+                    for o in responseParsed['objects']
+                    if not (
+                        (o.get("type") == "index-pattern")
+                        and (
+                            o.get("id")
+                            in [
+                                (
+                                    "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER"
+                                    if doReplacers
+                                    else app.config['MALCOLM_NETWORK_INDEX_PATTERN']
+                                ),
+                                (
+                                    "MALCOLM_OTHER_INDEX_PATTERN_REPLACER"
+                                    if doReplacers
+                                    else app.config['MALCOLM_OTHER_INDEX_PATTERN']
+                                ),
+                            ]
+                        )
+                    )
+                ]
+            return jsonify(responseParsed)
+
+        else:
+            # what we got back from the API wasn't valid JSON, so sad
+            return jsonify(error=f'Could not process export response for {dashid}')
+
+    except Exception as e:
+        errStr = f"{type(e).__name__}: {str(e)} exporting OpenSearch Dashboard {dashid}"
+        if debugApi:
+            print(errStr)
+        return jsonify(error=errStr)
+
+
 @app.route(
     f"{('/' + app.config['MALCOLM_API_PREFIX']) if app.config['MALCOLM_API_PREFIX'] else ''}/ingest-stats",
     methods=['GET'],
diff --git a/docs/api-dashboard-export.md b/docs/api-dashboard-export.md
new file mode 100644
index 000000000..6a4450c25
--- /dev/null
+++ b/docs/api-dashboard-export.md
@@ -0,0 +1,35 @@
+# Dashboard Export
+
+`GET` or `POST` - /mapi/dashboard-export/`<dashid>`
+
+Uses the [OpenSearch Dashboards](https://opensearch.org/docs/latest/dashboards/) or [Elastic Kibana](https://www.elastic.co/guide/en/kibana/current/dashboard-api-export.html) API to export the JSON document representing a dashboard (identified by `dashid`). If the query parameter `replace` is not set to `false`, this API will also perform some modifications on the dashboard as described in the [**Adding new visualizations and dashboards**](contributing-dashboards.md#DashboardsNewViz) section of the [contributor guide](contributing-guide.md#Contributing).
+
+Parameters:
+
+* `dashid` (URL parameter) - the [ID of the dashboard]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/dashboards/dashboards/) to be exported (e.g., `0ad3d7c2-3441-485e-9dfe-dbb22e84e576`)
+* `replace` (query parameter) - whether or not to perform the index pattern name replacements as described above (default: `true`)
+
+Example (response truncated for brevity's sake):
+
+```
+/mapi/dashboard-export/0ad3d7c2-3441-485e-9dfe-dbb22e84e576
+```
+
+```json
+{
+
+  "version": "1.3.1",
+  "objects": [
+    {
+      "id": "0ad3d7c2-3441-485e-9dfe-dbb22e84e576",
+      "type": "dashboard",
+      "namespaces": [
+        "default"
+      ],
+      "updated_at": "2024-04-29T15:49:16.000Z",
+      "version": "WzEzNjIsMV0=",
+      "attributes": {
+        "title": "Overview"
+…
+}
+```
\ No newline at end of file
diff --git a/docs/api.md b/docs/api.md
index b467631d4..cf6bb475a 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -1,5 +1,6 @@
 # <a name="API"></a>API
 
+* [Dashboard Export](api-dashboard-export.md)
 * [Document Ingest Statistics](api-ingest-stats.md)
 * [Document Lookup](api-document-lookup.md)
 * [Event Logging](api-event-logging.md)
diff --git a/docs/contributing-dashboards.md b/docs/contributing-dashboards.md
index 313e89fee..f37cb0499 100644
--- a/docs/contributing-dashboards.md
+++ b/docs/contributing-dashboards.md
@@ -4,36 +4,62 @@
 
 ## <a name="DashboardsNewViz"></a>Adding new visualizations and dashboards
 
-Visualizations and dashboards can be [easily created](dashboards.md#BuildDashboard) in OpenSearch Dashboards using its drag-and-drop WYSIWIG tools. Assuming users have created a new dashboard to package with Malcolm, the dashboard and its visualization components can be exported using the following steps:
+Visualizations and dashboards can be [easily created](dashboards.md#BuildDashboard) in OpenSearch Dashboards using its drag-and-drop WYSIWIG tools. Assuming users have created a new dashboard to package with Malcolm, the dashboard and its visualization components can be exported either of two ways.
+
+The easier (and preferred) method is to use the [dashboard export API](api-dashboard-export.md), as it handles the replacers (described below in the more complicated method) automatically.:
 
 1. Identify the ID of the dashboard (found in the URL: e.g., for `/dashboards/app/dashboards#/view/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` the ID would be `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`)
-1. Export the dashboard with that ID and save it in the `./dashboards./dashboards/` directory with the following command:
-   ```
-    export DASHID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx && \
-      docker compose exec dashboards curl -XGET \
-      "http://localhost:5601/dashboards/api/opensearch-dashboards/dashboards/export?dashboard=$DASHID" > \
-      ./dashboards/dashboards/$DASHID.json
-    ```
-1. It is preferrable for Malcolm to dynamically create the `arkime_sessions3-*` index template rather than including it in imported dashboards, so edit the `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.json` that was generated, carefully locating and removing the section with the `id` of `arkime_sessions3-*` and the `type` of `index-pattern` (including the comma preceding it):
-    ```
-        ,
-        {
-          "id": "arkime_sessions3-*",
-          "type": "index-pattern",
-          "namespaces": [
-            "default"
-          ],
-          "updated_at": "2021-12-13T18:21:42.973Z",
-          "version": "Wzk3MSwxXQ==",
-          …
-          "references": [],
-          "migrationVersion": {
-            "index-pattern": "7.6.0"
-          }
-        }
-    ```
-1. In your text editor, perform a global-search and replace, replacing the string `arkime_sessions3-*` with `MALCOLM_NETWORK_INDEX_PATTERN_REPLACER` and `malcolm_beats_*` with `MALCOLM_OTHER_INDEX_PATTERN_REPLACER`. These replacers are used to [allow customizing indexes for logs written to OpenSearch or Elasticsearch](https://github.com/idaholab/Malcolm/issues/313).
-1. Include the new dashboard either by using a [bind mount](contributing-local-modifications.md#Bind) for the `./dashboards/dashboards/` directory or by [rebuilding](development.md#Build) the `dashboards-helper` image. Dashboards are imported the first time Malcolm starts up.
+
+2. Using a web browser, enter the URL **https://localhost/mapi/dashboard-export/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx**, replacing `localhost` with the IP address or hostname of your Malcolm instance and the placeholder dashboard ID with the ID you identified in the previous step. Save the raw JSON document returned as `./dashboards/dashboards/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.json` (using the actual ID) under your Malcolm directory.
+
+**OR**
+
+2. Using the command line, export the dashboard with that ID and save it in the `./dashboards/dashboards/` directory with the following command:
+
+```
+export DASHID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx && \
+  docker compose exec api curl -sSL -XGET "http://localhost:5000/mapi/dashboard-export/$DASHID" > \
+  ./dashboards/dashboards/$DASHID.json
+```
+
+3. Include the new dashboard either by using a [bind mount](contributing-local-modifications.md#Bind) for the `./dashboards/dashboards/` directory or by [rebuilding](development.md#Build) the `dashboards-helper` image. Dashboards are imported the first time Malcolm starts up.
+
+
+The manual, more complicated way, consists of the following steps:
+
+1. Identify the ID of the dashboard (found in the URL: e.g., for `/dashboards/app/dashboards#/view/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` the ID would be `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`)
+
+2. Using the command line, export the dashboard with that ID and save it in the `./dashboards/dashboards/` directory with the following command:
+
+```
+export DASHID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx && \
+  docker compose exec dashboards curl -XGET \
+  "http://localhost:5601/dashboards/api/opensearch-dashboards/dashboards/export?dashboard=$DASHID" > \
+  ./dashboards/dashboards/$DASHID.json
+```
+
+3. It is preferrable for Malcolm to dynamically create the `arkime_sessions3-*` index template rather than including it in imported dashboards, so edit the `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.json` that was generated, carefully locating and removing the section with the `id` of `arkime_sessions3-*` and the `type` of `index-pattern` (including the comma preceding it):
+
+```
+    ,
+    {
+      "id": "arkime_sessions3-*",
+      "type": "index-pattern",
+      "namespaces": [
+        "default"
+      ],
+      "updated_at": "2021-12-13T18:21:42.973Z",
+      "version": "Wzk3MSwxXQ==",
+      …
+      "references": [],
+      "migrationVersion": {
+        "index-pattern": "7.6.0"
+      }
+    }
+```
+
+4. In your text editor, perform a global-search and replace, replacing the string `arkime_sessions3-*` with `MALCOLM_NETWORK_INDEX_PATTERN_REPLACER` and `malcolm_beats_*` with `MALCOLM_OTHER_INDEX_PATTERN_REPLACER`. These replacers are used to [allow customizing indexes for logs written to OpenSearch or Elasticsearch](https://github.com/idaholab/Malcolm/issues/313).
+5. Include the new dashboard either by using a [bind mount](contributing-local-modifications.md#Bind) for the `./dashboards/dashboards/` directory or by [rebuilding](development.md#Build) the `dashboards-helper` image. Dashboards are imported the first time Malcolm starts up.
 
 ## <a name="DashboardsPlugins"></a>OpenSearch Dashboards plugins