Merge pull request #393 from opencybersecurityalliance/develop

v1.7.4

.github/workflows/code-style.yml

@@ -7,12 +7,14 @@ on:
       - develop_*
     paths:
       - 'src/**'
+      - 'bin/**'
   pull_request:
     branches:
       - develop
       - develop_*
     paths:
       - 'src/**'
+      - 'bin/**'
     types:
       - opened
       - reopened

.github/workflows/unit-testing.yml

@@ -7,13 +7,15 @@ on:
       - develop_*
     paths:
       - 'src/**'
+      - 'bin/**'
       - 'tests/**'
   pull_request:
     branches:
       - develop
       - develop_*
     paths:
       - 'src/**'
+      - 'bin/**'
       - 'tests/**'
     types:
       - opened

.github/workflows/unused-import.yml

@@ -7,12 +7,14 @@ on:
       - develop_*
     paths:
       - 'src/**'
+      - 'bin/**'
   pull_request:
     branches:
       - develop
       - develop_*
     paths:
       - 'src/**'
+      - 'bin/**'
     types:
       - opened
       - reopened

CHANGELOG.rst

@@ -9,6 +9,25 @@ The format is based on `Keep a Changelog`_.
 Unreleased
 ==========
 
+1.7.4 (2023-08-03)
+==================
+
+Added
+-----
+
+- New simpler default STIX patterns for stix-shifter-diag
+- Connector verification/install in stix-shifter-diag #388
+- Custom pattern (string and file) support for stix-shifter-diag
+- Debug info output support for stix-shifter-diag
+- Current time as stop time support for default pattern in stix-shifter-diag
+- Query-translate-only mode for stix-shifter-diag
+
+Changed
+-------
+
+- Fix/change the order of LIMIT and timespan in Lark file according to Kestrel doc
+- stix-shifter update to v6 (6.0.3)
+
 1.7.3 (2023-07-26)
 ==================
 

bin/stix-shifter-diag

@@ -1,28 +1,99 @@
 #!/usr/bin/env python3
 
 import argparse
+import datetime
 from kestrel_datasource_stixshifter.diagnosis import Diagnosis
+from kestrel_datasource_stixshifter.connector import check_module_availability
+from firepit.timestamp import timefmt
 
-def gen_patterns():
-    time_range = "START t'2000-01-01T00:00:00.000Z' STOP t'3000-01-01T00:00:00.000Z'"
+
+def default_patterns(use_now_as_stop_time: bool):
+    start_time = "START t'2000-01-01T00:00:00.000Z'"
+    stop_time = (
+        f"STOP t'{timefmt(datetime.datetime.utcnow())}'"
+        if use_now_as_stop_time
+        else "STOP t'3000-01-01T00:00:00.000Z'"
+    )
     patterns = [
-        "[ipv4-addr:value LIKE '%']",
+        "[ipv4-addr:value != '255.255.255.255']",
         "[process:pid > 0]",
-        "[email-addr:value LIKE '%']",
+        "[email-addr:value != 'null@xyz.com']",
     ]
-    return [" ".join([p, time_range]) for p in patterns]
+    return [" ".join([p, start_time, stop_time]) for p in patterns]
+
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Kestrel stix-shifter data source interface diagnosis")
-    parser.add_argument("datasource", help="data source name specified in stixshifter.yaml")
+    parser = argparse.ArgumentParser(
+        description="Kestrel stix-shifter data source interface diagnosis"
+    )
+    parser.add_argument(
+        "datasource", help="data source name specified in stixshifter.yaml"
+    )
+    parser.add_argument(
+        "--ignore-cert",
+        help="ignore certificate (PKI) verification in connector verification",
+        action="store_false",
+    )
+    parser.add_argument(
+        "-p",
+        "--stix-pattern",
+        help="STIX pattern in double quotes",
+    )
+    parser.add_argument(
+        "-f",
+        "--pattern-file",
+        help="write your STIX pattern in a file and put the file path here to use for diagnosis",
+    )
+    parser.add_argument(
+        "--stop-at-now",
+        help="use the current timestamp as the STOP time instead of default year 3000 for default patterns",
+        action="store_true",
+    )
+    parser.add_argument(
+        "-t",
+        "--translate-only",
+        help="Only translate pattern; don't transmit",
+        action="store_true",
+    )
+    parser.add_argument(
+        "-d", "--debug", help="Enable DEBUG logging", action="store_true"
+    )
     args = parser.parse_args()
 
-    patterns = gen_patterns()
+    if args.debug:
+        logger = logging.getLogger()
+        logger.setLevel(logging.DEBUG)
+        ch = logging.StreamHandler()
+        ch.setLevel(logging.DEBUG)
+        formatter = logging.Formatter("%(levelname)s: %(message)s")
+        ch.setFormatter(formatter)
+        logger.addHandler(ch)
+
+    if args.stix_pattern:
+        patterns = [args.stix_pattern]
+    elif args.pattern_file:
+        with open(args.pattern_file) as pf:
+            patterns = [pf.read()]
+    else:
+        patterns = default_patterns(args.stop_at_now)
 
     diag = Diagnosis(args.datasource)
 
+    # 1. check config manually
     diag.diagnose_config()
-    diag.diagnose_ping()
+
+    # 2. setup connector and ping
+    check_module_availability(diag.connector_name, args.ignore_cert)
+
+    # 3. query translation test
     diag.diagnose_translate_query(patterns[0])
-    diag.diagnose_run_query_and_retrieval_result(patterns, 1)
-    diag.diagnose_run_query_and_retrieval_result(patterns, 5)
+
+    if not args.translate_only:
+        # 4. transmit ping test
+        diag.diagnose_ping()
+
+        # 5. single-batch query execution test
+        diag.diagnose_run_query_and_retrieval_result(patterns, 1)
+
+        # 6. multi-batch query execution test
+        diag.diagnose_run_query_and_retrieval_result(patterns, 5)

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = kestrel-lang
-version = 1.7.3
+version = 1.7.4
 description = Kestrel Threat Hunting Language
 long_description = file:README.rst
 long_description_content_type = text/x-rst
@@ -38,8 +38,8 @@ install_requires =
     docker>=5.0.0
     requests>=2.31.0
     nest-asyncio>=1.5.6
-    stix-shifter==5.3.1
-    stix-shifter-utils==5.3.1
+    stix-shifter==6.0.3
+    stix-shifter-utils==6.0.3
     firepit>=2.3.25
 tests_require =
     pytest

src/kestrel/syntax/kestrel.lark

@@ -36,9 +36,9 @@ assignment: VARIABLE "=" expression
 // All commands
 //
 
-find: "FIND"i ENTITY_TYPE RELATION (REVERSED)? VARIABLE where_clause? limit_clause? timespan?
+find: "FIND"i ENTITY_TYPE RELATION (REVERSED)? VARIABLE where_clause? timespan? limit_clause?
 
-get: "GET"i ENTITY_TYPE ("FROM"i datasource)? where_clause limit_clause? timespan?
+get: "GET"i ENTITY_TYPE ("FROM"i datasource)? where_clause timespan? limit_clause?
 
 group: "GROUP"i VARIABLE BY grp_spec ("WITH"i agg_list)?
 
@@ -147,8 +147,8 @@ grp_spec: grp_expr ("," grp_expr)*
 grp_expr: ATTRIBUTE
         | bin_func
 
-bin_func: "BIN"i "(" ATTRIBUTE "," INT timeunit? ")"
 // No other scalar funcs are supported yet
+bin_func: "BIN"i "(" ATTRIBUTE "," INT timeunit? ")"
 
 agg_list: agg ("," agg)*
 

src/kestrel_datasource_stixshifter/connector.py

@@ -21,12 +21,14 @@ def get_package_name(connector_name):
     return "stix-shifter-modules-" + connector_name.replace("_", "-")
 
 
-def verify_package_origin(connector_name, stixshifter_version):
+def verify_package_origin(connector_name, stixshifter_version, requests_verify=True):
     _logger.debug("go to PyPI to verify package genuineness from STIX-shifter project")
     package_name = get_package_name(connector_name)
 
     try:
-        pypi_response = requests.get(f"https://pypi.org/project/{package_name}")
+        pypi_response = requests.get(
+            f"https://pypi.org/project/{package_name}", verify=requests_verify
+        )
         pypi_etree = html.fromstring(pypi_response.content)
     except:
         raise DataSourceError(
@@ -58,13 +60,13 @@ def verify_package_origin(connector_name, stixshifter_version):
     _logger.info(f'"{package_name}" verified as a STIX-shifter package.')
 
 
-def install_package(connector_name):
+def install_package(connector_name, requests_verify=True):
     package_name = get_package_name(connector_name)
     _logger.debug(f"guess the connector package name: {package_name}")
 
     stixshifter_version = version("stix_shifter")
 
-    verify_package_origin(connector_name, stixshifter_version)
+    verify_package_origin(connector_name, stixshifter_version, requests_verify)
 
     package_w_ver = package_name + "==" + stixshifter_version
 
@@ -87,7 +89,7 @@ def install_package(connector_name):
         )
 
 
-def ensure_version_consistency(connector_name):
+def ensure_version_consistency(connector_name, requests_verify=True):
     """Check if the installed connector package has the same version as
     stix-shifter If the version is different, uninstall connector
     package and the install the same version as stix-shifter
@@ -110,17 +112,17 @@ def ensure_version_consistency(connector_name):
         )
     except:
         _logger.info(f"failed to uninstall package {package_w_ver}")
-    install_package(connector_name)
+    install_package(connector_name, requests_verify)
 
 
-def check_module_availability(connector_name):
+def check_module_availability(connector_name, requests_verify=True):
     try:
         importlib.import_module(
             "stix_shifter_modules." + connector_name + ".entry_point"
         )
 
-        ensure_version_consistency(connector_name)
+        ensure_version_consistency(connector_name, requests_verify)
 
     except:
         _logger.info(f'miss STIX-shifter connector "{connector_name}"')
-        install_package(connector_name)
+        install_package(connector_name, requests_verify)

src/kestrel_datasource_stixshifter/diagnosis.py

@@ -84,20 +84,26 @@ def diagnose_translate_query(self, stix_pattern, quiet=False):
             print()
             print("## Diagnose: stix-shifter query translation")
 
+        if not quiet:
+            print()
+            print("#### Input: STIX pattern")
+            print(stix_pattern)
+
         dsl = translate_query(
             self.connector_name,
             {},
             stix_pattern,
             self.connection_dict,
         )
 
+        if "queries" not in dsl:
+            raise Exception(str(dsl))
+
         if not quiet:
             print()
-            print("#### Input pattern")
-            print(stix_pattern)
-            print()
-            print("#### Output data source native query")
-            print(json.dumps(dsl, indent=4))
+            print(f"#### Output: {len(dsl['queries'])} data source native queries")
+            for query in dsl["queries"]:
+                print(query)
 
         return dsl
 

src/kestrel_datasource_stixshifter/interface.py

@@ -92,12 +92,16 @@
 
 #. any in-session edit through the ``CONFIG`` command.
 
-Once you added data source profiles into ``stixshifter.yaml``, you can test the data source with command:
+After added data source profiles into ``stixshifter.yaml``, you can test the data source:
 
 .. code-block:: console
 
     $ stix-shifter-diag data_source_name
 
+where ``data_source_name`` is any profile named in the ``stixshifter.yaml`` config file, usually used in ``FROM stixshifter://data_source_name`` in the ``GET`` command.
+
+The diagnosis utility will check config, test query translation, try connect to the data source to execute a small and a large query, and retrieve data back. Details of all steps will be printed for diagnosis purpose.
+
 If you launch Kestrel in debug mode, STIX-shifter debug mode is still not
 enabled by default. To record debug level logs of STIX-shifter, create
 environment variable ``KESTREL_STIXSHIFTER_DEBUG`` with any value.

tests/test_command_get.py

@@ -205,8 +205,8 @@ def test_get_single_stixshifter_stix_bundle_limit(set_no_prefetch_kestrel_config
                var = GET process
                      FROM HOST2
                      WHERE [ipv4-addr:value = '127.0.0.1']
-                     LIMIT 4
                      START 2019-01-01T00:00:00Z STOP 2023-01-01T00:00:00Z
+                     LIMIT 4
                """
 
         s.execute(stmt)
@@ -264,8 +264,8 @@ def test_get_multiple_stixshifter_stix_bundles_limit(set_no_prefetch_kestrel_con
                var = GET process
                      FROM HOST1,HOST2
                      WHERE ipv4-addr:value = '127.0.0.1'
-                     LIMIT 10
                      START 2019-01-01T00:00:00Z STOP 2023-01-01T00:00:00Z
+                     LIMIT 10
                """
 
         s.execute(stmt)
@@ -303,8 +303,8 @@ def test_get_multiple_stixshifter_stix_bundles_limit_1(set_no_prefetch_kestrel_c
                var = GET process
                      FROM HOST1,HOST2
                      WHERE ipv4-addr:value = '127.0.0.1'
-                     LIMIT 15
                      START 2019-01-01T00:00:00Z STOP 2023-01-01T00:00:00Z
+                     LIMIT 15
                """
 
         s.execute(stmt)
@@ -342,8 +342,8 @@ def test_get_multiple_stixshifter_stix_bundles_limit_2(set_no_prefetch_kestrel_c
                var = GET process
                      FROM HOST1,HOST2
                      WHERE ipv4-addr:value = '127.0.0.1'
-                     LIMIT 50
                      START 2019-01-01T00:00:00Z STOP 2023-01-01T00:00:00Z
+                     LIMIT 50
                """
 
         s.execute(stmt)