Pass fail analysis

application-test/kestrel/test/features/steps/kestrel_data_steps.py

@@ -81,7 +81,7 @@ def step_impl(context):
     print(f'len(exploit_candidates) = {len(exploit_candidates)}')
     assert exploit_candidates and len(exploit_candidates) in [718, 964]
     exploit_summary = context.summary_hunt_ttps.get('exploit_candidates', {})
-    assert exploit_summary and exploit_summary.get('#(RECORDS)') in [1118, 964]
+    assert exploit_summary and exploit_summary.get('#(RECORDS)') == 718
 
 
 @when(u'I start a cross-host campaign discovery with Kestrel')
@@ -149,9 +149,9 @@ def step_impl(context):
 @then(u'identify attacker activities on windows hosts')
 def step_impl(context):
     lateral_mov_112_proc = context.session.get_variable('lateral_mov_112_proc')
-    assert lateral_mov_112_proc and len(lateral_mov_112_proc) == 2
+    assert lateral_mov_112_proc and len(lateral_mov_112_proc) == 1
     lm_112_mov_summary = context.campaign_summary.get('lateral_mov_112_proc', {})
-    assert lm_112_mov_summary and lm_112_mov_summary.get('#(RECORDS)') == 20
+    assert lm_112_mov_summary and lm_112_mov_summary.get('#(RECORDS)') == 14
     lateral_mov_112_receiver = context.session.get_variable('lateral_mov_112_receiver')
     assert lateral_mov_112_receiver and len(lateral_mov_112_receiver) == 1
     splunkd_on_112 = context.session.get_variable('splunkd_on_112')
@@ -170,17 +170,17 @@ def step_impl(context):
     print(f'len(linux_proc) = {len(linux_proc)}')
     assert linux_proc and len(linux_proc) in [34, 35]
     linux_proc_summary = context.campaign_summary.get('linux_proc', {})
-    assert linux_proc_summary and linux_proc_summary.get('#(RECORDS)') == 35
+    assert linux_proc_summary and linux_proc_summary.get('#(RECORDS)') == 34
     node_children = context.session.get_variable('node_children')
     node_grand_children = context.session.get_variable('node_grand_children')
     print(f'len(node_children) = {len(node_children)}')
     print(f'len(node_grand_children) = {len(node_grand_children)}')
     assert node_children and len(node_children) in [303, 335]
     assert node_grand_children and len(node_grand_children) in [363, 534]
     node_children_summary = context.campaign_summary.get('node_children', {})
-    assert node_children_summary and node_children_summary.get('#(RECORDS)') == 335
+    assert node_children_summary and node_children_summary.get('#(RECORDS)') == 303
     node_grand_children_summary = context.campaign_summary.get('node_grand_children', {})
-    assert node_grand_children_summary and node_grand_children_summary.get('#(RECORDS)') == 534
+    assert node_grand_children_summary and node_grand_children_summary.get('#(RECORDS)') == 363
 
 
 @then(u'discover the C2 host / IP address if any')

application-test/kestrel/test/run-tests.sh

@@ -8,5 +8,6 @@ if [ ! -f "${HOME}"/fedsearchtest/kestrel-stixshifter-config.yaml ]; then
 fi
 export KESTREL_STIXSHIFTER_CONFIG="${HOME}"/fedsearchtest/kestrel-stixshifter-config.yaml
 cd application-test/kestrel/test || exit
-behave --logging-level CRITICAL
+behave --logging-level CRITICAL --format json --outfile /tmp/kestrel-behave.json --format pretty
 cd "${CRT_DIR}" || exit
+python3 federated-search-core/test/analyze-results.py /tmp/kestrel-behave.json

federated-search-core/test/analyze-results.py

@@ -0,0 +1,58 @@
+import argparse
+import json
+import logging
+import os
+import sys
+import traceback as tb
+
+logging.basicConfig(
+    stream=sys.stdout,
+    level=logging.INFO,
+    format="%(asctime)s %(levelname)s [%(filename)s:" "%(lineno)d]: %(message)s",
+)
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "behave_json_file", help="behave output (in JSON format) input to analysis"
+)
+parser.add_argument(
+    "-a",
+    "--analysis",
+    type=str,
+    default="PassFail",
+    help="analysis type PassFail: pass if all features passed, if not, fail",
+)
+
+args = parser.parse_args()
+
+behave_json_file = args.behave_json_file
+analysis_type = args.analysis
+logging.info(
+    f"Running with the following args: "
+    f"behave results file = {behave_json_file}, "
+    f"analysis type = {analysis_type}"
+)
+test_results = None
+try:
+    with open(behave_json_file, "r") as fp:
+        test_results = json.load(fp)
+except:
+    logging.error(
+        f"Failed to load test results from file "
+        f"{behave_json_file}: {tb.print_exc()}"
+    )
+    sys.exit(-1)
+if not test_results:
+    logging.error(f"No test results found in {behave_json_file}")
+    sys.exit(-1)
+if analysis_type == "PassFail":
+    pass_fail_result = True
+    for feature_result in test_results:
+        feature_status = feature_result.get("status", "NotFound")
+        if feature_status != "passed":
+            logging.info("Testing has FAILED")
+            sys.exit(-1)
+else:
+    logging.warn(f"Analysis of type {analysis_type} not supported yet")
+    sys.exit(-1)
+logging.info("Testing PASSED")