From 61475a6ef109604b47b528e8cd2b224361a626ce Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 6 Feb 2025 11:12:47 -0500 Subject: [PATCH 01/18] temporary change to islate generic ssrf test --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c0443b350..b7a9d2be9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,7 +34,7 @@ jobs: poetry run ruff format --check - name: Run tests run: | - poetry run pytest -vv --exitfirst --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot . + poetry run pytest -vv --exitfirst --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO -k TestGeneric_SSRF --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot . - name: Upload Debug Logs if: always() uses: actions/upload-artifact@v4 From 79ed370461659d0eae674c4a53f3fb3a0b2260fc Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 6 Feb 2025 11:19:18 -0500 Subject: [PATCH 02/18] temporary to troubleshoot generic_ssrf test --- .github/workflows/distro_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index 21f40c813..6040e06e7 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -61,7 +61,7 @@ jobs: export BBOT_DISTRO_TESTS=true poetry env use python3.11 poetry install - poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO . + poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 -k TestGeneric_SSRF --disable-warnings --log-cli-level=INFO . - name: Upload Debug Logs if: always() uses: actions/upload-artifact@v4 From 3710e16461f862ada7786dda64a92a29fd1738ff Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 6 Feb 2025 11:39:40 -0500 Subject: [PATCH 03/18] adding locks to mock interactsh to prevent race conditions --- bbot/test/conftest.py | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index f9807db81..8c7fe280d 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -8,6 +8,7 @@ from contextlib import suppress from omegaconf import OmegaConf from pytest_httpserver import HTTPServer +import time from bbot.core import CORE from bbot.core.helpers.misc import execute_sync_or_async @@ -53,6 +54,12 @@ def silence_live_logging(): handler.setLevel(logging.CRITICAL) +def stop_server(server): + server.stop() + while server.is_running(): + time.sleep(0.1) # Wait a bit before checking again + + @pytest.fixture def bbot_httpserver(): server = HTTPServer(host="127.0.0.1", port=8888, threaded=True) @@ -61,11 +68,7 @@ def bbot_httpserver(): yield server server.clear() - if server.is_running(): - server.stop() - - # this is to check if the client has made any request where no - # `assert_request` was called on it from the test + stop_server(server) # Ensure the server is fully stopped server.check_assertions() server.clear() @@ -84,11 +87,7 @@ def bbot_httpserver_ssl(): yield server server.clear() - if server.is_running(): - server.stop() - - # this is to check if the client has made any request where no - # `assert_request` was called on it from the test + stop_server(server) # Ensure the server is fully stopped server.check_assertions() server.clear() @@ -133,6 +132,7 @@ def __init__(self, name): self.correlation_id = "deadbeef-dead-beef-dead-beefdeadbeef" self.stop = False self.poll_task = None + self.lock = asyncio.Lock() def mock_interaction(self, subdomain_tag, msg=None): self.log.info(f"Mocking interaction to subdomain tag: {subdomain_tag}") @@ -149,7 +149,7 @@ async def deregister(self, callback=None): self.stop = True if self.poll_task is not None: self.poll_task.cancel() - with suppress(BaseException): + with suppress(asyncio.CancelledError): await self.poll_task async def poll_loop(self, callback=None): @@ -161,12 +161,16 @@ async def poll_loop(self, callback=None): async def poll(self, callback=None): poll_results = [] - for subdomain_tag in self.interactions: - result = {"full-id": f"{subdomain_tag}.fakedomain.fakeinteractsh.com", "protocol": "HTTP"} - poll_results.append(result) - if callback is not None: - await execute_sync_or_async(callback, result) - self.interactions = [] + async with self.lock: + try: + for subdomain_tag in self.interactions: + result = {"full-id": f"{subdomain_tag}.fakedomain.fakeinteractsh.com", "protocol": "HTTP"} + poll_results.append(result) + if callback is not None: + await execute_sync_or_async(callback, result) + self.interactions = [] + except Exception as e: + self.log.error(f"Error during poll: {e}") return poll_results From fd3709ada87e2cee50c1ccbc873d80efedd91026 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 6 Feb 2025 12:13:41 -0500 Subject: [PATCH 04/18] reverting temp test changes --- .github/workflows/distro_tests.yml | 2 +- .github/workflows/tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index 6040e06e7..21f40c813 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -61,7 +61,7 @@ jobs: export BBOT_DISTRO_TESTS=true poetry env use python3.11 poetry install - poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 -k TestGeneric_SSRF --disable-warnings --log-cli-level=INFO . + poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO . - name: Upload Debug Logs if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b7a9d2be9..c0443b350 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,7 +34,7 @@ jobs: poetry run ruff format --check - name: Run tests run: | - poetry run pytest -vv --exitfirst --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO -k TestGeneric_SSRF --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot . + poetry run pytest -vv --exitfirst --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot . - name: Upload Debug Logs if: always() uses: actions/upload-artifact@v4 From 62e7d5f53de599e7565a1c37868ae1f7c3d03603 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 6 Feb 2025 14:29:06 -0500 Subject: [PATCH 05/18] changing DNS events to FINDING, adding omit option --- bbot/modules/generic_ssrf.py | 33 +++++++++++++------ bbot/test/conftest.py | 14 ++++---- .../module_tests/test_module_generic_ssrf.py | 29 ++++++++++++++++ 3 files changed, 58 insertions(+), 18 deletions(-) diff --git a/bbot/modules/generic_ssrf.py b/bbot/modules/generic_ssrf.py index fb530a697..6c7c7ed49 100644 --- a/bbot/modules/generic_ssrf.py +++ b/bbot/modules/generic_ssrf.py @@ -154,6 +154,12 @@ class generic_ssrf(BaseModule): produced_events = ["VULNERABILITY"] flags = ["active", "aggressive", "web-thorough"] meta = {"description": "Check for generic SSRFs", "created_date": "2022-07-30", "author": "@liquidsec"} + options = { + "skip_dns_interaction": False, + } + options_desc = { + "skip_dns_interaction": "Do not report DNS interactions (only HTTP interaction)", + } in_scope_only = True deps_apt = ["curl"] @@ -163,7 +169,7 @@ async def setup(self): self.interactsh_subdomain_tags = {} self.parameter_subdomain_tags_map = {} self.severity = None - self.generic_only = self.config.get("generic_only", False) + self.skip_dns_interaction = self.config.get("skip_dns_interaction", False) if self.scan.config.get("interactsh_disable", False) is False: try: @@ -191,6 +197,10 @@ async def handle_event(self, event): await s.test(event) async def interactsh_callback(self, r): + protocol = r.get("protocol").upper() + if protocol == "DNS" and self.skip_dns_interaction: + return + full_id = r.get("full-id", None) subdomain_tag = full_id.split(".")[0] @@ -204,24 +214,27 @@ async def interactsh_callback(self, r): matched_severity = match[2] matched_echoed_response = str(match[3]) - # Check if any SSRF parameter is in the DNS request triggering_param = self.parameter_subdomain_tags_map.get(subdomain_tag, None) description = f"Out-of-band interaction: [{matched_technique}]" if triggering_param: self.debug(f"Found triggering parameter: {triggering_param}") description += f" [Triggering Parameter: {triggering_param}]" - description += f" [{r.get('protocol').upper()}] Echoed Response: {matched_echoed_response}" + description += f" [{protocol}] Echoed Response: {matched_echoed_response}" self.debug(f"Emitting event with description: {description}") # Debug the final description + event_type = "VULNERABILITY" if protocol == "HTTP" else "FINDING" + event_data = { + "host": str(matched_event.host), + "url": matched_event.data, + "description": description, + } + if protocol == "HTTP": + event_data["severity"] = matched_severity + await self.emit_event( - { - "severity": matched_severity, - "host": str(matched_event.host), - "url": matched_event.data, - "description": description, - }, - "VULNERABILITY", + event_data, + event_type, matched_event, context=f"{{module}} scanned {matched_event.data} and detected {{event.type}}: {matched_technique}", ) diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index 8c7fe280d..f6c2ebff9 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -160,18 +160,16 @@ async def poll_loop(self, callback=None): continue async def poll(self, callback=None): - poll_results = [] async with self.lock: - try: - for subdomain_tag in self.interactions: - result = {"full-id": f"{subdomain_tag}.fakedomain.fakeinteractsh.com", "protocol": "HTTP"} + poll_results = [] + for subdomain_tag in self.interactions: + for protocol in ["HTTP", "DNS"]: + result = {"full-id": f"{subdomain_tag}.fakedomain.fakeinteractsh.com", "protocol": protocol} poll_results.append(result) if callback is not None: await execute_sync_or_async(callback, result) - self.interactions = [] - except Exception as e: - self.log.error(f"Error during poll: {e}") - return poll_results + self.interactions = [] + return poll_results import threading diff --git a/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py b/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py index 4502511da..d0cc5255c 100644 --- a/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py +++ b/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py @@ -41,6 +41,18 @@ async def setup_after_prep(self, module_test): module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) def check(self, module_test, events): + total_vulnerabilities = 0 + total_findings = 0 + + for e in events: + if e.type == "VULNERABILITY": + total_vulnerabilities += 1 + elif e.type == "FINDING": + total_findings += 1 + + assert total_vulnerabilities == 30, "Incorrect number of vulnerabilities detected" + assert total_findings == 30, "Incorrect number of findings detected" + assert any( e.type == "VULNERABILITY" and "Out-of-band interaction: [Generic SSRF (GET)]" @@ -55,3 +67,20 @@ def check(self, module_test, events): e.type == "VULNERABILITY" and "Out-of-band interaction: [Generic XXE] [HTTP]" in e.data["description"] for e in events ), "Failed to detect Generic SSRF (XXE)" + + +class TestGeneric_SSRF_httponly(TestGeneric_SSRF): + config_overrides = {"modules": {"generic_ssrf": {"skip_dns_interaction": True}}} + + def check(self, module_test, events): + total_vulnerabilities = 0 + total_findings = 0 + + for e in events: + if e.type == "VULNERABILITY": + total_vulnerabilities += 1 + elif e.type == "FINDING": + total_findings += 1 + + assert total_vulnerabilities == 30, "Incorrect number of vulnerabilities detected" + assert total_findings == 0, "Incorrect number of findings detected" From 5bb903030eee72330dd865b340dd36e17344cba7 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 6 Feb 2025 15:08:49 -0500 Subject: [PATCH 06/18] increasing detection wait time --- bbot/modules/generic_ssrf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/generic_ssrf.py b/bbot/modules/generic_ssrf.py index 6c7c7ed49..6ccde510b 100644 --- a/bbot/modules/generic_ssrf.py +++ b/bbot/modules/generic_ssrf.py @@ -254,7 +254,7 @@ async def cleanup(self): async def finish(self): if self.scan.config.get("interactsh_disable", False) is False: - await self.helpers.sleep(2) + await self.helpers.sleep(5) try: for r in await self.interactsh_instance.poll(): await self.interactsh_callback(r) From feedda27be50a149157c9eba0db298edffdd6407 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 11 Feb 2025 10:58:20 -0500 Subject: [PATCH 07/18] increasing again to stop rare race condition --- bbot/modules/generic_ssrf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/generic_ssrf.py b/bbot/modules/generic_ssrf.py index 6ccde510b..dba56b5c4 100644 --- a/bbot/modules/generic_ssrf.py +++ b/bbot/modules/generic_ssrf.py @@ -254,7 +254,7 @@ async def cleanup(self): async def finish(self): if self.scan.config.get("interactsh_disable", False) is False: - await self.helpers.sleep(5) + await self.helpers.sleep(10) try: for r in await self.interactsh_instance.poll(): await self.interactsh_callback(r) From a3af65852b336206280ed528dda320383f533063 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 11 Feb 2025 13:24:56 -0500 Subject: [PATCH 08/18] make mock_interaction async --- bbot/test/conftest.py | 5 +++-- .../test_step_2/module_tests/test_module_dotnetnuke.py | 3 ++- .../test_step_2/module_tests/test_module_generic_ssrf.py | 7 ++++--- .../test_step_2/module_tests/test_module_host_header.py | 3 ++- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index f6c2ebff9..b8902fbc2 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -134,11 +134,12 @@ def __init__(self, name): self.poll_task = None self.lock = asyncio.Lock() - def mock_interaction(self, subdomain_tag, msg=None): + async def mock_interaction(self, subdomain_tag, msg=None): self.log.info(f"Mocking interaction to subdomain tag: {subdomain_tag}") if msg is not None: self.log.info(msg) - self.interactions.append(subdomain_tag) + async with self.lock: + self.interactions.append(subdomain_tag) async def register(self, callback=None): if callable(callback): diff --git a/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py b/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py index fc666b64e..1bfdfc794 100644 --- a/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py +++ b/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py @@ -1,3 +1,4 @@ +import asyncio import re from .base import ModuleTestBase from werkzeug.wrappers import Response @@ -142,7 +143,7 @@ def request_handler(self, request): subdomain_tag = None subdomain_tag = extract_subdomain_tag(request.full_path) if subdomain_tag: - self.interactsh_mock_instance.mock_interaction(subdomain_tag) + asyncio.run(self.interactsh_mock_instance.mock_interaction(subdomain_tag)) return Response("alive", status=200) async def setup_before_prep(self, module_test): diff --git a/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py b/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py index d0cc5255c..919e4360f 100644 --- a/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py +++ b/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py @@ -1,4 +1,5 @@ import re +import asyncio from werkzeug.wrappers import Response from .base import ModuleTestBase @@ -23,15 +24,15 @@ def request_handler(self, request): elif request.method == "POST": subdomain_tag = extract_subdomain_tag(request.data.decode()) if subdomain_tag: - self.interactsh_mock_instance.mock_interaction( + asyncio.run(self.interactsh_mock_instance.mock_interaction( subdomain_tag, msg=f"{request.method}: {request.data.decode()}" - ) + )) return Response("alive", status=200) async def setup_before_prep(self, module_test): self.interactsh_mock_instance = module_test.mock_interactsh("generic_ssrf") - self.interactsh_mock_instance.mock_interaction("asdf") + asyncio.run(self.interactsh_mock_instance.mock_interaction("asdf")) module_test.monkeypatch.setattr( module_test.scan.helpers, "interactsh", lambda *args, **kwargs: self.interactsh_mock_instance ) diff --git a/bbot/test/test_step_2/module_tests/test_module_host_header.py b/bbot/test/test_step_2/module_tests/test_module_host_header.py index a2d69e9b5..ffb6c05fa 100644 --- a/bbot/test/test_step_2/module_tests/test_module_host_header.py +++ b/bbot/test/test_step_2/module_tests/test_module_host_header.py @@ -1,3 +1,4 @@ +import asyncio import re from werkzeug.wrappers import Response @@ -23,7 +24,7 @@ def request_handler(self, request): # Standard (with reflection) if subdomain_tag: - self.interactsh_mock_instance.mock_interaction(subdomain_tag) + asyncio.run(self.interactsh_mock_instance.mock_interaction(subdomain_tag)) return Response(f"Alive, host is: {subdomain_tag}.{self.fake_host}", status=200) # Host Header Overrides From 4d2c88ae1d093ef9f130c321a7f27217ebc92cbe Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 11 Feb 2025 13:25:32 -0500 Subject: [PATCH 09/18] ruff format --- .../test_step_2/module_tests/test_module_generic_ssrf.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py b/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py index 919e4360f..627e2c42a 100644 --- a/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py +++ b/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py @@ -24,9 +24,11 @@ def request_handler(self, request): elif request.method == "POST": subdomain_tag = extract_subdomain_tag(request.data.decode()) if subdomain_tag: - asyncio.run(self.interactsh_mock_instance.mock_interaction( - subdomain_tag, msg=f"{request.method}: {request.data.decode()}" - )) + asyncio.run( + self.interactsh_mock_instance.mock_interaction( + subdomain_tag, msg=f"{request.method}: {request.data.decode()}" + ) + ) return Response("alive", status=200) From 406470aee643840e5d71871a604f2adad8efe76f Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 11 Feb 2025 14:02:46 -0500 Subject: [PATCH 10/18] why was this here? --- bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py b/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py index 627e2c42a..c0911fd66 100644 --- a/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py +++ b/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py @@ -34,7 +34,6 @@ def request_handler(self, request): async def setup_before_prep(self, module_test): self.interactsh_mock_instance = module_test.mock_interactsh("generic_ssrf") - asyncio.run(self.interactsh_mock_instance.mock_interaction("asdf")) module_test.monkeypatch.setattr( module_test.scan.helpers, "interactsh", lambda *args, **kwargs: self.interactsh_mock_instance ) From 11bffa30516bce2168a2a43c35758bb74a065a38 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 11 Feb 2025 15:01:58 -0500 Subject: [PATCH 11/18] better mocking of poll function --- bbot/test/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index b8902fbc2..6693c7eb1 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -163,13 +163,13 @@ async def poll_loop(self, callback=None): async def poll(self, callback=None): async with self.lock: poll_results = [] - for subdomain_tag in self.interactions: + while self.interactions: + subdomain_tag = self.interactions.pop(0) for protocol in ["HTTP", "DNS"]: result = {"full-id": f"{subdomain_tag}.fakedomain.fakeinteractsh.com", "protocol": protocol} poll_results.append(result) if callback is not None: await execute_sync_or_async(callback, result) - self.interactions = [] return poll_results From 369034704487e33bace7b1da3be9525f6f1f8dd1 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 11 Feb 2025 15:45:33 -0500 Subject: [PATCH 12/18] test debugging --- bbot/test/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index 6693c7eb1..ba9cb5121 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -147,6 +147,7 @@ async def register(self, callback=None): return "fakedomain.fakeinteractsh.com" async def deregister(self, callback=None): + await asyncio.sleep(2) self.stop = True if self.poll_task is not None: self.poll_task.cancel() From b9a926a786d630827411fdfb331a56d197b4aef6 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 11 Feb 2025 17:04:47 -0500 Subject: [PATCH 13/18] async queues <3 --- bbot/modules/generic_ssrf.py | 2 +- bbot/test/conftest.py | 30 ++++++++++++++---------------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/bbot/modules/generic_ssrf.py b/bbot/modules/generic_ssrf.py index dba56b5c4..6ccde510b 100644 --- a/bbot/modules/generic_ssrf.py +++ b/bbot/modules/generic_ssrf.py @@ -254,7 +254,7 @@ async def cleanup(self): async def finish(self): if self.scan.config.get("interactsh_disable", False) is False: - await self.helpers.sleep(10) + await self.helpers.sleep(5) try: for r in await self.interactsh_instance.poll(): await self.interactsh_callback(r) diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index ba9cb5121..3e5a061f1 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -128,18 +128,16 @@ class Interactsh_mock: def __init__(self, name): self.name = name self.log = logging.getLogger(f"bbot.interactsh.{self.name}") - self.interactions = [] + self.interactions = asyncio.Queue() # Use asyncio.Queue for safe concurrent access self.correlation_id = "deadbeef-dead-beef-dead-beefdeadbeef" self.stop = False self.poll_task = None - self.lock = asyncio.Lock() async def mock_interaction(self, subdomain_tag, msg=None): self.log.info(f"Mocking interaction to subdomain tag: {subdomain_tag}") if msg is not None: self.log.info(msg) - async with self.lock: - self.interactions.append(subdomain_tag) + await self.interactions.put(subdomain_tag) # Add to the queue async def register(self, callback=None): if callable(callback): @@ -147,7 +145,6 @@ async def register(self, callback=None): return "fakedomain.fakeinteractsh.com" async def deregister(self, callback=None): - await asyncio.sleep(2) self.stop = True if self.poll_task is not None: self.poll_task.cancel() @@ -158,20 +155,21 @@ async def poll_loop(self, callback=None): while not self.stop: data_list = await self.poll(callback) if not data_list: - await asyncio.sleep(1) + await asyncio.sleep(0.5) continue + await asyncio.sleep(2) + await self.poll(callback) async def poll(self, callback=None): - async with self.lock: - poll_results = [] - while self.interactions: - subdomain_tag = self.interactions.pop(0) - for protocol in ["HTTP", "DNS"]: - result = {"full-id": f"{subdomain_tag}.fakedomain.fakeinteractsh.com", "protocol": protocol} - poll_results.append(result) - if callback is not None: - await execute_sync_or_async(callback, result) - return poll_results + poll_results = [] + while not self.interactions.empty(): + subdomain_tag = await self.interactions.get() # Pop from the queue + for protocol in ["HTTP", "DNS"]: + result = {"full-id": f"{subdomain_tag}.fakedomain.fakeinteractsh.com", "protocol": protocol} + poll_results.append(result) + if callback is not None: + await execute_sync_or_async(callback, result) + return poll_results import threading From 4db19ca50701a4d6d7572802b784b425145f13fb Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 11 Feb 2025 17:32:22 -0500 Subject: [PATCH 14/18] fix test --- bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py b/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py index 1bfdfc794..097fc9163 100644 --- a/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py +++ b/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py @@ -172,7 +172,5 @@ def check(self, module_test, events): if e.type == "VULNERABILITY" and "DotNetNuke Blind-SSRF (CVE 2017-0929)" in e.data["description"]: dnn_dnnimagehandler_blindssrf = True - assert self.interactsh_mock_instance.interactions == [] - assert dnn_technology_detection, "DNN Technology Detection Failed" assert dnn_dnnimagehandler_blindssrf, "dnnimagehandler.ashx Blind SSRF Detection Failed" From 0a812de479329384388c9b607269dc2c7d00eff2 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 11 Feb 2025 18:24:14 -0500 Subject: [PATCH 15/18] more adjustments --- bbot/test/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index 3e5a061f1..8fa326452 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -145,6 +145,7 @@ async def register(self, callback=None): return "fakedomain.fakeinteractsh.com" async def deregister(self, callback=None): + await asyncio.sleep(2) self.stop = True if self.poll_task is not None: self.poll_task.cancel() From 422c490987f7f6ca3106a6acd369b056b5fe84f9 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 11 Feb 2025 19:06:42 -0500 Subject: [PATCH 16/18] more adjustments --- bbot/test/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index 8fa326452..da1f69dd2 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -170,6 +170,7 @@ async def poll(self, callback=None): poll_results.append(result) if callback is not None: await execute_sync_or_async(callback, result) + await asyncio.sleep(0.5) return poll_results From bc711589adf3558f40b034c350bf3ba8eb428919 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Wed, 12 Feb 2025 14:01:10 -0500 Subject: [PATCH 17/18] removing asyncio.run weirdness --- bbot/test/conftest.py | 21 ++++++++++++------- .../module_tests/test_module_dotnetnuke.py | 2 +- .../module_tests/test_module_host_header.py | 2 +- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index da1f69dd2..95b9c1fbc 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -9,6 +9,7 @@ from omegaconf import OmegaConf from pytest_httpserver import HTTPServer import time +import queue from bbot.core import CORE from bbot.core.helpers.misc import execute_sync_or_async @@ -128,16 +129,17 @@ class Interactsh_mock: def __init__(self, name): self.name = name self.log = logging.getLogger(f"bbot.interactsh.{self.name}") - self.interactions = asyncio.Queue() # Use asyncio.Queue for safe concurrent access + self.interactions = queue.Queue() # Use a thread-safe queue for sync access + self.async_interactions = asyncio.Queue() # Use an asyncio queue for async access self.correlation_id = "deadbeef-dead-beef-dead-beefdeadbeef" self.stop = False self.poll_task = None - async def mock_interaction(self, subdomain_tag, msg=None): + def mock_interaction(self, subdomain_tag, msg=None): self.log.info(f"Mocking interaction to subdomain tag: {subdomain_tag}") if msg is not None: self.log.info(msg) - await self.interactions.put(subdomain_tag) # Add to the queue + self.interactions.put(subdomain_tag) # Add to the thread-safe queue async def register(self, callback=None): if callable(callback): @@ -145,7 +147,7 @@ async def register(self, callback=None): return "fakedomain.fakeinteractsh.com" async def deregister(self, callback=None): - await asyncio.sleep(2) + await asyncio.sleep(1) self.stop = True if self.poll_task is not None: self.poll_task.cancel() @@ -158,19 +160,24 @@ async def poll_loop(self, callback=None): if not data_list: await asyncio.sleep(0.5) continue - await asyncio.sleep(2) + await asyncio.sleep(1) await self.poll(callback) async def poll(self, callback=None): poll_results = [] + # Transfer items from the thread-safe queue to the asyncio queue (thanks pytest!) while not self.interactions.empty(): - subdomain_tag = await self.interactions.get() # Pop from the queue + subdomain_tag = self.interactions.get() + await self.async_interactions.put(subdomain_tag) + + while not self.async_interactions.empty(): + subdomain_tag = await self.async_interactions.get() # Get the first element from the asyncio queue for protocol in ["HTTP", "DNS"]: result = {"full-id": f"{subdomain_tag}.fakedomain.fakeinteractsh.com", "protocol": protocol} poll_results.append(result) if callback is not None: await execute_sync_or_async(callback, result) - await asyncio.sleep(0.5) + await asyncio.sleep(0.1) return poll_results diff --git a/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py b/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py index 097fc9163..8035316de 100644 --- a/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py +++ b/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py @@ -143,7 +143,7 @@ def request_handler(self, request): subdomain_tag = None subdomain_tag = extract_subdomain_tag(request.full_path) if subdomain_tag: - asyncio.run(self.interactsh_mock_instance.mock_interaction(subdomain_tag)) + self.interactsh_mock_instance.mock_interaction(subdomain_tag) return Response("alive", status=200) async def setup_before_prep(self, module_test): diff --git a/bbot/test/test_step_2/module_tests/test_module_host_header.py b/bbot/test/test_step_2/module_tests/test_module_host_header.py index ffb6c05fa..2c4cf5a7d 100644 --- a/bbot/test/test_step_2/module_tests/test_module_host_header.py +++ b/bbot/test/test_step_2/module_tests/test_module_host_header.py @@ -24,7 +24,7 @@ def request_handler(self, request): # Standard (with reflection) if subdomain_tag: - asyncio.run(self.interactsh_mock_instance.mock_interaction(subdomain_tag)) + self.interactsh_mock_instance.mock_interaction(subdomain_tag) return Response(f"Alive, host is: {subdomain_tag}.{self.fake_host}", status=200) # Host Header Overrides From b1e8d11f3c04ae484288f8ed6214e8a6304e0201 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Wed, 12 Feb 2025 15:36:47 -0500 Subject: [PATCH 18/18] cleaner --- bbot/test/conftest.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index 95b9c1fbc..ed9aec159 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -129,8 +129,7 @@ class Interactsh_mock: def __init__(self, name): self.name = name self.log = logging.getLogger(f"bbot.interactsh.{self.name}") - self.interactions = queue.Queue() # Use a thread-safe queue for sync access - self.async_interactions = asyncio.Queue() # Use an asyncio queue for async access + self.interactions = asyncio.Queue() # Use an asyncio queue for async access self.correlation_id = "deadbeef-dead-beef-dead-beefdeadbeef" self.stop = False self.poll_task = None @@ -139,7 +138,7 @@ def mock_interaction(self, subdomain_tag, msg=None): self.log.info(f"Mocking interaction to subdomain tag: {subdomain_tag}") if msg is not None: self.log.info(msg) - self.interactions.put(subdomain_tag) # Add to the thread-safe queue + self.interactions.put_nowait(subdomain_tag) # Add to the thread-safe queue async def register(self, callback=None): if callable(callback): @@ -165,13 +164,8 @@ async def poll_loop(self, callback=None): async def poll(self, callback=None): poll_results = [] - # Transfer items from the thread-safe queue to the asyncio queue (thanks pytest!) while not self.interactions.empty(): - subdomain_tag = self.interactions.get() - await self.async_interactions.put(subdomain_tag) - - while not self.async_interactions.empty(): - subdomain_tag = await self.async_interactions.get() # Get the first element from the asyncio queue + subdomain_tag = await self.interactions.get() # Get the first element from the asyncio queue for protocol in ["HTTP", "DNS"]: result = {"full-id": f"{subdomain_tag}.fakedomain.fakeinteractsh.com", "protocol": protocol} poll_results.append(result)