From fac607d76c12b7b9d190a497061d4ae1547d194d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20BASLER?= Date: Tue, 16 Dec 2025 15:59:02 +0100 Subject: [PATCH 01/13] [Shodan] test poetry --- shodan/.dockerignore | 4 + shodan/.gitignore | 2 + shodan/CONTRIBUTING.md | 0 shodan/Dockerfile | 22 +++ shodan/README.md | 0 shodan/docker-compose.yml | 14 ++ shodan/pyproject.toml | 141 ++++++++++++++++++ shodan/requirements.txt | 5 + shodan/shodan/__init__.py | 3 + shodan/shodan/__main__.py | 55 +++++++ shodan/shodan/config.yml.sample | 11 ++ shodan/shodan/contracts/__init__.py | 19 +++ .../__init__.py | 0 .../contract.py | 0 .../__init__.py | 0 .../contract.py | 0 .../shodan/contracts/custom_query/__init__.py | 0 .../shodan/contracts/custom_query/contract.py | 0 .../contracts/cve_enumeration/__init__.py | 0 .../contracts/cve_enumeration/contract.py | 0 .../cve_specific_watchlist/__init__.py | 0 .../cve_specific_watchlist/contract.py | 0 .../contracts/domain_discovery/__init__.py | 0 .../contracts/domain_discovery/contract.py | 0 .../contracts/host_enumeration/__init__.py | 0 .../contracts/host_enumeration/contract.py | 0 shodan/shodan/contracts/shodan_contracts.py | 51 +++++++ shodan/shodan/helpers/__init__.py | 0 .../shodan/helpers/shodan_command_builder.py | 0 shodan/shodan/helpers/shodan_output_parser.py | 0 shodan/shodan/helpers/shodan_process.py | 0 shodan/shodan/img/icon-shodan.png | Bin 0 -> 1844 bytes shodan/shodan/injector/__init__.py | 0 shodan/shodan/injector/openaev_shodan.py | 18 +++ shodan/shodan/models/__init__.py | 3 + shodan/shodan/models/configs/__init__.py | 13 ++ shodan/shodan/models/configs/base_settings.py | 20 +++ shodan/shodan/models/configs/config_loader.py | 121 +++++++++++++++ .../shodan/models/configs/injector_configs.py | 45 ++++++ .../shodan/models/configs/shodan_configs.py | 16 ++ shodan/tests/__init__.py | 0 41 files changed, 563 insertions(+) create mode 100644 shodan/.dockerignore create mode 100644 shodan/.gitignore create mode 100644 shodan/CONTRIBUTING.md create mode 100644 shodan/Dockerfile create mode 100644 shodan/README.md create mode 100644 shodan/docker-compose.yml create mode 100644 shodan/pyproject.toml create mode 100644 shodan/requirements.txt create mode 100644 shodan/shodan/__init__.py create mode 100644 shodan/shodan/__main__.py create mode 100644 shodan/shodan/config.yml.sample create mode 100644 shodan/shodan/contracts/__init__.py create mode 100644 shodan/shodan/contracts/cloud_provider_asset_discovery/__init__.py create mode 100644 shodan/shodan/contracts/cloud_provider_asset_discovery/contract.py create mode 100644 shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/__init__.py create mode 100644 shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/contract.py create mode 100644 shodan/shodan/contracts/custom_query/__init__.py create mode 100644 shodan/shodan/contracts/custom_query/contract.py create mode 100644 shodan/shodan/contracts/cve_enumeration/__init__.py create mode 100644 shodan/shodan/contracts/cve_enumeration/contract.py create mode 100644 shodan/shodan/contracts/cve_specific_watchlist/__init__.py create mode 100644 shodan/shodan/contracts/cve_specific_watchlist/contract.py create mode 100644 shodan/shodan/contracts/domain_discovery/__init__.py create mode 100644 shodan/shodan/contracts/domain_discovery/contract.py create mode 100644 shodan/shodan/contracts/host_enumeration/__init__.py create mode 100644 shodan/shodan/contracts/host_enumeration/contract.py create mode 100644 shodan/shodan/contracts/shodan_contracts.py create mode 100644 shodan/shodan/helpers/__init__.py create mode 100644 shodan/shodan/helpers/shodan_command_builder.py create mode 100644 shodan/shodan/helpers/shodan_output_parser.py create mode 100644 shodan/shodan/helpers/shodan_process.py create mode 100644 shodan/shodan/img/icon-shodan.png create mode 100644 shodan/shodan/injector/__init__.py create mode 100644 shodan/shodan/injector/openaev_shodan.py create mode 100644 shodan/shodan/models/__init__.py create mode 100644 shodan/shodan/models/configs/__init__.py create mode 100644 shodan/shodan/models/configs/base_settings.py create mode 100644 shodan/shodan/models/configs/config_loader.py create mode 100644 shodan/shodan/models/configs/injector_configs.py create mode 100644 shodan/shodan/models/configs/shodan_configs.py create mode 100644 shodan/tests/__init__.py diff --git a/shodan/.dockerignore b/shodan/.dockerignore new file mode 100644 index 00000000..8d8716b4 --- /dev/null +++ b/shodan/.dockerignore @@ -0,0 +1,4 @@ +config.yml +src/__pycache__ +__pycache__ + diff --git a/shodan/.gitignore b/shodan/.gitignore new file mode 100644 index 00000000..b93baaf2 --- /dev/null +++ b/shodan/.gitignore @@ -0,0 +1,2 @@ +config.yml +__pycache__ \ No newline at end of file diff --git a/shodan/CONTRIBUTING.md b/shodan/CONTRIBUTING.md new file mode 100644 index 00000000..e69de29b diff --git a/shodan/Dockerfile b/shodan/Dockerfile new file mode 100644 index 00000000..ec661071 --- /dev/null +++ b/shodan/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.13-alpine AS build + +WORKDIR /opt/openaev-injector-shodan +COPY shodan /opt/openaev-injector-shodan + +# Build dependencies + pip install +RUN apk add --no-cache --virtual .build-dependencies \ + build-base git libffi-dev libxml2-dev libxslt-dev && \ + pip install --no-cache-dir -r shodan/requirements.txt && \ + apk del .build-dependencies + +FROM python:3.13-alpine AS runner + +WORKDIR /opt/openaev-injector-shodan + +# Runtime libraries +RUN apk add --no-cache libmagic libffi libxml2 libxslt + +COPY --from=build /opt/openaev-injector-shodan/shodan ./shodan +COPY --from=build /usr/local/lib/python3.13/site-packages /usr/local/lib/python3.13/site-packages + +CMD ["python3", "-m", "shodan"] \ No newline at end of file diff --git a/shodan/README.md b/shodan/README.md new file mode 100644 index 00000000..e69de29b diff --git a/shodan/docker-compose.yml b/shodan/docker-compose.yml new file mode 100644 index 00000000..c5530f61 --- /dev/null +++ b/shodan/docker-compose.yml @@ -0,0 +1,14 @@ +version: '3.8' +services: + injector-shodan: + image: openaev/injector-shodan:2.0.5 + environment: + - OPENAEV_URL=http://localhost + - OPENAEV_TOKEN=ChangeMe + - INJECTOR_ID=ChangeMe + - INJECTOR_NAME=Shodan + - INJECTOR_LOG_LEVEL=error + restart: always + depends_on: + openaev: + condition: service_healthy diff --git a/shodan/pyproject.toml b/shodan/pyproject.toml new file mode 100644 index 00000000..5dea98d8 --- /dev/null +++ b/shodan/pyproject.toml @@ -0,0 +1,141 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[project.urls] +Homepage = "https://filigran.io/" +Repository = "https://github.com/OpenAEV-Platform/injectors/tree/main/shodan" +Documentation = "https://github.com/OpenAEV-Platform/injectors/tree/main/shodan/README.md" +Issues = "https://github.com/OpenAEV-Platform/injectors/issues" + +[project] +name = "openaev-shodan-injector" +version = "2.0.5" +description = "An injector for running with Shodan" +readme = "README.md" +authors = [{ name = "Filigran", email = "contact@filigran.io" }] +license = "Apache-2.0" +requires-python = ">=3.11, <3.13" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +dependencies = [ + "pydantic~=2.11.7", + "pydantic-settings~=2.11.0", + "requests~=2.32.5" +] + +[project.optional-dependencies] +prod = [ + "pyoaev ~=2.0.5" +] +current = [ + "pyoaev @ git+https://github.com/OpenAEV-Platform/client-python.git@release/current" +] +dev = [ + "black~=25.1", # Code formatter + "isort~=7.0.0", # Import sorter + "ruff~=0.14.2", # linter + "mypy~=1.18.2", # Type validator + "pip_audit~=2.9.0", # Security checker + "pre-commit~=4.3.0", # Git hooks + "flake8~=7.3.0", # Linter + "types-PyYAML~=6.0.12", # stubs for untyped module +] +test = [ + "pytest~=8.4.1", + "polyfactory~=2.22.2", +] + +[tool.poetry] +packages = [{ include = "shodan" }] + +[project.scripts] +ShodanInjector = "shodan.__main__:main" + +[tool.setuptools.packages.find] +where = ["."] + +[tool.isort] +profile = "black" +src_paths = ["shodan"] + +[tool.pytest.ini_options] +testpaths = ["./tests"] + +[tool.ruff] +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "site-packages", + "venv", +] + +target-version = "py312" + +[tool.ruff.lint] +# Never enforce `I001` (unsorted import). Already handle with isort +# Never enforce `E501` (line length violations). Already handle with black +# Never enforce `F821` (Undefined name `null`). incorrect issue with notebook +# Never enforce `D213` (Multi-line docstring summary should start at the second line) conflict with our docstring convention +# Never enforce `D211` (NoBlankLinesBeforeClass)` +# Never enforce `G004` (logging-f-string) Logging statement uses f-string +# Never enforce `TRY003`() Avoid specifying long messages outside the exception class not useful +# Never enforce `D104` (Missing docstring in public package) +# Never enforce `D407` (Missing dashed underline after section) +# Never enforce `D408` (Section underline should be in the line following the section’s name) +# Never enforce `D409` (Section underline should match the length of its name) +ignore = [ + "I001", + "D203", + "E501", + "F821", + "D205", + "D213", + "D211", + "G004", + "TRY003", + "D104", + "D407", + "D408", + "D409", +] +select = ["E", "F", "W", "D", "G", "T", "B", "C", "N", "I", "S"] + +[tool.mypy] +strict = true +exclude = [ + '^tests', + '^docs', + '^build', + '^dist', + '^venv', + '^site-packages', + '^__pypackages__', + '^.venv', +] +plugins = ["pydantic.mypy"] \ No newline at end of file diff --git a/shodan/requirements.txt b/shodan/requirements.txt new file mode 100644 index 00000000..2edfa69b --- /dev/null +++ b/shodan/requirements.txt @@ -0,0 +1,5 @@ +# Local dependency (client-python) +-e ../../client-python + +# Local project with extras (dev + test) +-e .[dev,test] \ No newline at end of file diff --git a/shodan/shodan/__init__.py b/shodan/shodan/__init__.py new file mode 100644 index 00000000..e1d414cc --- /dev/null +++ b/shodan/shodan/__init__.py @@ -0,0 +1,3 @@ +from shodan.models import ConfigLoader + +__all__ = ["ConfigLoader"] \ No newline at end of file diff --git a/shodan/shodan/__main__.py b/shodan/shodan/__main__.py new file mode 100644 index 00000000..ff8b526e --- /dev/null +++ b/shodan/shodan/__main__.py @@ -0,0 +1,55 @@ +"""Main entry point for the injector.""" + +import logging +import os +import sys +from pathlib import Path +from pydantic import ValidationError +from pyoaev.helpers import OpenAEVInjectorHelper +from shodan.models import ConfigLoader +from shodan.injector.openaev_shodan import ShodanInjector +from shodan.contracts.shodan_contracts import ShodanContracts +# from shodan.injector.exception import InjectorConfigError + +LOG_PREFIX = "[MAIN]" + +def main() -> None: + """Define the main function to run the injector.""" + logger = logging.getLogger(__name__) + + try: + logger.info(f"{LOG_PREFIX} Starting Shodan Injector...") + + # Injector Config + config = ConfigLoader() + # config_dict = config.model_dump(mode="json") + + # Prepare Helper + shodan_contracts = ShodanContracts().contracts + config_helper_adpater = config.to_config_injector_helper_adapter(contracts=shodan_contracts) + icon_bytes = Path("shodan/img/icon-shodan.png").read_bytes() + + helper = OpenAEVInjectorHelper(config=config_helper_adpater, icon=icon_bytes) + + logger.info( # type: ignore[has-type] + f"{LOG_PREFIX} The initialization of the Shodan injector configuration was successful." + ) + + injector = ShodanInjector(config, helper) + injector.start() + + except ValidationError as err: + logger.error(f"{LOG_PREFIX} Configuration error: {err}") + sys.exit(2) + + except KeyboardInterrupt: + logger.info(f"{LOG_PREFIX} Injector stopped by user (Ctrl+C)") + os._exit(0) + + except Exception as err: + logger.exception(f"{LOG_PREFIX} Fatal error starting injector: {err}") + sys.exit(1) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/shodan/shodan/config.yml.sample b/shodan/shodan/config.yml.sample new file mode 100644 index 00000000..d7aee2c0 --- /dev/null +++ b/shodan/shodan/config.yml.sample @@ -0,0 +1,11 @@ +openaev: + url: 'http://localhost:3001' + token: 'ChangeMe' + +injector: + id: 'ChangeMe' + name: 'Shodan' + log_level: 'WARN' + +shodan: + api_key: 'ChangeMe' \ No newline at end of file diff --git a/shodan/shodan/contracts/__init__.py b/shodan/shodan/contracts/__init__.py new file mode 100644 index 00000000..f871b0ff --- /dev/null +++ b/shodan/shodan/contracts/__init__.py @@ -0,0 +1,19 @@ +from shodan.contracts import ( + CloudProviderAssetDiscovery, + CriticalPortsAndExposedAdminInterface, + CustomQuery, + CVEEnumeration, + CVESpecificWatchlist, + DomainDiscovery, + HostEnumeration, +) + +__all__ = [ + "CloudProviderAssetDiscovery", + "CriticalPortsAndExposedAdminInterface", + "CustomQuery", + "CVEEnumeration", + "CVESpecificWatchlist", + "DomainDiscovery", + "HostEnumeration", +] \ No newline at end of file diff --git a/shodan/shodan/contracts/cloud_provider_asset_discovery/__init__.py b/shodan/shodan/contracts/cloud_provider_asset_discovery/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/shodan/contracts/cloud_provider_asset_discovery/contract.py b/shodan/shodan/contracts/cloud_provider_asset_discovery/contract.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/__init__.py b/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/contract.py b/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/contract.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/shodan/contracts/custom_query/__init__.py b/shodan/shodan/contracts/custom_query/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/shodan/contracts/custom_query/contract.py b/shodan/shodan/contracts/custom_query/contract.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/shodan/contracts/cve_enumeration/__init__.py b/shodan/shodan/contracts/cve_enumeration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/shodan/contracts/cve_enumeration/contract.py b/shodan/shodan/contracts/cve_enumeration/contract.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/shodan/contracts/cve_specific_watchlist/__init__.py b/shodan/shodan/contracts/cve_specific_watchlist/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/shodan/contracts/cve_specific_watchlist/contract.py b/shodan/shodan/contracts/cve_specific_watchlist/contract.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/shodan/contracts/domain_discovery/__init__.py b/shodan/shodan/contracts/domain_discovery/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/shodan/contracts/domain_discovery/contract.py b/shodan/shodan/contracts/domain_discovery/contract.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/shodan/contracts/host_enumeration/__init__.py b/shodan/shodan/contracts/host_enumeration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/shodan/contracts/host_enumeration/contract.py b/shodan/shodan/contracts/host_enumeration/contract.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/shodan/contracts/shodan_contracts.py b/shodan/shodan/contracts/shodan_contracts.py new file mode 100644 index 00000000..5d699e12 --- /dev/null +++ b/shodan/shodan/contracts/shodan_contracts.py @@ -0,0 +1,51 @@ +from pyoaev.contracts import ContractBuilder +from pyoaev.contracts.contract_config import ( + Contract, + ContractAttachment, + ContractCardinality, + ContractCheckbox, + ContractConfig, + ContractElement, + ContractOutputElement, + ContractOutputType, + ContractText, + ContractTextArea, + ContractTuple, + SupportedLanguage, + prepare_contracts, +) + +class ShodanContracts: + text_field = ContractText( + key="text", + label="Labeltext", + defaultValue="", + mandatory=True, + ) + my_contract_fields: list[ContractElement] = ( + ContractBuilder() + .add_fields([text_field]) + .build_fields() + ) + contracts = {"data": prepare_contracts([ + Contract( + contract_id="b62a0ce2-7c43-4f78-abcd-5a92413b66cf", + config=ContractConfig( + type="openaev_basetest", + label={ + SupportedLanguage.en: "Base Test", + SupportedLanguage.fr: "Base Test", + }, + color_dark="#00bcd4", + color_light="#00bcd4", + expose=True, + ), + label={ + SupportedLanguage.en: "Base Test - EN", + SupportedLanguage.fr: "Base Test - FR", + }, + fields=my_contract_fields, + outputs=[], + manual=False, + ) + ])} \ No newline at end of file diff --git a/shodan/shodan/helpers/__init__.py b/shodan/shodan/helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/shodan/helpers/shodan_command_builder.py b/shodan/shodan/helpers/shodan_command_builder.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/shodan/helpers/shodan_output_parser.py b/shodan/shodan/helpers/shodan_output_parser.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/shodan/helpers/shodan_process.py b/shodan/shodan/helpers/shodan_process.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/shodan/img/icon-shodan.png b/shodan/shodan/img/icon-shodan.png new file mode 100644 index 0000000000000000000000000000000000000000..c44f0cedaf2331152afb9793e3577b7368e4af2a GIT binary patch literal 1844 zcmV-42g~@0P){ntuo)$u>IshKgsJSS&7hDvO1Zmti_Nkjfh z56H3h(7}BO3<(6Jo?TM+P8!=Pwzb-+w9ho9_ExcZ{l+JWojSQn_5UzJ$kh2<5NLu2 zdhyYJ=smfAKQQH=W89m_4qmZU)tFDDe!g`{%6Z9Moe?^tDQ&?}_@h3&S3W%^_rFVM zF+YUOo9K>CW9+Wrmu=11HU9!ZN>@uXXy(UQDoZm4DMDnNVFUj6zmB@W=WMLvAGE(LZKk|+ciCwT*8;gmy@i6 zRZwb=>tl$i=!PybyFHsY%2hv zx??l(nuQT)&TBy=NZH#BG^N#@MN46C+FCt{Z?pqpp^*)~@fwvHnL=#Bk{mvC0BPby zfGVNgU2Y`I*|;`xTfZ~04SZi zHe3bG^kYQG)^ub3*@$S=0I1KT$dsEbPkkraVNO4)P)`V9=F(-nQO6$w(^XOTT09ngj2N{6Jk-E~b|e!a6$R3048B3gT4g;rR!r z7|OP&x4@dA6%u**{#GiBa{%|S=7ZXO@A%r1(Rv98A`wfo%fPkV^FBt4M)R+ z%F!Syp)Ws2+u`RIIa(kiO2BwH$w0TaBD5`8W@;bg^|)e_56Olf!n}tS!*_D2EX}X^ zMCRrw(BPb=V;|rz>u#q{Rn=AS01D293St33U-(mQF%_G9WE6QtBsVQAYJdHWTnOP{ z5iG1M``aI^J?Ts=0^Oa`md(uVA3uT=dlGj-&~$NIU=our{t?)uTUHnW_N+e*at`QL zWc*L4Ew|N75j5SEJ17eB3vcP~+R7V_N2PE^gZ*eVC_&@_yz4h5x9^r!YMj9yX;O?} zF~jsdUG_a`(T0k0BXL-j9I=u3_)){WjHz-@FH{5T#N~aO=hlG-moqT{%C29TpgsEQ ze4`COTnLPK0$zj5cm3Nj5t|fw$l7)}lx4313Uj~O=>AZ+5K!Gm5o78HG$|A`NH6BP z0w()8Jl0^+xTbzgDvk88S|+XScYW}M}jjYr6Yrop-|i_`synl9%y0pY@U=9as#%Hh&sv2oMw!{TM33 z@5_ds3l}S6!a3X~xU&(=TE2m-e@k!0XIk%D5_z(K?ky;OSOXXYVO>~UrIP@m4Uecp zKXPw`WT~SMP=)8@Y`!HTjc=Tz>z&aQ$Vu=(uj>_=J6oBGoWRV1hHmhH2D1^+z%))4 z&evp3cJetUHu6?8E#Or4_q*UhNi=JaGB154+d;zRJi}sk*{iT7MV^Ha)s8`(p iA^-mxROR?rfB^s@o~|4FOYw350000 None: + self.helper.injector_logger.info(data) + + def start(self): + self.helper.listen(message_callback=self.process_message) diff --git a/shodan/shodan/models/__init__.py b/shodan/shodan/models/__init__.py new file mode 100644 index 00000000..b2e90438 --- /dev/null +++ b/shodan/shodan/models/__init__.py @@ -0,0 +1,3 @@ +from shodan.models.configs.config_loader import ConfigLoader + +__all__ = ["ConfigLoader"] \ No newline at end of file diff --git a/shodan/shodan/models/configs/__init__.py b/shodan/shodan/models/configs/__init__.py new file mode 100644 index 00000000..f37c3c0c --- /dev/null +++ b/shodan/shodan/models/configs/__init__.py @@ -0,0 +1,13 @@ +from shodan.models.configs.base_settings import _SettingsLoader +from shodan.models.configs.injector_configs import ( + _BaseInjectorConfig, + _BaseOpenAEVConfig, +) +from shodan.models.configs.shodan_configs import _ConfigLoaderShodan + +__all__ = [ + "_SettingsLoader", + "_BaseOpenAEVConfig", + "_BaseInjectorConfig", + "_ConfigLoaderShodan", +] diff --git a/shodan/shodan/models/configs/base_settings.py b/shodan/shodan/models/configs/base_settings.py new file mode 100644 index 00000000..993cd8f8 --- /dev/null +++ b/shodan/shodan/models/configs/base_settings.py @@ -0,0 +1,20 @@ +"""Base class for global config models.""" + +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class _SettingsLoader(BaseSettings): + """Base class for global config models. + + Provides common configuration settings and prevents attributes from being + modified after initialization by using frozen=True in the model config. + """ + + model_config = SettingsConfigDict( + frozen=True, + extra="allow", + env_nested_delimiter="_", + env_nested_max_split=1, + validate_default=True, + enable_decoding=False, + ) diff --git a/shodan/shodan/models/configs/config_loader.py b/shodan/shodan/models/configs/config_loader.py new file mode 100644 index 00000000..16c190f9 --- /dev/null +++ b/shodan/shodan/models/configs/config_loader.py @@ -0,0 +1,121 @@ +"""Base class for global config models.""" +from pathlib import Path +from pydantic import Field, BaseModel + +from pydantic_settings import ( + BaseSettings, + DotEnvSettingsSource, + EnvSettingsSource, + PydanticBaseSettingsSource, + YamlConfigSettingsSource, +) + +from shodan.models.configs import ( + _SettingsLoader, + _BaseOpenAEVConfig, + _BaseInjectorConfig, + _ConfigLoaderShodan, +) + +class _BaseInjectorConfigHelperAdapter: + def __init__(self, data: dict): + self.data = data + + def get_conf(self, key, default=None): + value = self.data.get(key, default) + if isinstance(value, dict) and "data" in value: + value = value["data"] + return value + +class _BaseInjectorConfigUtils: + + def to_flatten(self, contracts=None) -> dict: + + flatten_config = {} + for field_name in ["openaev", "injector"]: + value = getattr(self, field_name, None) + if isinstance(value, BaseModel): + for subfield, subvalue in value.__dict__.items(): + flatten_config[f"{field_name}_{subfield}"] = str(subvalue) + elif value is not None: + flatten_config[field_name] = str(value) + if contracts: + flatten_config["injector_contracts"] = contracts + return flatten_config + + def to_config_injector_helper_adapter(self, contracts) -> _BaseInjectorConfigHelperAdapter: + """Returns an OpenAEVInjectorHelper-compatible object""" + flatten_dict = self.to_flatten(contracts) + return _BaseInjectorConfigHelperAdapter(flatten_dict) + + +class ConfigLoader(_BaseInjectorConfigUtils, _SettingsLoader): + """Configuration loader for the injector.""" + + openaev: _BaseOpenAEVConfig = Field( + default_factory=_BaseOpenAEVConfig, + description="Base OpenAEV configurations.", + ) + injector: _BaseInjectorConfig = Field( + default_factory=_BaseInjectorConfig, + description="Base Injector configurations.", + ) + shodan: _ConfigLoaderShodan = Field( + default_factory=_ConfigLoaderShodan, + description="Shodan configurations.", + ) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource]: + """Pydantic settings customisation sources. + + Defines the priority order for loading configuration settings: + 1. .env file (if exists) + 2. config.yml file (if exists) + 3. Environment variables (fallback) + + Args: + settings_cls: The settings class being configured. + init_settings: Initialization settings source. + env_settings: Environment variables settings source. + dotenv_settings: .env file settings source. + file_secret_settings: File secrets settings source. + + Returns: + Tuple containing the selected settings source. + + """ + env_path = Path(__file__).parents[2] / ".env" + yaml_path = Path(__file__).parents[2] / "config.yml" + + if env_path.exists(): + return ( + DotEnvSettingsSource( + settings_cls, + env_file=env_path, + env_ignore_empty=True, + env_file_encoding="utf-8", + ), + ) + elif yaml_path.exists(): + return ( + YamlConfigSettingsSource( + settings_cls, + yaml_file=yaml_path, + yaml_file_encoding="utf-8", + ), + ) + else: + return ( + EnvSettingsSource( + settings_cls, + env_ignore_empty=True, + ), + ) \ No newline at end of file diff --git a/shodan/shodan/models/configs/injector_configs.py b/shodan/shodan/models/configs/injector_configs.py new file mode 100644 index 00000000..dfaa44d2 --- /dev/null +++ b/shodan/shodan/models/configs/injector_configs.py @@ -0,0 +1,45 @@ +"""Base class for global config models.""" + +from abc import ABC + +from typing import Literal +from pydantic import ( + BaseModel, + ConfigDict, + Field, + HttpUrl, +) + +class BaseConfigModel(BaseModel, ABC): + """Base class for global config models + To prevent attributes from being modified after initialization. + """ + model_config = ConfigDict(extra="allow", str_min_length=1, frozen=True, validate_default=True) + +class _BaseOpenAEVConfig(BaseConfigModel, ABC): + url: HttpUrl = Field( + description="The base URL of the OpenAEV instance.", + ) + token: str = Field( + description="The API token to connect to OpenAEV.", + ) + +class _BaseInjectorConfig(BaseConfigModel, ABC): + """Base class for connector configuration.""" + + id: str = Field( + default="shodan--a87488ad-2c72-4592-b429-69259d7bcef1", + description="A unique UUIDv4 identifier for this injector instance.", + ) + name: str = Field( + default="Shodan", + description="Name of the injector.", + ) + type: str = Field( + default="openaev_shodan", + description="Identifies the functional type of the injector in OpenAEV" + ) + log_level: Literal["debug", "info", "warning", "error"] = Field( + default="error", + description="The minimum level of logs to display.", + ) \ No newline at end of file diff --git a/shodan/shodan/models/configs/shodan_configs.py b/shodan/shodan/models/configs/shodan_configs.py new file mode 100644 index 00000000..36a84d55 --- /dev/null +++ b/shodan/shodan/models/configs/shodan_configs.py @@ -0,0 +1,16 @@ +"""Configuration for Shodan injector.""" + +from pydantic import Field, SecretStr +from shodan.models.configs import _SettingsLoader + + +class _ConfigLoaderShodan(_SettingsLoader): + """Shodan API configuration settings.""" + + base_url: str = Field( + default="https://api.shodan.io", + description="URL for the Shodan API.", + ) + api_key: SecretStr = Field( + description="API Key for the Shodan API.", + ) \ No newline at end of file diff --git a/shodan/tests/__init__.py b/shodan/tests/__init__.py new file mode 100644 index 00000000..e69de29b From d1d13e627c674824bf18dcd6464cb283451e7818 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20BASLER?= Date: Tue, 13 Jan 2026 15:34:49 +0100 Subject: [PATCH 02/13] Add contracts --- shodan/pyproject.toml | 5 +- shodan/shodan/__main__.py | 15 +- shodan/shodan/config.yml.sample | 4 +- shodan/shodan/contracts/__init__.py | 19 +- .../__init__.py | 3 + .../contract.py | 84 +++++ .../__init__.py | 3 + .../contract.py | 87 +++++ .../shodan/contracts/custom_query/__init__.py | 3 + .../shodan/contracts/custom_query/contract.py | 80 +++++ .../contracts/cve_enumeration/__init__.py | 3 + .../contracts/cve_enumeration/contract.py | 79 +++++ .../cve_specific_watchlist/__init__.py | 3 + .../cve_specific_watchlist/contract.py | 85 +++++ .../contracts/domain_discovery/__init__.py | 3 + .../contracts/domain_discovery/contract.py | 79 +++++ .../contracts/host_enumeration/__init__.py | 3 + .../contracts/host_enumeration/contract.py | 74 +++++ shodan/shodan/contracts/shodan_contracts.py | 311 +++++++++++++++--- shodan/shodan/injector/openaev_shodan.py | 78 ++++- shodan/shodan/models/configs/config_loader.py | 1 - .../shodan/models/configs/shodan_configs.py | 4 +- shodan/shodan/services/__init__.py | 5 + shodan/shodan/services/client_api.py | 96 ++++++ shodan/shodan/services/utils.py | 0 shodan/tests/conftest.py | 0 .../cloud_provider_asset_discovery.py | 0 ...tical_ports_and_exposed_admin_interface.py | 0 shodan/tests/shodan_contracts/custom_query.py | 0 .../tests/shodan_contracts/cve_enumeration.py | 0 .../cve_specific_watchlist.py | 0 .../shodan_contracts/domain_discovery.py | 0 .../shodan_contracts/host_enumeration.py | 0 shodan/tests/test-requirements.txt | 0 34 files changed, 1063 insertions(+), 64 deletions(-) create mode 100644 shodan/shodan/services/__init__.py create mode 100644 shodan/shodan/services/client_api.py create mode 100644 shodan/shodan/services/utils.py create mode 100644 shodan/tests/conftest.py create mode 100644 shodan/tests/shodan_contracts/cloud_provider_asset_discovery.py create mode 100644 shodan/tests/shodan_contracts/critical_ports_and_exposed_admin_interface.py create mode 100644 shodan/tests/shodan_contracts/custom_query.py create mode 100644 shodan/tests/shodan_contracts/cve_enumeration.py create mode 100644 shodan/tests/shodan_contracts/cve_specific_watchlist.py create mode 100644 shodan/tests/shodan_contracts/domain_discovery.py create mode 100644 shodan/tests/shodan_contracts/host_enumeration.py create mode 100644 shodan/tests/test-requirements.txt diff --git a/shodan/pyproject.toml b/shodan/pyproject.toml index 5dea98d8..4abca131 100644 --- a/shodan/pyproject.toml +++ b/shodan/pyproject.toml @@ -25,7 +25,10 @@ classifiers = [ dependencies = [ "pydantic~=2.11.7", "pydantic-settings~=2.11.0", - "requests~=2.32.5" + "requests~=2.32.5", + "aiohttp~=3.13.2", + "limiter==0.5.0", + "tenacity~=9.1.2", ] [project.optional-dependencies] diff --git a/shodan/shodan/__main__.py b/shodan/shodan/__main__.py index ff8b526e..c4ab3b49 100644 --- a/shodan/shodan/__main__.py +++ b/shodan/shodan/__main__.py @@ -11,7 +11,7 @@ from shodan.contracts.shodan_contracts import ShodanContracts # from shodan.injector.exception import InjectorConfigError -LOG_PREFIX = "[MAIN]" +LOG_PREFIX = "[SHODAN_MAIN]" def main() -> None: """Define the main function to run the injector.""" @@ -22,16 +22,15 @@ def main() -> None: # Injector Config config = ConfigLoader() - # config_dict = config.model_dump(mode="json") - # Prepare Helper - shodan_contracts = ShodanContracts().contracts - config_helper_adpater = config.to_config_injector_helper_adapter(contracts=shodan_contracts) - icon_bytes = Path("shodan/img/icon-shodan.png").read_bytes() + # Prepare config for Helper + shodan_contracts = ShodanContracts(config).contracts() + config_helper_adapter = config.to_config_injector_helper_adapter(contracts=shodan_contracts) + icon_bytes = Path("img/icon-shodan.png").read_bytes() - helper = OpenAEVInjectorHelper(config=config_helper_adpater, icon=icon_bytes) + helper = OpenAEVInjectorHelper(config=config_helper_adapter, icon=icon_bytes) - logger.info( # type: ignore[has-type] + logger.info( f"{LOG_PREFIX} The initialization of the Shodan injector configuration was successful." ) diff --git a/shodan/shodan/config.yml.sample b/shodan/shodan/config.yml.sample index d7aee2c0..5dcc10b8 100644 --- a/shodan/shodan/config.yml.sample +++ b/shodan/shodan/config.yml.sample @@ -1,11 +1,11 @@ openaev: - url: 'http://localhost:3001' + url: 'ChangeMe' token: 'ChangeMe' injector: id: 'ChangeMe' name: 'Shodan' - log_level: 'WARN' + log_level: 'error' shodan: api_key: 'ChangeMe' \ No newline at end of file diff --git a/shodan/shodan/contracts/__init__.py b/shodan/shodan/contracts/__init__.py index f871b0ff..ab18d29d 100644 --- a/shodan/shodan/contracts/__init__.py +++ b/shodan/shodan/contracts/__init__.py @@ -1,12 +1,11 @@ -from shodan.contracts import ( - CloudProviderAssetDiscovery, - CriticalPortsAndExposedAdminInterface, - CustomQuery, - CVEEnumeration, - CVESpecificWatchlist, - DomainDiscovery, - HostEnumeration, -) +from .cloud_provider_asset_discovery import CloudProviderAssetDiscovery +from .critical_ports_and_exposed_admin_interface import CriticalPortsAndExposedAdminInterface +from .custom_query import CustomQuery +from .cve_enumeration import CVEEnumeration +from .cve_specific_watchlist import CVESpecificWatchlist +from .domain_discovery import DomainDiscovery +from .host_enumeration import HostEnumeration +from .shodan_contracts import InjectorKey, ShodanContractId __all__ = [ "CloudProviderAssetDiscovery", @@ -16,4 +15,6 @@ "CVESpecificWatchlist", "DomainDiscovery", "HostEnumeration", + "InjectorKey", + "ShodanContractId" ] \ No newline at end of file diff --git a/shodan/shodan/contracts/cloud_provider_asset_discovery/__init__.py b/shodan/shodan/contracts/cloud_provider_asset_discovery/__init__.py index e69de29b..4d814cb8 100644 --- a/shodan/shodan/contracts/cloud_provider_asset_discovery/__init__.py +++ b/shodan/shodan/contracts/cloud_provider_asset_discovery/__init__.py @@ -0,0 +1,3 @@ +from .contract import CloudProviderAssetDiscovery + +__all__ = ["CloudProviderAssetDiscovery"] \ No newline at end of file diff --git a/shodan/shodan/contracts/cloud_provider_asset_discovery/contract.py b/shodan/shodan/contracts/cloud_provider_asset_discovery/contract.py index e69de29b..71bed6b1 100644 --- a/shodan/shodan/contracts/cloud_provider_asset_discovery/contract.py +++ b/shodan/shodan/contracts/cloud_provider_asset_discovery/contract.py @@ -0,0 +1,84 @@ +from pyoaev.contracts import ContractBuilder +from pyoaev.contracts.contract_config import ( + Contract, + ContractConfig, + ContractElement, + ContractOutputElement, + ContractText, + SupportedLanguage, +) + +class CloudProviderAssetDiscovery: + + @staticmethod + def contract_with_specific_fields( + base_fields: list[ContractElement], + source_selector_key:str, + target_selector_field:str, + ) -> list[ContractElement]: + + mandatory_conditions = dict( + mandatoryConditionFields=[source_selector_key], + mandatoryConditionValues={source_selector_key: target_selector_field}, + ) + + visible_conditions = dict( + visibleConditionFields=[source_selector_key], + visibleConditionValues={source_selector_key: target_selector_field}, + ) + + specific_fields = [ + ContractText( + key="cloud_provider", + label="Cloud Provider", + mandatory=True, + **(mandatory_conditions | visible_conditions), + ), + ContractText( + key="hostname", + label="Hostname", + mandatory=True, + **(mandatory_conditions | visible_conditions), + ), + ContractText( + key="organization", + label="Organization", + **visible_conditions, + ), + ] + + contract_fields = ( + ContractBuilder() + .add_fields(base_fields + specific_fields) + .build_fields() + ) + return contract_fields + + @staticmethod + def contract_with_specific_outputs(base_outputs: list[ContractOutputElement]) -> list[ContractOutputElement]: + specific_outputs = [] + contract_outputs = ( + ContractBuilder() + .add_outputs(base_outputs + specific_outputs) + .build_outputs() + ) + return contract_outputs + + @staticmethod + def contract( + contract_id: str, + contract_config: ContractConfig, + contract_with_specific_fields: list[ContractElement], + contract_with_specific_outputs: list[ContractOutputElement] + ) -> Contract: + return Contract( + contract_id=contract_id, + config=contract_config, + label={ + SupportedLanguage.en: "Shodan - Cloud Provider Asset Discovery", + SupportedLanguage.fr: "Shodan - Découverte des actifs chez les fournisseurs cloud", + }, + fields=contract_with_specific_fields, + outputs=contract_with_specific_outputs, + manual=False, + ) \ No newline at end of file diff --git a/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/__init__.py b/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/__init__.py index e69de29b..71548d99 100644 --- a/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/__init__.py +++ b/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/__init__.py @@ -0,0 +1,3 @@ +from .contract import CriticalPortsAndExposedAdminInterface + +__all__ = ["CriticalPortsAndExposedAdminInterface"] \ No newline at end of file diff --git a/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/contract.py b/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/contract.py index e69de29b..05f7cb25 100644 --- a/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/contract.py +++ b/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/contract.py @@ -0,0 +1,87 @@ +from pyoaev.contracts import ContractBuilder +from pyoaev.contracts.contract_config import ( + Contract, + ContractConfig, + ContractElement, + ContractOutputElement, + ContractText, + ContractTuple, + SupportedLanguage, +) + + +class CriticalPortsAndExposedAdminInterface: + + @staticmethod + def contract_with_specific_fields( + base_fields: list[ContractElement], + source_selector_key:str, + target_selector_field:str + ) -> list[ContractElement]: + + mandatory_conditions = dict( + mandatoryConditionFields=[source_selector_key], + mandatoryConditionValues={source_selector_key: target_selector_field}, + ) + + visible_conditions = dict( + visibleConditionFields=[source_selector_key], + visibleConditionValues={source_selector_key: target_selector_field}, + ) + + specific_fields = [ + ContractTuple( + key="port", + label="Port", + mandatory=True, + defaultValue=["20","21","22","23","25","53","80","110","111","135","139","143","443","445","993","995","1723","3306","3389","5900","8080"], + **(mandatory_conditions | visible_conditions), + ), + ContractText( + key="hostname", + label="Hostname", + mandatory=True, + **(mandatory_conditions | visible_conditions), + ), + ContractText( + key="organization", + label="Organization", + **visible_conditions, + ), + ] + + contract_fields = ( + ContractBuilder() + .add_fields(base_fields + specific_fields) + .build_fields() + ) + return contract_fields + + @staticmethod + def contract_with_specific_outputs(base_outputs: list[ContractOutputElement]) -> list[ContractOutputElement]: + specific_outputs = [] + contract_outputs = ( + ContractBuilder() + .add_outputs(base_outputs + specific_outputs) + .build_outputs() + ) + return contract_outputs + + @staticmethod + def contract( + contract_id: str, + contract_config: ContractConfig, + contract_with_specific_fields: list[ContractElement], + contract_with_specific_outputs: list[ContractOutputElement] + ) -> Contract: + return Contract( + contract_id=contract_id, + config=contract_config, + label={ + SupportedLanguage.en: "Shodan - Critical ports and exposed admin interface", + SupportedLanguage.fr: "Shodan - Ports critiques et interface d'administration exposée", + }, + fields=contract_with_specific_fields, + outputs=contract_with_specific_outputs, + manual=False, + ) \ No newline at end of file diff --git a/shodan/shodan/contracts/custom_query/__init__.py b/shodan/shodan/contracts/custom_query/__init__.py index e69de29b..9e40dd73 100644 --- a/shodan/shodan/contracts/custom_query/__init__.py +++ b/shodan/shodan/contracts/custom_query/__init__.py @@ -0,0 +1,3 @@ +from .contract import CustomQuery + +__all__ = ["CustomQuery"] \ No newline at end of file diff --git a/shodan/shodan/contracts/custom_query/contract.py b/shodan/shodan/contracts/custom_query/contract.py index e69de29b..554bd46f 100644 --- a/shodan/shodan/contracts/custom_query/contract.py +++ b/shodan/shodan/contracts/custom_query/contract.py @@ -0,0 +1,80 @@ +from pyoaev.contracts import ContractBuilder +from pyoaev.contracts.contract_config import ( + Contract, + ContractConfig, + ContractElement, + ContractOutputElement, + ContractText, + SupportedLanguage, +) + + +class CustomQuery: + + @staticmethod + def contract_with_specific_fields( + base_fields: list[ContractElement], + source_selector_key:str, + target_selector_field:str, + ) -> list[ContractElement]: + + mandatory_conditions = dict( + mandatoryConditionFields=[source_selector_key], + mandatoryConditionValues={source_selector_key: target_selector_field}, + ) + + visible_conditions = dict( + visibleConditionFields=[source_selector_key], + visibleConditionValues={source_selector_key: target_selector_field}, + ) + + specific_fields = [ + ContractText( + key="custom_request_overview", + label="Custom Request Overview", + readOnly=True, + **visible_conditions, + ), + ContractText( + key="custom_request", + label="Custom Request", + mandatory=True, + **(mandatory_conditions | visible_conditions), + ), + ] + + contract_fields = ( + ContractBuilder() + .add_fields(base_fields + specific_fields) + .build_fields() + ) + return contract_fields + + @staticmethod + def contract_with_specific_outputs(base_outputs: list[ContractOutputElement]) -> list[ContractOutputElement]: + specific_outputs = [] + contract_outputs = ( + ContractBuilder() + .add_outputs(base_outputs + specific_outputs) + .build_outputs() + ) + return contract_outputs + + @staticmethod + def contract( + contract_id: str, + contract_config: ContractConfig, + contract_with_specific_fields: list[ContractElement], + contract_with_specific_outputs: list[ContractOutputElement] + ) -> Contract: + return Contract( + contract_id=contract_id, + config=contract_config, + label={ + SupportedLanguage.en: "Shodan - Custom query", + SupportedLanguage.fr: "Shodan - Requête personnalisée", + }, + fields=contract_with_specific_fields, + outputs=contract_with_specific_outputs, + manual=False, + ) diff --git a/shodan/shodan/contracts/cve_enumeration/__init__.py b/shodan/shodan/contracts/cve_enumeration/__init__.py index e69de29b..8a5deeb8 100644 --- a/shodan/shodan/contracts/cve_enumeration/__init__.py +++ b/shodan/shodan/contracts/cve_enumeration/__init__.py @@ -0,0 +1,3 @@ +from .contract import CVEEnumeration + +__all__ = ["CVEEnumeration"] \ No newline at end of file diff --git a/shodan/shodan/contracts/cve_enumeration/contract.py b/shodan/shodan/contracts/cve_enumeration/contract.py index e69de29b..ad499cb9 100644 --- a/shodan/shodan/contracts/cve_enumeration/contract.py +++ b/shodan/shodan/contracts/cve_enumeration/contract.py @@ -0,0 +1,79 @@ +from pyoaev.contracts import ContractBuilder +from pyoaev.contracts.contract_config import ( + Contract, + ContractConfig, + ContractElement, + ContractOutputElement, + ContractText, + SupportedLanguage, +) + + +class CVEEnumeration: + + @staticmethod + def contract_with_specific_fields( + base_fields: list[ContractElement], + source_selector_key:str, + target_selector_field:str, + ) -> list[ContractElement]: + + mandatory_conditions = dict( + mandatoryConditionFields=[source_selector_key], + mandatoryConditionValues={source_selector_key: target_selector_field}, + ) + + visible_conditions = dict( + visibleConditionFields=[source_selector_key], + visibleConditionValues={source_selector_key: target_selector_field}, + ) + + specific_fields = [ + ContractText( + key="hostname", + label="Hostname", + mandatory=True, + **(mandatory_conditions | visible_conditions), + ), + ContractText( + key="organization", + label="Organization", + **visible_conditions, + ), + ] + + contract_fields = ( + ContractBuilder() + .add_fields(base_fields + specific_fields) + .build_fields() + ) + return contract_fields + + @staticmethod + def contract_with_specific_outputs(base_outputs: list[ContractOutputElement]) -> list[ContractOutputElement]: + specific_outputs = [] + contract_outputs = ( + ContractBuilder() + .add_outputs(base_outputs + specific_outputs) + .build_outputs() + ) + return contract_outputs + + @staticmethod + def contract( + contract_id: str, + contract_config: ContractConfig, + contract_with_specific_fields: list[ContractElement], + contract_with_specific_outputs: list[ContractOutputElement] + ) -> Contract: + return Contract( + contract_id=contract_id, + config=contract_config, + label={ + SupportedLanguage.en: "Shodan - CVE Enumeration", + SupportedLanguage.fr: "Shodan - Énumération des CVE", + }, + fields=contract_with_specific_fields, + outputs=contract_with_specific_outputs, + manual=False, + ) diff --git a/shodan/shodan/contracts/cve_specific_watchlist/__init__.py b/shodan/shodan/contracts/cve_specific_watchlist/__init__.py index e69de29b..72e01d20 100644 --- a/shodan/shodan/contracts/cve_specific_watchlist/__init__.py +++ b/shodan/shodan/contracts/cve_specific_watchlist/__init__.py @@ -0,0 +1,3 @@ +from .contract import CVESpecificWatchlist + +__all__ = ["CVESpecificWatchlist"] \ No newline at end of file diff --git a/shodan/shodan/contracts/cve_specific_watchlist/contract.py b/shodan/shodan/contracts/cve_specific_watchlist/contract.py index e69de29b..ebf648a8 100644 --- a/shodan/shodan/contracts/cve_specific_watchlist/contract.py +++ b/shodan/shodan/contracts/cve_specific_watchlist/contract.py @@ -0,0 +1,85 @@ +from pyoaev.contracts import ContractBuilder +from pyoaev.contracts.contract_config import ( + Contract, + ContractConfig, + ContractElement, + ContractOutputElement, + ContractText, + SupportedLanguage, +) + + +class CVESpecificWatchlist: + + @staticmethod + def contract_with_specific_fields( + base_fields: list[ContractElement], + source_selector_key:str, + target_selector_field:str, + ) -> list[ContractElement]: + + mandatory_conditions = dict( + mandatoryConditionFields=[source_selector_key], + mandatoryConditionValues={source_selector_key: target_selector_field}, + ) + + visible_conditions = dict( + visibleConditionFields=[source_selector_key], + visibleConditionValues={source_selector_key: target_selector_field}, + ) + + specific_fields = [ + ContractText( + key="vulnerability", + label="Vulnerability", + mandatory=True, + **(mandatory_conditions | visible_conditions), + ), + ContractText( + key="hostname", + label="Hostname", + mandatory=True, + **(mandatory_conditions | visible_conditions), + ), + ContractText( + key="organization", + label="Organization", + **visible_conditions, + ), + ] + + contract_fields = ( + ContractBuilder() + .add_fields(base_fields + specific_fields) + .build_fields() + ) + return contract_fields + + @staticmethod + def contract_with_specific_outputs(base_outputs: list[ContractOutputElement]) -> list[ContractOutputElement]: + specific_outputs = [] + contract_outputs = ( + ContractBuilder() + .add_outputs(base_outputs + specific_outputs) + .build_outputs() + ) + return contract_outputs + + @staticmethod + def contract( + contract_id: str, + contract_config: ContractConfig, + contract_with_specific_fields: list[ContractElement], + contract_with_specific_outputs: list[ContractOutputElement] + ) -> Contract: + return Contract( + contract_id=contract_id, + config=contract_config, + label={ + SupportedLanguage.en: "Shodan - CVE specific watchlist", + SupportedLanguage.fr: "Shodan - Liste de surveillance spécifique aux CVE", + }, + fields=contract_with_specific_fields, + outputs=contract_with_specific_outputs, + manual=False, + ) diff --git a/shodan/shodan/contracts/domain_discovery/__init__.py b/shodan/shodan/contracts/domain_discovery/__init__.py index e69de29b..28fb9169 100644 --- a/shodan/shodan/contracts/domain_discovery/__init__.py +++ b/shodan/shodan/contracts/domain_discovery/__init__.py @@ -0,0 +1,3 @@ +from .contract import DomainDiscovery + +__all__ = ["DomainDiscovery"] \ No newline at end of file diff --git a/shodan/shodan/contracts/domain_discovery/contract.py b/shodan/shodan/contracts/domain_discovery/contract.py index e69de29b..08b719ff 100644 --- a/shodan/shodan/contracts/domain_discovery/contract.py +++ b/shodan/shodan/contracts/domain_discovery/contract.py @@ -0,0 +1,79 @@ +from pyoaev.contracts import ContractBuilder +from pyoaev.contracts.contract_config import ( + Contract, + ContractConfig, + ContractElement, + ContractOutputElement, + ContractText, + SupportedLanguage, +) + + +class DomainDiscovery: + + @staticmethod + def contract_with_specific_fields( + base_fields: list[ContractElement], + source_selector_key:str, + target_selector_field:str, + ) -> list[ContractElement]: + + mandatory_conditions = dict( + mandatoryConditionFields=[source_selector_key], + mandatoryConditionValues={source_selector_key: target_selector_field}, + ) + + visible_conditions = dict( + visibleConditionFields=[source_selector_key], + visibleConditionValues={source_selector_key: target_selector_field}, + ) + + specific_fields = [ + ContractText( + key="hostname", + label="Hostname", + mandatory=True, + **(mandatory_conditions | visible_conditions), + ), + ContractText( + key="organization", + label="Organization", + **visible_conditions, + ), + ] + + contract_fields = ( + ContractBuilder() + .add_fields(base_fields + specific_fields) + .build_fields() + ) + return contract_fields + + @staticmethod + def contract_with_specific_outputs(base_outputs: list[ContractOutputElement]) -> list[ContractOutputElement]: + specific_outputs = [] + contract_outputs = ( + ContractBuilder() + .add_outputs(base_outputs + specific_outputs) + .build_outputs() + ) + return contract_outputs + + @staticmethod + def contract( + contract_id: str, + contract_config: ContractConfig, + contract_with_specific_fields: list[ContractElement], + contract_with_specific_outputs: list[ContractOutputElement] + ) -> Contract: + return Contract( + contract_id=contract_id, + config=contract_config, + label={ + SupportedLanguage.en: "Shodan - Domain discovery", + SupportedLanguage.fr: "Shodan - Découverte de domaine", + }, + fields=contract_with_specific_fields, + outputs=contract_with_specific_outputs, + manual=False, + ) diff --git a/shodan/shodan/contracts/host_enumeration/__init__.py b/shodan/shodan/contracts/host_enumeration/__init__.py index e69de29b..7928d665 100644 --- a/shodan/shodan/contracts/host_enumeration/__init__.py +++ b/shodan/shodan/contracts/host_enumeration/__init__.py @@ -0,0 +1,3 @@ +from .contract import HostEnumeration + +__all__ = ["HostEnumeration"] \ No newline at end of file diff --git a/shodan/shodan/contracts/host_enumeration/contract.py b/shodan/shodan/contracts/host_enumeration/contract.py index e69de29b..76ede56d 100644 --- a/shodan/shodan/contracts/host_enumeration/contract.py +++ b/shodan/shodan/contracts/host_enumeration/contract.py @@ -0,0 +1,74 @@ +from pyoaev.contracts import ContractBuilder +from pyoaev.contracts.contract_config import ( + Contract, + ContractConfig, + ContractElement, + ContractOutputElement, + ContractText, + SupportedLanguage, +) + + +class HostEnumeration: + + @staticmethod + def contract_with_specific_fields( + base_fields: list[ContractElement], + source_selector_key:str, + target_selector_field:str, + ) -> list[ContractElement]: + + mandatory_conditions = dict( + mandatoryConditionFields=[source_selector_key], + mandatoryConditionValues={source_selector_key: target_selector_field}, + ) + + visible_conditions = dict( + visibleConditionFields=[source_selector_key], + visibleConditionValues={source_selector_key: target_selector_field}, + ) + + specific_fields = [ + ContractText( + key="host", + label="Host", + mandatory=True, + **(mandatory_conditions | visible_conditions), + ), + ] + + contract_fields = ( + ContractBuilder() + .add_fields(base_fields + specific_fields) + .build_fields() + ) + return contract_fields + + @staticmethod + def contract_with_specific_outputs(base_outputs: list[ContractOutputElement]) -> list[ContractOutputElement]: + specific_outputs = [] + contract_outputs = ( + ContractBuilder() + .add_outputs(base_outputs + specific_outputs) + .build_outputs() + ) + return contract_outputs + + @staticmethod + def contract( + contract_id: str, + contract_config: ContractConfig, + contract_with_specific_fields: list[ContractElement], + contract_with_specific_outputs: list[ContractOutputElement] + ) -> Contract: + return Contract( + contract_id=contract_id, + config=contract_config, + label={ + SupportedLanguage.en: "Shodan - Host Enumeration", + SupportedLanguage.fr: "Shodan - Énumération des hôtes", + }, + fields=contract_with_specific_fields, + outputs=contract_with_specific_outputs, + manual=False, + ) diff --git a/shodan/shodan/contracts/shodan_contracts.py b/shodan/shodan/contracts/shodan_contracts.py index 5d699e12..ca109dc9 100644 --- a/shodan/shodan/contracts/shodan_contracts.py +++ b/shodan/shodan/contracts/shodan_contracts.py @@ -1,51 +1,286 @@ -from pyoaev.contracts import ContractBuilder +from dataclasses import dataclass +from enum import Enum, StrEnum +from shodan.models import ConfigLoader from pyoaev.contracts.contract_config import ( Contract, - ContractAttachment, + ContractAsset, + ContractAssetGroup, ContractCardinality, - ContractCheckbox, - ContractConfig, ContractElement, - ContractOutputElement, - ContractOutputType, - ContractText, - ContractTextArea, - ContractTuple, + ContractConfig, + ContractCheckbox, + ContractExpectations, + ContractSelect, + Expectation, + ExpectationType, SupportedLanguage, prepare_contracts, ) -class ShodanContracts: - text_field = ContractText( - key="text", - label="Labeltext", - defaultValue="", - mandatory=True, +from shodan.contracts import ( + CloudProviderAssetDiscovery, + CriticalPortsAndExposedAdminInterface, + CustomQuery, + CVEEnumeration, + CVESpecificWatchlist, + DomainDiscovery, + HostEnumeration, +) + +TYPE = "openaev_shodan" + +class InjectorKey(StrEnum): + TARGETS_KEY="targets" + TARGET_SELECTOR_KEY="target_selector" + TARGET_PROPERTY_SELECTOR_KEY="target_property_selector" + AUTO_CREATE_ASSETS="auto_create_assets" + EXPECTATIONS_KEY="expectations" + +class ShodanContractId(StrEnum): + CLOUD_PROVIDER_ASSET_DISCOVERY="1887b988-1553-4e46-bac1-ceeee0483f3a" + CRITICAL_PORTS_AND_EXPOSED_ADMIN_INTERFACE="5349febd-ec73-408b-aa61-24a86a1ba0a7" + CUSTOM_QUERY="c7640b4c-8a12-458a-b761-6a1287501a58" + CVE_ENUMERATION="8cdccd58-78ed-4e17-be2e-7683ec611569" + CVE_SPECIFIC_WATCHLIST="462087b4-8012-4e21-9575-b9c854ef5811" + DOMAIN_DISCOVERY="faf73809-1128-4192-aa90-a08828f8ace5" + HOST_ENUMERATION="dc6b8b73-09dd-4388-b7cc-108bf16d26cd" + +@dataclass +class FieldDefinition: + name: str + target: str | list[str] + label: str + mandatory: bool = False + +class TypeOfFields(Enum): + ASSETS = FieldDefinition( + name="field_assets", + target="assets", + label="Targeted assets", ) - my_contract_fields: list[ContractElement] = ( - ContractBuilder() - .add_fields([text_field]) - .build_fields() + ASSET_GROUPS = FieldDefinition( + name="field_asset_groups", + target="asset-groups", + label="Targeted asset groups", ) - contracts = {"data": prepare_contracts([ - Contract( - contract_id="b62a0ce2-7c43-4f78-abcd-5a92413b66cf", - config=ContractConfig( - type="openaev_basetest", - label={ - SupportedLanguage.en: "Base Test", - SupportedLanguage.fr: "Base Test", - }, - color_dark="#00bcd4", - color_light="#00bcd4", - expose=True, - ), + ASSETS_PROPERTY = FieldDefinition( + name="field_assets_property", + target=["assets", "asset-groups"], + label="Targeted assets property", + ) + +@dataclass +class SelectorFieldDefinition: + key: str + label: str + +class TargetSelectorField(Enum): + ASSETS = SelectorFieldDefinition( + key="assets", + label="Assets", + ) + ASSET_GROUPS = SelectorFieldDefinition( + key="asset-groups", + label="Asset groups", + ) + MANUAL = SelectorFieldDefinition( + key="manual", + label="Manual", + ) + + @property + def key(self) -> str: + return self.value.key + + @property + def label(self) -> str: + return self.value.label + +class TargetProperty(Enum): + AUTOMATIC = "Automatic" + HOSTNAME = "Hostname" + SEEN_IP = "Seen IP" + LOCAL_IP = "Local IP (first)" + + @staticmethod + def default_value(value: str = "automatic"): + return value.lower() + +class ShodanContracts: + def __init__(self, config: ConfigLoader): + # Load configuration file + self.config = config + + # -- CONFIG -- + @staticmethod + def _base_contract_config(): + return ContractConfig( + type=TYPE, label={ - SupportedLanguage.en: "Base Test - EN", - SupportedLanguage.fr: "Base Test - FR", + SupportedLanguage.en: "Shodan", + SupportedLanguage.fr: "Shodan", }, - fields=my_contract_fields, - outputs=[], - manual=False, + color_dark="#ff5722", + color_light="#ff5722", + expose=True, ) - ])} \ No newline at end of file + + # -- BUILDER CONTRACT FIELDS -- + @staticmethod + def _build_target_selector(selector_default_value:str) -> ContractSelect: + + prefix = "only_" + + choices = { + item_selector.key: item_selector.label + for item_selector in TargetSelectorField + } + effective_default = selector_default_value + default_start_with_only = selector_default_value.startswith(prefix) + + if default_start_with_only: + effective_default = selector_default_value.removeprefix(prefix) + choices = { + effective_default: choices[effective_default] + } + + return ContractSelect( + key=InjectorKey.TARGET_SELECTOR_KEY, + label="Type of targets", + defaultValue=[effective_default], + mandatory=True, + choices=choices, + ) + + @staticmethod + def _build_field(field_type: TypeOfFields) -> ContractElement: + + builder_contract_fields_mapping = { + "field_assets": lambda **kwargs: ContractAsset( + cardinality=ContractCardinality.Multiple, + **kwargs + ), + "field_asset_groups": lambda **kwargs: ContractAssetGroup( + cardinality=ContractCardinality.Multiple, + **kwargs + ), + "field_assets_property": lambda **kwargs: ContractSelect( + key=InjectorKey.TARGET_PROPERTY_SELECTOR_KEY, + defaultValue=[TargetProperty.default_value()], + choices={ + item_property.name.lower(): item_property.value + for item_property in TargetProperty + }, + **kwargs + ), + } + + field_type_config = field_type.value + builder = builder_contract_fields_mapping[field_type_config.name] + + return builder( + label=field_type_config.label, + mandatory=field_type_config.mandatory, + mandatoryConditionFields=[InjectorKey.TARGET_SELECTOR_KEY], + mandatoryConditionValues={InjectorKey.TARGET_SELECTOR_KEY: field_type_config.target}, + visibleConditionFields=[InjectorKey.TARGET_SELECTOR_KEY], + visibleConditionValues={InjectorKey.TARGET_SELECTOR_KEY: field_type_config.target}, + ) + + @staticmethod + def _build_auto_create_assets_checkbox() -> ContractElement: + return ContractCheckbox( + key=InjectorKey.AUTO_CREATE_ASSETS, + label="Auto-Create assets", + defaultValue=False, + mandatory=False, + ) + + @staticmethod + def _build_expectations(): + return ContractExpectations( + key=InjectorKey.EXPECTATIONS_KEY, + label="Expectations", + mandatory=False, + cardinality=ContractCardinality.Multiple, + predefinedExpectations=[ + Expectation( + expectation_type=ExpectationType.vulnerability, + expectation_name="Not vulnerable", + expectation_description="", + expectation_score=100, + expectation_expectation_group=False, + ) + ], + ) + + def _base_fields(self, selector_default_value:str) -> list[ContractElement]: + + # Build Target Selector (Assets, Asset Groups, Manual) + target_selector = self._build_target_selector(selector_default_value) + + # Build all fields + target_assets = self._build_field(TypeOfFields.ASSETS) + target_asset_groups = self._build_field(TypeOfFields.ASSET_GROUPS) + target_assets_property = self._build_field(TypeOfFields.ASSETS_PROPERTY) + + # Build Checkbox Auto-Create Assets + checkbox_auto_create_assets = self._build_auto_create_assets_checkbox() + + # Build expectations + expectations = self._build_expectations() + + return [ + target_selector, + target_assets, + target_asset_groups, + target_assets_property, + checkbox_auto_create_assets, + expectations, + ] + + # -- OUTPUTS -- + @staticmethod + def _base_outputs(): + return [] + + def _build_contract( + self, + contract_id:str, + contract_cls: CloudProviderAssetDiscovery | CriticalPortsAndExposedAdminInterface | CustomQuery | + CVEEnumeration | CVESpecificWatchlist | DomainDiscovery | HostEnumeration, + contract_selector_default: str, + ) -> Contract: + return contract_cls.contract( + contract_id=contract_id, + contract_config=self._base_contract_config(), + contract_with_specific_fields=contract_cls.contract_with_specific_fields( + base_fields=self._base_fields(contract_selector_default), + source_selector_key=InjectorKey.TARGET_SELECTOR_KEY, + target_selector_field=TargetSelectorField.MANUAL.key, + ), + contract_with_specific_outputs=contract_cls.contract_with_specific_outputs( + self._base_outputs() + ), + ) + + def contracts(self): + + selector_default = TargetSelectorField.ASSET_GROUPS.key + + shodan_contract_definitions = [ + (ShodanContractId.CLOUD_PROVIDER_ASSET_DISCOVERY, CloudProviderAssetDiscovery, selector_default), + (ShodanContractId.CRITICAL_PORTS_AND_EXPOSED_ADMIN_INTERFACE, CriticalPortsAndExposedAdminInterface, selector_default), + (ShodanContractId.CUSTOM_QUERY, CustomQuery, "only_manual"), + (ShodanContractId.CVE_ENUMERATION, CVEEnumeration, selector_default), + (ShodanContractId.CVE_SPECIFIC_WATCHLIST, CVESpecificWatchlist, selector_default), + (ShodanContractId.DOMAIN_DISCOVERY, DomainDiscovery, selector_default), + (ShodanContractId.HOST_ENUMERATION, HostEnumeration, selector_default), + ] + + contracts = [ + self._build_contract(contract_id, contract_cls, contract_selector_default) + for contract_id, contract_cls, contract_selector_default in shodan_contract_definitions + ] + + return {"data": prepare_contracts(contracts)} diff --git a/shodan/shodan/injector/openaev_shodan.py b/shodan/shodan/injector/openaev_shodan.py index b9bcc1c1..b72550bb 100644 --- a/shodan/shodan/injector/openaev_shodan.py +++ b/shodan/shodan/injector/openaev_shodan.py @@ -1,7 +1,10 @@ +import time +from datetime import datetime, timezone +from shodan.contracts import InjectorKey, ShodanContractId +from shodan.services import ShodanClientAPI from shodan.models import ConfigLoader from pyoaev.helpers import OpenAEVInjectorHelper - -LOG_PREFIX = "[INJECTOR]" +LOG_PREFIX = "[SHODAN_INJECTOR]" class ShodanInjector: def __init__(self, config: ConfigLoader, helper: OpenAEVInjectorHelper): @@ -11,8 +14,77 @@ def __init__(self, config: ConfigLoader, helper: OpenAEVInjectorHelper): self.config = config self.helper = helper + def _shodan_execution(self, start, inject_id, data): + + contract_id = data["injection"]["inject_injector_contract"]["convertedContent"]["contract_id"] + contract_name = ShodanContractId(contract_id).name + + inject_content = data["injection"]["inject_content"] + selector_key = inject_content[InjectorKey.TARGET_SELECTOR_KEY] + + + if selector_key == "manual": + shodan_api = ShodanClientAPI.get_shodan_search(contract_name, inject_content) + + elif selector_key == "assert": + selector_property = inject_content[InjectorKey.TARGET_PROPERTY_SELECTOR_KEY] + + elif selector_key == "assert-groups": + selector_property = inject_content[InjectorKey.TARGET_PROPERTY_SELECTOR_KEY] + + else: + return None + + return [] + def process_message(self, data: dict) -> None: - self.helper.injector_logger.info(data) + # Initialization to get the current start utc iso format. + start_utc_isoformat = datetime.now(timezone.utc).isoformat(timespec="seconds") + self.helper.injector_logger.info( + f"{LOG_PREFIX} - Starting injector...", + { + "start_utc_isoformat": start_utc_isoformat, + }, + ) + start = time.time() + inject_id = data["injection"]["inject_id"] + + # Notify API of reception and expected number of operations + reception_data = {"tracking_total_count": 1} + self.helper.api.inject.execution_reception( + inject_id=inject_id, data=reception_data + ) + + # Execute inject + try: + result = self._shodan_execution(start, inject_id, data) + callback_data = { + # "execution_message": result["message"], + # "execution_output_structured": json.dumps(result["outputs"]), + "execution_status": "SUCCESS", + "execution_duration": int(time.time() - start), + "execution_action": "complete", + } + self.helper.api.inject.execution_callback( + inject_id=inject_id, data=callback_data + ) + + except Exception as err: + self.helper.injector_logger.error( + f"{LOG_PREFIX} - An error has occurred", {"error": str(err)} + ) + callback_data = { + "execution_message": str(err), + "execution_status": "ERROR", + "execution_duration": int(time.time() - start), + "execution_action": "complete", + } + self.helper.api.inject.execution_callback( + inject_id=inject_id, data=callback_data + ) + + + def start(self): self.helper.listen(message_callback=self.process_message) diff --git a/shodan/shodan/models/configs/config_loader.py b/shodan/shodan/models/configs/config_loader.py index 16c190f9..50e53486 100644 --- a/shodan/shodan/models/configs/config_loader.py +++ b/shodan/shodan/models/configs/config_loader.py @@ -30,7 +30,6 @@ def get_conf(self, key, default=None): class _BaseInjectorConfigUtils: def to_flatten(self, contracts=None) -> dict: - flatten_config = {} for field_name in ["openaev", "injector"]: value = getattr(self, field_name, None) diff --git a/shodan/shodan/models/configs/shodan_configs.py b/shodan/shodan/models/configs/shodan_configs.py index 36a84d55..b33d9875 100644 --- a/shodan/shodan/models/configs/shodan_configs.py +++ b/shodan/shodan/models/configs/shodan_configs.py @@ -9,8 +9,8 @@ class _ConfigLoaderShodan(_SettingsLoader): base_url: str = Field( default="https://api.shodan.io", - description="URL for the Shodan API.", + description="This is the base URL for the Shodan API.", ) api_key: SecretStr = Field( - description="API Key for the Shodan API.", + description="This is the API key for the Shodan API.", ) \ No newline at end of file diff --git a/shodan/shodan/services/__init__.py b/shodan/shodan/services/__init__.py new file mode 100644 index 00000000..73331fa7 --- /dev/null +++ b/shodan/shodan/services/__init__.py @@ -0,0 +1,5 @@ +from shodan.services.client_api import ShodanClientAPI + +__all__ = [ + "ShodanClientAPI" +] diff --git a/shodan/shodan/services/client_api.py b/shodan/shodan/services/client_api.py new file mode 100644 index 00000000..f0d30606 --- /dev/null +++ b/shodan/shodan/services/client_api.py @@ -0,0 +1,96 @@ +from aiohttp import ClientSession +from limiter import Limiter +from tenacity import retry, stop_after_attempt, wait_exponential_jitter + +from urllib.parse import urljoin, urlencode +from shodan.contracts import ShodanContractId +from shodan.models import ConfigLoader +from pyoaev.helpers import OpenAEVInjectorHelper +LOG_PREFIX = "[SHODAN_CLIENT_API]" + +class ShodanClientAPI: + def __init__(self, config: ConfigLoader, helper: OpenAEVInjectorHelper): + """Initialize the Injector with necessary configurations""" + + # Load configuration file and connection helper + self.config = config + self.helper = helper + + self.base_url = self.config.shodan.base_url + self.api_key = self.config.shodan.api_key + + # self.api_retry = self.config.shodan.api_retry + # self.api_backoff = self.config.shodan.api_backoff.total_seconds() + # + # # Limiter config + # self.rate_limiter = Limiter( + # rate=self.config.shodan.api_leaky_bucket_rate, + # capacity=self.config.shodan.api_leaky_bucket_capacity, + # bucket="shodan", + # ) + + # Define headers in session and update when needed + self.headers = { + "key": self.api_key.get_secret_value(), + "Content-Type": "application/json", + } + + def _build_url(self, endpoint: str, params: dict | None = None) -> str: + url = urljoin(self.base_url, endpoint) + + if not params: + return url + + full_url = f"{url}?{urlencode(params)}" + return full_url + + def _get_user_info(self): + pass + + async def _request_data(self, url_built): + @retry( + stop=stop_after_attempt(max_attempt_number=self.api_retry), + wait=wait_exponential_jitter( + initial=1, max=self.api_backoff, exp_base=2, jitter=1 + ), + ) + async def _retry_wrapped(): + async with ClientSession( + headers=self.headers, + raise_for_status=True, + trust_env=True, + ) as session: + async with session.get(url=url_built) as response: + return await response.json() + + async with self.rate_limiter: + return await _retry_wrapped() + + + def _get_host_enumeration(self, inject_content): + + ip = inject_content.get("host") + endpoint = f"shodan/host/{ip}" + + url_built = self._build_url(endpoint) + return self._request_data(url_built) + + + def get_shodan_search(self, contract_name, inject_content): + # Parameters available -> query / facets (optional) / page (optional) / minify (optional) + # https://api.shodan.io/shodan/host/search?key={YOUR_API_KEY}&query={query}&facets={facets} + + contract_handler = { + ShodanContractId.CVE_ENUMERATION: self._get_cve_enumeration, + ShodanContractId.CVE_SPECIFIC_WATCHLIST: self._get_cve_specific_watchlist, + ShodanContractId.CLOUD_PROVIDER_ASSET_DISCOVERY: self._get_cloud_provider_asset_discovery, + ShodanContractId.CRITICAL_PORTS_AND_EXPOSED_ADMIN_INTERFACE: self._get_critical_ports_and_exposed_admin_interface, + ShodanContractId.CUSTOM_QUERY: self._get_custom_query, + ShodanContractId.DOMAIN_DISCOVERY: self._get_domain_discovery, + ShodanContractId.HOST_ENUMERATION: self._get_host_enumeration, + } + + shodan_credit_user = self._get_user_info() + + return [] + diff --git a/shodan/shodan/services/utils.py b/shodan/shodan/services/utils.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/tests/conftest.py b/shodan/tests/conftest.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/tests/shodan_contracts/cloud_provider_asset_discovery.py b/shodan/tests/shodan_contracts/cloud_provider_asset_discovery.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/tests/shodan_contracts/critical_ports_and_exposed_admin_interface.py b/shodan/tests/shodan_contracts/critical_ports_and_exposed_admin_interface.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/tests/shodan_contracts/custom_query.py b/shodan/tests/shodan_contracts/custom_query.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/tests/shodan_contracts/cve_enumeration.py b/shodan/tests/shodan_contracts/cve_enumeration.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/tests/shodan_contracts/cve_specific_watchlist.py b/shodan/tests/shodan_contracts/cve_specific_watchlist.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/tests/shodan_contracts/domain_discovery.py b/shodan/tests/shodan_contracts/domain_discovery.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/tests/shodan_contracts/host_enumeration.py b/shodan/tests/shodan_contracts/host_enumeration.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/tests/test-requirements.txt b/shodan/tests/test-requirements.txt new file mode 100644 index 00000000..e69de29b From 06d9fb776012169574e55cf1fc4c78d0eb1c114a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20BASLER?= Date: Tue, 27 Jan 2026 16:46:51 +0100 Subject: [PATCH 03/13] [Shodan] Add client, Add Readme, Add output trace message --- shodan/CONTRIBUTING.md | 1 + shodan/README.md | 291 ++++++++ shodan/docker-compose.yml | 14 +- shodan/pyproject.toml | 6 +- shodan/shodan/__main__.py | 2 +- shodan/shodan/config.yml.sample | 13 +- .../contract.py | 121 +++- .../contract.py | 107 ++- .../shodan/contracts/custom_query/contract.py | 102 ++- .../contracts/cve_enumeration/contract.py | 106 ++- .../cve_specific_watchlist/contract.py | 107 ++- .../contracts/domain_discovery/contract.py | 106 ++- .../contracts/host_enumeration/contract.py | 102 ++- shodan/shodan/contracts/shodan_contracts.py | 9 + shodan/shodan/helpers/__init__.py | 0 .../shodan/helpers/shodan_command_builder.py | 0 shodan/shodan/helpers/shodan_output_parser.py | 0 shodan/shodan/helpers/shodan_process.py | 0 shodan/shodan/injector/__init__.py | 19 + shodan/shodan/injector/openaev_shodan.py | 121 +++- .../shodan/models/configs/shodan_configs.py | 25 +- shodan/shodan/services/__init__.py | 4 +- shodan/shodan/services/client_api.py | 365 ++++++++-- shodan/shodan/services/utils.py | 677 ++++++++++++++++++ 24 files changed, 2199 insertions(+), 99 deletions(-) delete mode 100644 shodan/shodan/helpers/__init__.py delete mode 100644 shodan/shodan/helpers/shodan_command_builder.py delete mode 100644 shodan/shodan/helpers/shodan_output_parser.py delete mode 100644 shodan/shodan/helpers/shodan_process.py diff --git a/shodan/CONTRIBUTING.md b/shodan/CONTRIBUTING.md index e69de29b..6d3f6659 100644 --- a/shodan/CONTRIBUTING.md +++ b/shodan/CONTRIBUTING.md @@ -0,0 +1 @@ +WIP \ No newline at end of file diff --git a/shodan/README.md b/shodan/README.md index e69de29b..094f8ef6 100644 --- a/shodan/README.md +++ b/shodan/README.md @@ -0,0 +1,291 @@ +# OpenAEV Shodan Injector + +## Table of Contents + +- [OpenAEV SHODAN Injector](#openaev-nuclei-injector) + - [Prerequisites](#prerequisites) + - [Configuration variables](#configuration-variables) + - [OpenAEV environment variables](#openaev-environment-variables) + - [Base injector environment variables](#base-injector-environment-variables) + - [Base Shodan environment variables](#base-shodan-environment-variables) + - [Deployment](#deployment) + - [Docker Deployment](#docker-deployment) + - [Manual Deployment](#manual-deployment) + - [Behavior](#behavior) + - [Template Selection](#template-selection) + - [Target Selection](#target-selection) + - [Resources](#resources) + +--- + +## Prerequisites + +This injector uses [Shodan](https://www.shodan.io/) to collect information about exposed assets. + +To operate correctly, the injector requires: + +- A **valid Shodan API key**, which must be provided through an environment variable (`SHODAN_API_KEY`) or via the `config.yml` file. +- You can create an account on this page: https://account.shodan.io/register + +In addition, for proper integration with the OpenAEV platform: + +- It **must be able to reach the OpenAEV platform** via the URL you provide (through `OPENAEV_URL` or `config.yml`). +- It **must be able to reach the RabbitMQ broker** used by the OpenAEV platform. + +Depending on your deployment method: + +- When using the **Docker deployment**, ensure the Shodan API key is correctly injected into the container environment. +- When running manually (e.g., in development), the Shodan API key must be available in your local environment. + +--- + +## Configuration variables + +Configuration is provided either through environment variables (Docker) or a config +file (`config.yml`, manual). + +### OpenAEV environment variables + +| Parameter | config.yml | Docker environment variable | Mandatory | Description | +|---------------|------------|-----------------------------|-----------|------------------------------------------------------| +| OpenAEV URL | url | `OPENAEV_URL` | Yes | The URL of the OpenAEV platform. | +| OpenAEV Token | token | `OPENAEV_TOKEN` | Yes | The default admin token set in the OpenAEV platform. | + +### Base injector environment variables + +| Parameter | config.yml | Docker environment variable | Default | Mandatory | Description | +|---------------|-------------|------------------------------|------------------|-----------|----------------------------------------------------------------------------------------| +| Injector ID | id | `INJECTOR_ID` | `shodan--` | No | A unique `UUIDv4` identifier for this injector instance. | +| Injector Name | name | `INJECTOR_NAME` | `Shodan` | No | Name of the injector. | +| Log Level | log_level | `INJECTOR_LOG_LEVEL` | `error` | No | Determines the verbosity of the logs. Options: `debug`, `info`, `warning`, or `error`. | + + +### Base Shodan environment variables + +| Parameter | config.yml | Docker environment variable | Default | Mandatory | Description | +|----------------------------------|---------------------------|------------------------------------|-------------------------|-----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Shodan Base URL | token | `SHODAN_BASE_URL` | `https://api.shodan.io` | No | This is the base URL for the Shodan API. | +| Shodan API Key | api_key | `SHODAN_API_KEY` | / | Yes | This is the API key for the Shodan API. | +| Shodan API leaky bucket rate | api_leaky_bucket_rate | `SHODAN_API_LEAKY_BUCKET_RATE` | `10` | No | Bucket refill rate (in tokens per second). Controls the rate at which API calls are allowed. For example, a rate of 10 means that 10 calls can be made per second, if the bucket is not empty. | +| Shodan API leaky bucket capacity | api_leaky_bucket_capacity | `SHODAN_API_LEAKY_BUCKET_CAPACITY` | `10` | No | Maximum bucket capacity (in tokens). Defines the number of calls that can be made immediately in a burst. Once the bucket is empty, it refills at the rate defined by 'api_leaky_bucket_rate'. | +| Shodan API Retry | api_retry | `SHODAN_API_RETRY` | `5` | No | Maximum number of attempts (including the initial request) in case of API failure. | +| Shodan API backoff | api_backoff | `SHODAN_API_BACKOFF` | `PT30S` | No | Maximum exponential backoff delay between retry attempts (ISO 8601 duration format). | + +--- + +## Deployment + +### Docker Deployment + +Build the Docker image using the provided `Dockerfile`: + +```bash +docker build -t openaev/injector-shodan:latest . +``` + +Edit the `docker-compose.yml` file with your OpenAEV configuration, then start the container: + +```bash +docker compose up -d +``` + +> ✅ The Docker image already includes the Shodan injector and its runtime dependencies. +Only a valid `SHODAN_API_KEY` and access to the OpenAEV platform are required at runtime. + +--- + +### Manual Deployment + +This injector can be run manually in different modes, depending on whether you want **production**, **current**, or **development/testing**. + +--- + +#### Prerequisites + +- Python 3.11+ installed +- Poetry installed (https://python-poetry.org/) / uv / pip +- A valid `SHODAN_API_KEY` set as environment variable or in `config.yml` +- Access to the OpenAEV platform (`OPENAEV_URL` / `OPENAEV_TOKEN`) and RabbitMQ + + +#### Install and Run with Pip + +Use pip if you prefer a classic Python workflow or for **development/testing with local dependencies**. + +```bash +# Create a virtual environment +python -m venv .venv + +# Activate the venv +# Linux / macOS +source .venv/bin/activate +# Windows - Git Bash +source .venv/Scripts/activate + +# Install prod dependencies (stable Pyoaev release) +pip install .[prod] + +# Install current release from Git / latest +pip install .[current] + +# Install dependencies (Dev + Test includes local client-python) +pip install -r requirements.txt +``` + +Run the injector: +The injector can be started using either the Python module or the console entry point, depending on how it was installed. + +Run as a Python module (recommended for Docker and simple setups) +```bash +python -m shodan +``` + +Run using the console entry point +```bash +# Requires the package to be installed and the virtual environment to be active +ShodanInjector +``` + +**Why use pip:** + +- Manages local modifiable dependencies (`../../client-python`) that Poetry cannot resolve automatically while complying with PEP. +- Handles extras and entry points (ShodanInjector) defined in `pyproject.toml` +- Installs development/test extras (`.[dev,test]`) in a Poetry-compatible venv. + +#### Install and Run with Poetry +Use Poetry for production or current releases, or to manage dependencies automatically in an isolated virtual environment. + +```bash +# Install prod dependencies (stable Pyoaev release) +poetry install --with prod + +# Install current release from Git / latest +poetry install --with current + +# Tips For development/testing with local Pyoaev (client-python) +poetry run pip install -r requirements.txt +``` + +Run the injector: +```bash +poetry run ShodanInjector +``` + +Run using the console entry point +```bash +# Requires the package to be installed and the virtual environment to be active +ShodanInjector +``` + +**Why use Poetry:** + +- Automatically creates and manages a virtual environment +- Handles extras and entry points (ShodanInjector) defined in `pyproject.toml` +- Keeps your environment isolated and PEP 621 compliant + +#### Commands Summary: +The table below summarizes the installation and run commands for different workflows (pip or Poetry, prod/current/dev). + +| Installation | Install Command | Run Command | Notes | +|-----------------|----------------------------------------------|-----------------------------------------|------------------------------------------------------------| +| Pip Dev/Test | `pip install -r requirements.txt` | `python -m shodan` / `ShodanInjector` | Requires venv active, handles local editable client-python | +| Poetry Prod | `poetry install --with prod` | `poetry run ShodanInjector` | Installs stable dependencies, venv managed automatically | +| Poetry Current | `poetry install --with current` | `poetry run ShodanInjector` | Installs latest release from Git | +| Poetry Dev/Test | `poetry run pip install -r requirements.txt` | `poetry run ShodanInjector` | Handles local editable client-python + dev/test extras | + +--- + +## Behavior + +For the Shodan injector, we have 7 contracts available. + +### Supported Contracts + +- Cloud Provider Asset Discovery +- Critical Ports And Exposed Admin Interface +- Custom Query +- CVE Enumeration +- CVE Specific Watchlist (The only contract that requires a plan only available to academic users, Small Business API subscribers, and higher.) +- Domain Discovery +- Host Enumeration + +### Target Selection + +Targets are selected based on the `target_selector` field. + +#### If target type is **Assets**: + +| Selected Property | Uses Asset Field | +|-------------------|----------------------------| +| Automatic | Automatic | +| Seen IP | endpoint_seen_ip | +| Local IP | First IP from endpoint_ips | +| Hostname | endpoint_hostname | + +#### If target type is **Manual**: + +Direct values separated by commas or spaces between IP addresses or hostnames are used. + +### Fields available in manual mode by contract + +- Cloud Provider Asset Discovery + - Cloud Provider (default: `Google,Microsoft,Amazon,Azure`) - mandatory + - Hostname - mandatory + - Organization - Optional (default: `hostname value is used`) + +- Critical Ports And Exposed Admin Interface + - Port (default: `20,21,22,23,25,53,80,110,111,135,139,143,443,445,993,995,1723,3306,3389,5900,8080`) - mandatory + - Hostname - mandatory + - Organization - Optional (default: `hostname value is used`) + +- Custom Query + - HTTP Method (choices: `GET, POST, PUT, DELETE`, default: `GET`) - mandatory + - Custom Query - mandatory + +- CVE Enumeration + - Hostname - mandatory + - Organization - Optional (default: `hostname value is used`) + +- CVE Specific Watchlist + - Vulnerability - mandatory + - Hostname - mandatory + - Organization - Optional (default: `hostname value is used`) + +- Domain Discovery + - Hostname - mandatory + - Organization - Optional (default: `hostname value is used`) + +- Host Enumeration + - Host - mandatory + + +### Output Parsing +The injector captures the fields filled in by the user and analyzes the JSON output of the Shodan response, +then returns several sections in the report if successful: + +- **Section Title** – Contains the title of the report and the date and time it was created. +- **Section Config** – Groups all the configuration information entered by the user for the current contract. +- **Section Info** – Makes an additional final call to the Shodan API to determine the user's remaining quota and plan. +- **Section External API** – Processes information from Shodan API responses based on the contract and fields filled in. This section also includes: + - Call Success – List of successful calls + As well as various other relevant information related to API calls + - Call Failed – List of failed calls + As well as various other relevant information related to API calls +- **Section Table** – Visually displays the details of each call, based on the configuration defined for the contract in the injector. +- **OPTIONAL** - JSON return of the response directly (In the case of a "custom query", we return the JSON directly rather than the table section.) + +### Auto-Create Assets +- Feature currently under development + +--- + +## Resources + +**Filigran** +- Homepage: https://filigran.io/ +- Repository: https://github.com/OpenAEV-Platform/injectors/tree/main/shodan +- Documentation: https://github.com/OpenAEV-Platform/injectors/tree/main/shodan/README.md +- Issues: https://github.com/OpenAEV-Platform/injectors/issues + +**Shodan** +- Homepage: https://www.shodan.io/ +- Register Page: https://account.shodan.io/register +- REST API Documentation: https://developer.shodan.io/api \ No newline at end of file diff --git a/shodan/docker-compose.yml b/shodan/docker-compose.yml index c5530f61..143ffd19 100644 --- a/shodan/docker-compose.yml +++ b/shodan/docker-compose.yml @@ -1,13 +1,19 @@ version: '3.8' services: injector-shodan: - image: openaev/injector-shodan:2.0.5 + image: openaev/injector-shodan:2.0.14 environment: - OPENAEV_URL=http://localhost - OPENAEV_TOKEN=ChangeMe - - INJECTOR_ID=ChangeMe - - INJECTOR_NAME=Shodan - - INJECTOR_LOG_LEVEL=error +# - INJECTOR_ID=shodan--a87488ad-2c72-4592-b429-69259d7bcef1 +# - INJECTOR_NAME=Shodan +# - INJECTOR_LOG_LEVEL=error + - SHODAN_API_KEY=ChangeMe +# - SHODAN_BASE_URL=https://api.shodan.io +# - SHODAN_API_LEAKY_BUCKET_RATE=10 +# - SHODAN_API_LEAKY_BUCKET_CAPACITY=10 +# - SHODAN_API_RETRY=5 +# - SHODAN_API_BACKOFF=PT30S restart: always depends_on: openaev: diff --git a/shodan/pyproject.toml b/shodan/pyproject.toml index 4abca131..8fcf7508 100644 --- a/shodan/pyproject.toml +++ b/shodan/pyproject.toml @@ -10,7 +10,7 @@ Issues = "https://github.com/OpenAEV-Platform/injectors/issues" [project] name = "openaev-shodan-injector" -version = "2.0.5" +version = "2.0.14" description = "An injector for running with Shodan" readme = "README.md" authors = [{ name = "Filigran", email = "contact@filigran.io" }] @@ -26,14 +26,14 @@ dependencies = [ "pydantic~=2.11.7", "pydantic-settings~=2.11.0", "requests~=2.32.5", - "aiohttp~=3.13.2", "limiter==0.5.0", "tenacity~=9.1.2", + "rich~=14.2.0", ] [project.optional-dependencies] prod = [ - "pyoaev ~=2.0.5" + "pyoaev ~=2.0.14" ] current = [ "pyoaev @ git+https://github.com/OpenAEV-Platform/client-python.git@release/current" diff --git a/shodan/shodan/__main__.py b/shodan/shodan/__main__.py index c4ab3b49..7abec3cd 100644 --- a/shodan/shodan/__main__.py +++ b/shodan/shodan/__main__.py @@ -26,7 +26,7 @@ def main() -> None: # Prepare config for Helper shodan_contracts = ShodanContracts(config).contracts() config_helper_adapter = config.to_config_injector_helper_adapter(contracts=shodan_contracts) - icon_bytes = Path("img/icon-shodan.png").read_bytes() + icon_bytes = Path("shodan/img/icon-shodan.png").read_bytes() helper = OpenAEVInjectorHelper(config=config_helper_adapter, icon=icon_bytes) diff --git a/shodan/shodan/config.yml.sample b/shodan/shodan/config.yml.sample index 5dcc10b8..5371ea15 100644 --- a/shodan/shodan/config.yml.sample +++ b/shodan/shodan/config.yml.sample @@ -3,9 +3,14 @@ openaev: token: 'ChangeMe' injector: - id: 'ChangeMe' - name: 'Shodan' - log_level: 'error' +# id: 'shodan--a87488ad-2c72-4592-b429-69259d7bcef1' +# name: 'Shodan' +# log_level: 'error' shodan: - api_key: 'ChangeMe' \ No newline at end of file + api_key: 'ChangeMe' +# base_url: 'https://api.shodan.io' +# api_leaky_bucket_rate: 10 +# api_leaky_bucket_capacity: 10 +# api_retry: 5 +# api_backoff: 'PT30S' \ No newline at end of file diff --git a/shodan/shodan/contracts/cloud_provider_asset_discovery/contract.py b/shodan/shodan/contracts/cloud_provider_asset_discovery/contract.py index 71bed6b1..97f890ac 100644 --- a/shodan/shodan/contracts/cloud_provider_asset_discovery/contract.py +++ b/shodan/shodan/contracts/cloud_provider_asset_discovery/contract.py @@ -5,11 +5,127 @@ ContractElement, ContractOutputElement, ContractText, + ContractTuple, SupportedLanguage, ) class CloudProviderAssetDiscovery: + @staticmethod + def output_trace_config(): + return { + "header": { + "title": "SHODAN - CLOUD PROVIDER ASSET DISCOVERY", + "subtitle": None, + }, + "sections_config": { + "header": { + "icon": "CONFIG", + "title": "[CONFIG] Summary of all configurations used for the contract.", + }, + "keys_list_to_string": ["cloud_provider"], + "keys_to_exclude": [], + }, + "sections_info": { + "header": { + "icon": "INFO", + "title": "[INFO] The Shodan information for the remaining credits and the user's plan.", + }, + "keys_list_to_string": [], + "keys_to_exclude": [], + }, + "sections_external_api": { + "header": { + "icon": "API", + "title": "[SHODAN] Call API completed", + }, + "call_success": { + "icon": "SUCCESS", + "title": "Call Success", + "count_at_path": "matches" + }, + "call_failed": { + "icon": "FAILED", + "title": "Call Failed", + }, + }, + "tables": [ + { + "header": { + "icon": "SEARCH", + "title": None, + }, + "config": { + "search_entity": None, + "columns": [ + { + "title": "Hostnames", + "path": "matches.hostnames", + "mode": "align_to_single", + }, + { + "title": "IP", + "path": "matches.ip_str", + "mode": "single", + }, + { + "title": "Port", + "path": "matches.port", + "mode": "align_to_single", + }, + { + "title": "Cloud Provider", + "path": "matches.cloud.provider", + "mode": "align_to_single", + }, + { + "title": "OS", + "path": "matches.os", + "mode": "align_to_single", + }, + { + "title": "Vulnerabilities (score)", + "path": "matches.vulns.*", + "use_key": True, + "extra": "matches.vulns.*.cvss", + "mode": "align_to_single", + }, + ], + }, + } + ], + "options": { + # "split_output": False, + "show_header": { + "is_active": True, + "show_subtitle": True, + }, + "show_sections": { + "is_active": True, + "sec_config": True, + "sec_info": True, + "sec_external_api": True, + }, + "show_tables": { + "is_active": True, + "show_lines": True, + "max_display_by_cell": 4, + "show_index": { + "is_active": False, + "index_start": 1, + }, + }, + "show_separator": { + "is_active": False, + }, + "show_json": { + "is_active": False, + "indent": 2, + "sort_keys": False, + }, + }, + } + @staticmethod def contract_with_specific_fields( base_fields: list[ContractElement], @@ -28,16 +144,15 @@ def contract_with_specific_fields( ) specific_fields = [ - ContractText( + ContractTuple( key="cloud_provider", label="Cloud Provider", - mandatory=True, + defaultValue=["Google","Microsoft","Amazon","Azure"], **(mandatory_conditions | visible_conditions), ), ContractText( key="hostname", label="Hostname", - mandatory=True, **(mandatory_conditions | visible_conditions), ), ContractText( diff --git a/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/contract.py b/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/contract.py index 05f7cb25..336ee831 100644 --- a/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/contract.py +++ b/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/contract.py @@ -12,6 +12,111 @@ class CriticalPortsAndExposedAdminInterface: + @staticmethod + def output_trace_config(): + return { + "header": { + "title": "SHODAN - CRITICAL PORTS AND EXPOSED ADMIN INTERFACE", + "subtitle": None, + }, + "sections_config": { + "header": { + "icon": "CONFIG", + "title": "[CONFIG] Summary of all configurations used for the contract.", + }, + "keys_list_to_string": ["port"], + "keys_to_exclude": [], + }, + "sections_info": { + "header": { + "icon": "INFO", + "title": "[INFO] The Shodan information for the remaining credits and the user's plan.", + }, + "keys_list_to_string": [], + "keys_to_exclude": [], + }, + "sections_external_api": { + "header": { + "icon": "API", + "title": "[SHODAN] Call API completed", + }, + "call_success": { + "icon": "SUCCESS", + "title": "Call Success", + "count_at_path": "matches" + }, + "call_failed": { + "icon": "FAILED", + "title": "Call Failed", + }, + }, + "tables": [ + { + "header": { + "icon": "SEARCH", + "title": None, + }, + "config": { + "search_entity": None, + "columns": [ + { + "title": "Port", + "path": "matches.port", + "mode": "single", + }, + { + "title": "Hostnames", + "path": "matches.hostnames", + "mode": "align_to_single", + }, + { + "title": "IP", + "path": "matches.ip_str", + "mode": "align_to_single", + }, + { + "title": "Vulnerabilities (score)", + "path": "matches.vulns.*", + "use_key": True, + "extra": "matches.vulns.*.cvss", + "mode": "align_to_single", + }, + ], + }, + } + ], + "options": { + # "split_output": False, + "show_header": { + "is_active": True, + "show_subtitle": True, + }, + "show_sections": { + "is_active": True, + "sec_config": True, + "sec_info": True, + "sec_external_api": True, + }, + "show_tables": { + "is_active": True, + "show_lines": True, + "max_display_by_cell": 4, + "show_index": { + "is_active": False, + "index_start": 1, + }, + }, + "show_separator": { + "is_active": False, + }, + "show_json": { + "is_active": False, + "indent": 2, + "sort_keys": False, + }, + }, + } + @staticmethod def contract_with_specific_fields( base_fields: list[ContractElement], @@ -33,14 +138,12 @@ def contract_with_specific_fields( ContractTuple( key="port", label="Port", - mandatory=True, defaultValue=["20","21","22","23","25","53","80","110","111","135","139","143","443","445","993","995","1723","3306","3389","5900","8080"], **(mandatory_conditions | visible_conditions), ), ContractText( key="hostname", label="Hostname", - mandatory=True, **(mandatory_conditions | visible_conditions), ), ContractText( diff --git a/shodan/shodan/contracts/custom_query/contract.py b/shodan/shodan/contracts/custom_query/contract.py index 554bd46f..119ecd5c 100644 --- a/shodan/shodan/contracts/custom_query/contract.py +++ b/shodan/shodan/contracts/custom_query/contract.py @@ -5,12 +5,93 @@ ContractElement, ContractOutputElement, ContractText, + ContractSelect, SupportedLanguage, ) class CustomQuery: + @staticmethod + def output_trace_config(): + return { + "header": { + "title": "SHODAN - CUSTOM QUERY", + "subtitle": None, + }, + "sections_config": { + "header": { + "icon": "CONFIG", + "title": "[CONFIG] Summary of all configurations used for the contract.", + }, + "keys_to_exclude": [], + }, + "sections_info": { + "header": { + "icon": "INFO", + "title": "[INFO] The Shodan information for the remaining credits and the user's plan.", + }, + "keys_to_exclude": [], + }, + "sections_external_api": { + "header": { + "icon": "API", + "title": "[SHODAN] Call API completed", + }, + "call_success": { + "icon": "SUCCESS", + "title": "Call Success", + "count_at_path": "matches" + }, + "call_failed": { + "icon": "FAILED", + "title": "Call Failed", + }, + }, + "tables": [ + { + "header": { + "icon": "SEARCH", + "title": None, + }, + "config": { + "search_entity": None, + "columns": [], + }, + } + ], + "options": { + # "split_output": False, + "show_header": { + "is_active": True, + "show_subtitle": True, + }, + "show_sections": { + "is_active": True, + "sec_config": True, + "sec_info": True, + "sec_external_api": True, + }, + "show_tables": { + "is_active": False, + "show_lines": True, + "max_display_by_cell": 4, + "show_index": { + "is_active": False, + "index_start": 1, + }, + }, + "show_separator": { + "is_active": False, + }, + "show_json": { + "is_active": True, + "indent": 2, + "sort_keys": False, + }, + }, + } + @staticmethod def contract_with_specific_fields( base_fields: list[ContractElement], @@ -29,16 +110,21 @@ def contract_with_specific_fields( ) specific_fields = [ - ContractText( - key="custom_request_overview", - label="Custom Request Overview", - readOnly=True, - **visible_conditions, + ContractSelect( + key="http_method", + label="HTTP Method", + defaultValue=["get"], + choices={ + "get":"GET", + "post":"POST", + "put":"PUT", + "delete":"DELETE", + }, + **(mandatory_conditions | visible_conditions), ), ContractText( - key="custom_request", - label="Custom Request", - mandatory=True, + key="custom_query", + label="Custom Query", **(mandatory_conditions | visible_conditions), ), ] diff --git a/shodan/shodan/contracts/cve_enumeration/contract.py b/shodan/shodan/contracts/cve_enumeration/contract.py index ad499cb9..092aa678 100644 --- a/shodan/shodan/contracts/cve_enumeration/contract.py +++ b/shodan/shodan/contracts/cve_enumeration/contract.py @@ -11,6 +11,111 @@ class CVEEnumeration: + @staticmethod + def output_trace_config(): + return { + "header": { + "title": "SHODAN - CVE ENUMERATION", + "subtitle": None, + }, + "sections_config": { + "header": { + "icon": "CONFIG", + "title": "[CONFIG] Summary of all configurations used for the contract.", + }, + "keys_list_to_string": [], + "keys_to_exclude": [], + }, + "sections_info": { + "header": { + "icon": "INFO", + "title": "[INFO] The Shodan information for the remaining credits and the user's plan.", + }, + "keys_list_to_string": [], + "keys_to_exclude": [], + }, + "sections_external_api": { + "header": { + "icon": "API", + "title": "[SHODAN] Call API completed", + }, + "call_success": { + "icon": "SUCCESS", + "title": "Call Success", + "count_at_path": "matches" + }, + "call_failed": { + "icon": "FAILED", + "title": "Call Failed", + }, + }, + "tables": [ + { + "header": { + "icon": "SEARCH", + "title": None, + }, + "config": { + "search_entity": None, + "columns": [ + { + "title": "Hostnames", + "path": "matches.hostnames", + "mode": "single", + }, + { + "title": "IP", + "path": "matches.ip_str", + "mode": "align_to_single", + }, + { + "title": "Port", + "path": "matches.port", + "mode": "align_to_single", + }, + { + "title": "Vulnerabilities (score)", + "path": "matches.vulns.*", + "use_key": True, + "extra": "matches.vulns.*.cvss", + "mode": "align_to_single", + }, + ], + }, + } + ], + "options": { + # "split_output": False, + "show_header": { + "is_active": True, + "show_subtitle": True, + }, + "show_sections": { + "is_active": True, + "sec_config": True, + "sec_info": True, + "sec_external_api": True, + }, + "show_tables": { + "is_active": True, + "show_lines": True, + "max_display_by_cell": 4, + "show_index": { + "is_active": False, + "index_start": 1, + }, + }, + "show_separator": { + "is_active": True, + }, + "show_json": { + "is_active": False, + "indent": 2, + "sort_keys": False, + }, + }, + } + @staticmethod def contract_with_specific_fields( base_fields: list[ContractElement], @@ -32,7 +137,6 @@ def contract_with_specific_fields( ContractText( key="hostname", label="Hostname", - mandatory=True, **(mandatory_conditions | visible_conditions), ), ContractText( diff --git a/shodan/shodan/contracts/cve_specific_watchlist/contract.py b/shodan/shodan/contracts/cve_specific_watchlist/contract.py index ebf648a8..71b9d4d0 100644 --- a/shodan/shodan/contracts/cve_specific_watchlist/contract.py +++ b/shodan/shodan/contracts/cve_specific_watchlist/contract.py @@ -11,6 +11,111 @@ class CVESpecificWatchlist: + @staticmethod + def output_trace_config(): + return { + "header": { + "title": "SHODAN - CVE SPECIFIC WATCH LIST", + "subtitle": None, + }, + "sections_config": { + "header": { + "icon": "CONFIG", + "title": "[CONFIG] Summary of all configurations used for the contract.", + }, + "keys_list_to_string": [], + "keys_to_exclude": [], + }, + "sections_info": { + "header": { + "icon": "INFO", + "title": "[INFO] The Injector information for the remaining credits and the user's plan.", + }, + "keys_list_to_string": [], + "keys_to_exclude": [], + }, + "sections_external_api": { + "header": { + "icon": "API", + "title": "[SHODAN] Call API completed", + }, + "call_success": { + "icon": "SUCCESS", + "title": "Call Success", + "count_at_path": "data" + }, + "call_failed": { + "icon": "FAILED", + "title": "Call Failed", + }, + }, + "tables": [ + { + "header": { + "icon": "SEARCH", + "title": None, + }, + "config": { + "search_entity": "ip_str", + "columns": [ + { + "title": "Port", + "path": "data.port", + "mode": "single", + }, + { + "title": "Hostnames", + "path": "hostnames", + "mode": "align_to_single", + }, + { + "title": "IP", + "path": "ip_str", + "mode": "align_to_single", + }, + { + "title": "Vulnerabilities (score)", + "path": "data.vulns.*", + "use_key": True, + "extra": "data.vulns.*.cvss", + "mode": "align_to_single", + }, + ], + }, + } + ], + "options": { + # "split_output": False, + "show_header": { + "is_active": True, + "show_subtitle": True, + }, + "show_sections": { + "is_active": True, + "sec_config": True, + "sec_info": True, + "sec_external_api": True, + }, + "show_tables": { + "is_active": True, + "show_lines": True, + "max_display_by_cell": 4, + "show_index": { + "is_active": False, + "index_start": 1, + }, + }, + "show_separator": { + "is_active": True, + }, + "show_json": { + "is_active": False, + "indent": 2, + "sort_keys": False, + }, + }, + } + @staticmethod def contract_with_specific_fields( base_fields: list[ContractElement], @@ -32,13 +137,11 @@ def contract_with_specific_fields( ContractText( key="vulnerability", label="Vulnerability", - mandatory=True, **(mandatory_conditions | visible_conditions), ), ContractText( key="hostname", label="Hostname", - mandatory=True, **(mandatory_conditions | visible_conditions), ), ContractText( diff --git a/shodan/shodan/contracts/domain_discovery/contract.py b/shodan/shodan/contracts/domain_discovery/contract.py index 08b719ff..141626ec 100644 --- a/shodan/shodan/contracts/domain_discovery/contract.py +++ b/shodan/shodan/contracts/domain_discovery/contract.py @@ -11,6 +11,111 @@ class DomainDiscovery: + @staticmethod + def output_trace_config(): + return { + "header": { + "title": "SHODAN - DOMAIN DISCOVERY", + "subtitle": None, + }, + "sections_config": { + "header": { + "icon": "CONFIG", + "title": "[CONFIG] Summary of all configurations used for the contract.", + }, + "keys_list_to_string": [], + "keys_to_exclude": [], + }, + "sections_info": { + "header": { + "icon": "INFO", + "title": "[INFO] The Shodan information for the remaining credits and the user's plan.", + }, + "keys_list_to_string": [], + "keys_to_exclude": [], + }, + "sections_external_api": { + "header": { + "icon": "API", + "title": "[SHODAN] Call API completed", + }, + "call_success": { + "icon": "SUCCESS", + "title": "Call Success", + "count_at_path": "matches" + }, + "call_failed": { + "icon": "FAILED", + "title": "Call Failed", + }, + }, + "tables": [ + { + "header": { + "icon": "SEARCH", + "title": None, + }, + "config": { + "search_entity": None, + "columns": [ + { + "title": "Hostnames", + "path": "matches.hostnames", + "mode": "single", + }, + { + "title": "IP", + "path": "matches.ip_str", + "mode": "align_to_single", + }, + { + "title": "Port", + "path": "matches.port", + "mode": "align_to_single", + }, + { + "title": "Vulnerabilities (score)", + "path": "matches.vulns.*", + "use_key": True, + "extra": "matches.vulns.*.cvss", + "mode": "align_to_single", + }, + ], + }, + } + ], + "options": { + # "split_output": False, + "show_header": { + "is_active": True, + "show_subtitle": True, + }, + "show_sections": { + "is_active": True, + "sec_config": True, + "sec_info": True, + "sec_external_api": True, + }, + "show_tables": { + "is_active": True, + "show_lines": True, + "max_display_by_cell": 4, + "show_index": { + "is_active": False, + "index_start": 1, + }, + }, + "show_separator": { + "is_active": True, + }, + "show_json": { + "is_active": False, + "indent": 2, + "sort_keys": False, + }, + }, + } + @staticmethod def contract_with_specific_fields( base_fields: list[ContractElement], @@ -32,7 +137,6 @@ def contract_with_specific_fields( ContractText( key="hostname", label="Hostname", - mandatory=True, **(mandatory_conditions | visible_conditions), ), ContractText( diff --git a/shodan/shodan/contracts/host_enumeration/contract.py b/shodan/shodan/contracts/host_enumeration/contract.py index 76ede56d..8921eb07 100644 --- a/shodan/shodan/contracts/host_enumeration/contract.py +++ b/shodan/shodan/contracts/host_enumeration/contract.py @@ -11,6 +11,107 @@ class HostEnumeration: + @staticmethod + def output_trace_config(): + return { + "header": { + "title": "SHODAN - HOST ENUMERATION", + "subtitle": None, + }, + "sections_config": { + "header": { + "icon": "CONFIG", + "title": "[CONFIG] Summary of all configurations used for the contract.", + }, + "keys_list_to_string": [], + "keys_to_exclude": [], + }, + "sections_info": { + "header": { + "icon": "INFO", + "title": "[INFO] The Shodan information for the remaining credits and the user's plan.", + }, + "keys_list_to_string": [], + "keys_to_exclude": [], + }, + "sections_external_api": { + "header": { + "icon": "API", + "title": "[SHODAN] Call API completed", + }, + "call_success": { + "icon": "SUCCESS", + "title": "Call Success", + "count_at_path": "data" + }, + "call_failed": { + "icon": "FAILED", + "title": "Call Failed", + }, + }, + "tables": [ + { + "header": { + "icon": "SEARCH", + "title": None, + }, + "config": { + "search_entity": "ip_str", + "columns": [ + { + "title": "Port", + "path": "data.port", + "mode": "single", + }, + { + "title": "Hostnames", + "path": "hostnames", + "mode": "repeat", + }, + { + "title": "Vulnerabilities (score)", + "path": "data.vulns.*", + "use_key": True, + "extra": "data.vulns.*.cvss", + "mode": "align_to_single", + }, + ], + }, + } + ], + "options": { + # "split_output": False, + "show_header": { + "is_active": True, + "show_subtitle": True, + }, + "show_sections": { + "is_active": True, + "sec_config": True, + "sec_info": True, + "sec_external_api": True, + }, + "show_tables": { + "is_active": True, + "show_lines": True, + "max_display_by_cell": 4, + "show_index": { + "is_active": False, + "index_start": 1, + }, + }, + "show_separator": { + "is_active": False, + }, + "show_json": { + "is_active": False, + "indent": 2, + "sort_keys": False, + }, + }, + } + + @staticmethod def contract_with_specific_fields( base_fields: list[ContractElement], @@ -32,7 +133,6 @@ def contract_with_specific_fields( ContractText( key="host", label="Host", - mandatory=True, **(mandatory_conditions | visible_conditions), ), ] diff --git a/shodan/shodan/contracts/shodan_contracts.py b/shodan/shodan/contracts/shodan_contracts.py index ca109dc9..42652150 100644 --- a/shodan/shodan/contracts/shodan_contracts.py +++ b/shodan/shodan/contracts/shodan_contracts.py @@ -10,6 +10,8 @@ ContractConfig, ContractCheckbox, ContractExpectations, + ContractOutputElement, + ContractOutputType, ContractSelect, Expectation, ExpectationType, @@ -242,6 +244,13 @@ def _base_fields(self, selector_default_value:str) -> list[ContractElement]: # -- OUTPUTS -- @staticmethod def _base_outputs(): + # output_assets = ContractOutputElement( + # type=ContractOutputType.ASSET, + # field="found_assets", + # isMultiple=True, + # isFindingCompatible=False, + # labels=["shodan"], + # ) return [] def _build_contract( diff --git a/shodan/shodan/helpers/__init__.py b/shodan/shodan/helpers/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/shodan/shodan/helpers/shodan_command_builder.py b/shodan/shodan/helpers/shodan_command_builder.py deleted file mode 100644 index e69de29b..00000000 diff --git a/shodan/shodan/helpers/shodan_output_parser.py b/shodan/shodan/helpers/shodan_output_parser.py deleted file mode 100644 index e69de29b..00000000 diff --git a/shodan/shodan/helpers/shodan_process.py b/shodan/shodan/helpers/shodan_process.py deleted file mode 100644 index e69de29b..00000000 diff --git a/shodan/shodan/injector/__init__.py b/shodan/shodan/injector/__init__.py index e69de29b..f871b0ff 100644 --- a/shodan/shodan/injector/__init__.py +++ b/shodan/shodan/injector/__init__.py @@ -0,0 +1,19 @@ +from shodan.contracts import ( + CloudProviderAssetDiscovery, + CriticalPortsAndExposedAdminInterface, + CustomQuery, + CVEEnumeration, + CVESpecificWatchlist, + DomainDiscovery, + HostEnumeration, +) + +__all__ = [ + "CloudProviderAssetDiscovery", + "CriticalPortsAndExposedAdminInterface", + "CustomQuery", + "CVEEnumeration", + "CVESpecificWatchlist", + "DomainDiscovery", + "HostEnumeration", +] \ No newline at end of file diff --git a/shodan/shodan/injector/openaev_shodan.py b/shodan/shodan/injector/openaev_shodan.py index b72550bb..b9ae92f5 100644 --- a/shodan/shodan/injector/openaev_shodan.py +++ b/shodan/shodan/injector/openaev_shodan.py @@ -1,9 +1,20 @@ import time from datetime import datetime, timezone -from shodan.contracts import InjectorKey, ShodanContractId -from shodan.services import ShodanClientAPI +from shodan.contracts import ( + CloudProviderAssetDiscovery, + CriticalPortsAndExposedAdminInterface, + CustomQuery, + CVEEnumeration, + CVESpecificWatchlist, + DomainDiscovery, + HostEnumeration, + InjectorKey, + ShodanContractId, +) +from shodan.services import ShodanClientAPI, Utils from shodan.models import ConfigLoader from pyoaev.helpers import OpenAEVInjectorHelper + LOG_PREFIX = "[SHODAN_INJECTOR]" class ShodanInjector: @@ -13,35 +24,103 @@ def __init__(self, config: ConfigLoader, helper: OpenAEVInjectorHelper): # Load configuration file and connection helper self.config = config self.helper = helper + self.shodan_client_api = ShodanClientAPI(self.config, self.helper) + self.utils = Utils() - def _shodan_execution(self, start, inject_id, data): + def _prepare_output_message(self, contract_name:str, inject_content:dict, results, user_info:dict): + self.helper.injector_logger.info( + f"{LOG_PREFIX} - Start preparing the output message rendering.", + ) + # Retrieving the contract-specific output trace configuration. + output_trace_config = { + "CLOUD_PROVIDER_ASSET_DISCOVERY": CloudProviderAssetDiscovery.output_trace_config(), + "CRITICAL_PORTS_AND_EXPOSED_ADMIN_INTERFACE": CriticalPortsAndExposedAdminInterface.output_trace_config(), + "CUSTOM_QUERY": CustomQuery.output_trace_config(), + "CVE_ENUMERATION": CVEEnumeration.output_trace_config(), + "CVE_SPECIFIC_WATCHLIST": CVESpecificWatchlist.output_trace_config(), + "DOMAIN_DISCOVERY": DomainDiscovery.output_trace_config(), + "HOST_ENUMERATION": HostEnumeration.output_trace_config(), + } + if contract_name not in output_trace_config: + raise ValueError( + f"{LOG_PREFIX} - The contract name is unknown.", + {"contract_name": contract_name} + ) + + contract_output_trace_config = output_trace_config.get(contract_name) + + # Data Sections Info + usage_limits = user_info.get("usage_limits", {}) + data_sections_info = { + "plan": user_info.get("plan"), + "scan_credits_remaining": f"{user_info.get('scan_credits')} / {usage_limits.get('scan_credits')}", + "query_credits_remaining": f"{user_info.get('query_credits')} / {usage_limits.get('query_credits')}", + } + + # Data Section External API + results_data = results.get("data") + + rendering_output_message = self.utils.generate_output_message( + output_trace_config=contract_output_trace_config, + data_sections_config=[inject_content], + data_sections_info=[data_sections_info], + data_sections_external_api=results_data, + auto_create_assets=inject_content.get("auto_create_assets", None), + ) + self.helper.injector_logger.info( + f"{LOG_PREFIX} - Finalization of the preparation of the output message rendering.", + ) + return rendering_output_message + + + def _shodan_execution(self, data): + + # Contract information contract_id = data["injection"]["inject_injector_contract"]["convertedContent"]["contract_id"] contract_name = ShodanContractId(contract_id).name inject_content = data["injection"]["inject_content"] selector_key = inject_content[InjectorKey.TARGET_SELECTOR_KEY] - + self.helper.injector_logger.info( + f"{LOG_PREFIX} - Starting the execution of Shodan...", + { + "contract_id": contract_id, + "contract_name": contract_name, + "type_of_target" : selector_key + }, + ) if selector_key == "manual": - shodan_api = ShodanClientAPI.get_shodan_search(contract_name, inject_content) + shodan_results, shodan_credit_user = self.shodan_client_api.process_shodan_search( + contract_id, inject_content + ) - elif selector_key == "assert": - selector_property = inject_content[InjectorKey.TARGET_PROPERTY_SELECTOR_KEY] + output_json = "" + output_message = self._prepare_output_message( + contract_name, inject_content, shodan_results, shodan_credit_user + ) + return output_json, output_message + + elif selector_key == "assets": + output_json = "" + output_message = "Assets - Currently not supported" + return output_json, output_message - elif selector_key == "assert-groups": - selector_property = inject_content[InjectorKey.TARGET_PROPERTY_SELECTOR_KEY] + elif selector_key == "asset-groups": + output_json = "" + output_message = "Asset-groups - Currently not supported" + return output_json, output_message else: - return None + return None, None - return [] def process_message(self, data: dict) -> None: # Initialization to get the current start utc iso format. start_utc_isoformat = datetime.now(timezone.utc).isoformat(timespec="seconds") self.helper.injector_logger.info( - f"{LOG_PREFIX} - Starting injector...", + f"{LOG_PREFIX} - Triggering of the Shodan injector...", { "start_utc_isoformat": start_utc_isoformat, }, @@ -57,17 +136,24 @@ def process_message(self, data: dict) -> None: # Execute inject try: - result = self._shodan_execution(start, inject_id, data) + output_json, output_message = self._shodan_execution(data) + execution_duration = int(time.time() - start) callback_data = { - # "execution_message": result["message"], + "execution_message": output_message, # "execution_output_structured": json.dumps(result["outputs"]), "execution_status": "SUCCESS", - "execution_duration": int(time.time() - start), + "execution_duration": execution_duration, "execution_action": "complete", } self.helper.api.inject.execution_callback( inject_id=inject_id, data=callback_data ) + self.helper.injector_logger.info( + f"{LOG_PREFIX} - The injector has completed its execution.", + { + "execution_duration": f"{execution_duration} s" + } + ) except Exception as err: self.helper.injector_logger.error( @@ -83,8 +169,7 @@ def process_message(self, data: dict) -> None: inject_id=inject_id, data=callback_data ) - - - def start(self): self.helper.listen(message_callback=self.process_message) + + diff --git a/shodan/shodan/models/configs/shodan_configs.py b/shodan/shodan/models/configs/shodan_configs.py index b33d9875..80d4cb45 100644 --- a/shodan/shodan/models/configs/shodan_configs.py +++ b/shodan/shodan/models/configs/shodan_configs.py @@ -1,6 +1,11 @@ """Configuration for Shodan injector.""" -from pydantic import Field, SecretStr +from datetime import timedelta +from pydantic import ( + Field, + SecretStr, + PositiveInt, +) from shodan.models.configs import _SettingsLoader @@ -13,4 +18,22 @@ class _ConfigLoaderShodan(_SettingsLoader): ) api_key: SecretStr = Field( description="This is the API key for the Shodan API.", + ) + api_leaky_bucket_rate: PositiveInt = Field( + default=10, + description="Bucket refill rate (in tokens per second). Controls the rate at which API calls are allowed. " + "For example, a rate of 10 means that 10 calls can be made per second, if the bucket is not empty.", + ) + api_leaky_bucket_capacity: PositiveInt = Field( + default=10, + description="Maximum bucket capacity (in tokens). Defines the number of calls that can be made immediately in a " + "burst. Once the bucket is empty, it refills at the rate defined by 'api_leaky_bucket_rate'.", + ) + api_retry: PositiveInt = Field( + default=5, + description="Maximum number of attempts (including the initial request) in case of API failure.", + ) + api_backoff: timedelta = Field( + default="PT30S", + description="Maximum exponential backoff delay between retry attempts (ISO 8601 duration format).", ) \ No newline at end of file diff --git a/shodan/shodan/services/__init__.py b/shodan/shodan/services/__init__.py index 73331fa7..53803ca8 100644 --- a/shodan/shodan/services/__init__.py +++ b/shodan/shodan/services/__init__.py @@ -1,5 +1,7 @@ from shodan.services.client_api import ShodanClientAPI +from shodan.services.utils import Utils __all__ = [ - "ShodanClientAPI" + "ShodanClientAPI", + "Utils", ] diff --git a/shodan/shodan/services/client_api.py b/shodan/shodan/services/client_api.py index f0d30606..beb646b7 100644 --- a/shodan/shodan/services/client_api.py +++ b/shodan/shodan/services/client_api.py @@ -1,13 +1,49 @@ -from aiohttp import ClientSession +import requests +import re from limiter import Limiter -from tenacity import retry, stop_after_attempt, wait_exponential_jitter - -from urllib.parse import urljoin, urlencode +from tenacity import retry, stop_after_attempt, wait_exponential_jitter, RetryError +from dataclasses import dataclass +from enum import Enum +from urllib.parse import urljoin, quote_plus from shodan.contracts import ShodanContractId from shodan.models import ConfigLoader from pyoaev.helpers import OpenAEVInjectorHelper + LOG_PREFIX = "[SHODAN_CLIENT_API]" +class MissingRequiredFieldError(ValueError): + pass + +@dataclass +class ShodanRestAPIDefinition: + http_method: str + endpoint: str + path_parameter: bool = False + +class ShodanRestAPI(Enum): + SEARCH_SHODAN = ShodanRestAPIDefinition( + http_method="GET", + endpoint="shodan/host/search", + ) + HOST_INFORMATION = ShodanRestAPIDefinition( + http_method="GET", + path_parameter=True, + endpoint="shodan/host/{target}", + ) + API_PLAN_INFORMATION = ShodanRestAPIDefinition( + http_method="GET", + endpoint="api-info" + ) + + @property + def http_method(self) -> str: + return self.value.http_method + + @property + def endpoint(self) -> str: + return self.value.endpoint + + class ShodanClientAPI: def __init__(self, config: ConfigLoader, helper: OpenAEVInjectorHelper): """Initialize the Injector with necessary configurations""" @@ -19,67 +55,284 @@ def __init__(self, config: ConfigLoader, helper: OpenAEVInjectorHelper): self.base_url = self.config.shodan.base_url self.api_key = self.config.shodan.api_key - # self.api_retry = self.config.shodan.api_retry - # self.api_backoff = self.config.shodan.api_backoff.total_seconds() - # - # # Limiter config - # self.rate_limiter = Limiter( - # rate=self.config.shodan.api_leaky_bucket_rate, - # capacity=self.config.shodan.api_leaky_bucket_capacity, - # bucket="shodan", - # ) - - # Define headers in session and update when needed - self.headers = { - "key": self.api_key.get_secret_value(), - "Content-Type": "application/json", - } + self.api_retry = self.config.shodan.api_retry + self.api_backoff = self.config.shodan.api_backoff.total_seconds() + + # Limiter config + self.rate_limiter = Limiter( + rate=self.config.shodan.api_leaky_bucket_rate, + capacity=self.config.shodan.api_leaky_bucket_capacity, + bucket="shodan", + ) - def _build_url(self, endpoint: str, params: dict | None = None) -> str: - url = urljoin(self.base_url, endpoint) - if not params: + @staticmethod + def _split_target(raw_input: str) -> list[str]: + return [target for target in re.split(r"[,\s]+", raw_input or "") if target] + + @staticmethod + def _secure_url(url: str) -> str: + if not url: return url + if "?query" in url: + return url.split("&key=")[0] + return url.split("?key=")[0] + + @staticmethod + def _build_query(query_params: dict[str,tuple[str,str]]) -> str: + query=["query="] + for key, (value, operator) in query_params.items(): + if operator == "and": + query.append(f"{key}:{value} ") + elif operator == "or": + query.append(f"{key}:{value},") + else: + query.append(f"{key}:{value}") + break + return "".join(query).strip() + + def _build_url(self, endpoint: str, query_params: str | None = None, is_custom_query:bool=False) -> str: + url = urljoin(self.base_url, endpoint) + api_key = f"key={self.api_key.get_secret_value()}" + if not query_params and "?query=" not in url: + return f"{url}?{api_key}" + if is_custom_query and "?query=" in url: + return f"{url}&{api_key}" + return f"{url}?{query_params}&{api_key}" - full_url = f"{url}?{urlencode(params)}" - return full_url def _get_user_info(self): - pass + self.helper.injector_logger.info( + f"{LOG_PREFIX} - Preparation for user quota recovery....", + ) + target_url = self._build_url(ShodanRestAPI.API_PLAN_INFORMATION.endpoint) + return self._request_data( + method=ShodanRestAPI.API_PLAN_INFORMATION.http_method, + url=target_url, + ) - async def _request_data(self, url_built): - @retry( - stop=stop_after_attempt(max_attempt_number=self.api_retry), - wait=wait_exponential_jitter( - initial=1, max=self.api_backoff, exp_base=2, jitter=1 - ), + # CONTRACT - CVE ENUMERATION + def _get_cve_enumeration(self, inject_content): + hostname = inject_content.get("hostname") + if not hostname: + return None + + filters = { + "has_vuln": ("true", "and"), + "hostname": ("{target},*.{target}", "or"), + "org": (inject_content.get("organization") or "{target}", "end"), + } + return self._process_request( + raw_input=hostname, + request_api=ShodanRestAPI.SEARCH_SHODAN, + filters_template=filters, + ) + + # CONTRACT - CVE SPECIFIC WATCHLIST + def _get_cve_specific_watchlist(self, inject_content): + hostname = inject_content.get("hostname") + if not hostname: + return None + + filters = { + "vuln": (inject_content.get("vulnerability"), "and"), + "hostname": ("{target},*.{target}", "and"), + "org": (inject_content.get("organization") or "{target}", "end"), + } + return self._process_request( + raw_input=hostname, + request_api=ShodanRestAPI.SEARCH_SHODAN, + filters_template=filters, + ) + + # CONTRACT - CLOUD PROVIDER ASSET DISCOVERY + def _get_cloud_provider_asset_discovery(self, inject_content): + hostname = inject_content.get("hostname") + if not hostname: + return None + + cloud_provider = inject_content.get("cloud_provider") + # Please note that the default values are by nature a list. + if isinstance(cloud_provider, list): + cloud_provider = ",".join(cloud_provider) + + filters = { + "cloud.provider": (cloud_provider, "and"), + "hostname": ("{target},*.{target}", "or"), + "org": (inject_content.get("organization") or "{target}", "end"), + } + return self._process_request( + raw_input=hostname, + request_api=ShodanRestAPI.SEARCH_SHODAN, + filters_template=filters, + ) + + # CONTRACT - CRITICAL PORTS AND EXPOSED ADMIN INTERFACE + def _get_critical_ports_and_exposed_admin_interface(self, inject_content): + hostname = inject_content.get("hostname") + if not hostname: + raise MissingRequiredFieldError( + f"{LOG_PREFIX} - The 'custom_request' field is required and cannot be empty." + ) + + port = inject_content.get("port") + # Please note that the default values are by nature a list. + if isinstance(port, list): + port = ",".join(port) + + filters = { + "port": (port, "and"), + "hostname": ("{target},*.{target}", "or"), + "org": (inject_content.get("organization") or "{target}", "end"), + } + return self._process_request( + raw_input=hostname, + request_api=ShodanRestAPI.SEARCH_SHODAN, + filters_template=filters, + ) + + # CONTRACT - CUSTOM QUERY + def _get_custom_query(self, inject_content): + custom_request = inject_content.get("custom_request") + http_method = inject_content.get("http_method") + if not custom_request: + raise MissingRequiredFieldError( + f"{LOG_PREFIX} - The 'Custom Request' field is required and cannot be empty." + ) + + return self._process_request( + raw_input=custom_request, + request_api=None, + filters_template=None, + is_custom_query=True, + http_method_custom_query=http_method, ) - async def _retry_wrapped(): - async with ClientSession( - headers=self.headers, - raise_for_status=True, - trust_env=True, - ) as session: - async with session.get(url=url_built) as response: - return await response.json() - async with self.rate_limiter: - return await _retry_wrapped() + # CONTRACT - DOMAIN DISCOVERY + def _get_domain_discovery(self, inject_content): + hostname = inject_content.get("hostname") + if not hostname: + raise MissingRequiredFieldError( + f"{LOG_PREFIX} - The 'hostname' field is required and cannot be empty." + ) + filters = { + "hostname": ("{target},*.{target}", "or"), + "org": (inject_content.get("organization") or "{target}", "end"), + } + return self._process_request( + raw_input=hostname, + request_api=ShodanRestAPI.SEARCH_SHODAN, + filters_template=filters, + ) + # CONTRACT - HOST ENUMERATION def _get_host_enumeration(self, inject_content): + host = inject_content.get("host") + if not host: + raise MissingRequiredFieldError( + f"{LOG_PREFIX} - The 'host' field is required and cannot be empty." + ) + return self._process_request( + raw_input=host, + request_api=ShodanRestAPI.HOST_INFORMATION, + ) + + def _process_request( + self, + raw_input:str, + request_api: ShodanRestAPI | None, + filters_template:dict=None, + is_custom_query:bool=False, + http_method_custom_query:str|None=None + ): + + if is_custom_query: + targets = [raw_input] + http_method=http_method_custom_query + endpoint_template=raw_input + path_parameter=None + else: + targets = self._split_target(raw_input) + http_method = request_api.value.http_method + endpoint_template = request_api.value.endpoint + path_parameter = request_api.value.path_parameter + + results = [] + for target in targets: + new_endpoint = endpoint_template + query_params = None + encoded_for_shodan = None + if path_parameter: + new_endpoint = endpoint_template.format(target=target) - ip = inject_content.get("host") - endpoint = f"shodan/host/{ip}" + if filters_template: + filters = {} + for key, (target_value, operator) in filters_template.items(): + filters[key] = (target_value.format(target=target), operator) - url_built = self._build_url(endpoint) - return self._request_data(url_built) + query_params = self._build_query(filters) + encoded_for_shodan = quote_plus(query_params, safe="=:,*.") + target_url = self._build_url( + endpoint=new_endpoint, + query_params=encoded_for_shodan if filters_template else query_params, + is_custom_query=is_custom_query, + ) + try: + result = self._request_data( + method=http_method, + url=target_url, + ) + results.append({ + "target": target, + "url": f"{http_method} {self._secure_url(target_url)}", + "result": result, + }) + except RetryError as retry_exc: + inner_exception = retry_exc.last_attempt.exception() - def get_shodan_search(self, contract_name, inject_content): - # Parameters available -> query / facets (optional) / page (optional) / minify (optional) - # https://api.shodan.io/shodan/host/search?key={YOUR_API_KEY}&query={query}&facets={facets} + request = inner_exception.request + request_filtered = { + "method": request.method, + "url": f"{http_method} {self._secure_url(target_url)}", + } + response = inner_exception.response + response_filtered = { + "status_code": response.status_code, + "reason": response.reason, + "error": response.text, + } + + results.append({ + "target": target, + "is_error": True, + "request": request_filtered, + "response": response_filtered, + }) + return {"targets":targets, "data": results} + + + def _request_data(self, method: str, url: str): + @retry( + stop=stop_after_attempt(max_attempt_number=self.api_retry), + wait=wait_exponential_jitter( + initial=1, max=self.api_backoff, exp_base=2, jitter=1 + ), + ) + def _retry_wrapped(): + response = requests.request(method=method, url=url) + response.raise_for_status() + return response.json() + + with self.rate_limiter: + return _retry_wrapped() + + def process_shodan_search(self, contract_id: ShodanContractId, inject_content:dict): + self.helper.injector_logger.info( + f"{LOG_PREFIX} - Starting the Shodan search process...", + ) contract_handler = { ShodanContractId.CVE_ENUMERATION: self._get_cve_enumeration, ShodanContractId.CVE_SPECIFIC_WATCHLIST: self._get_cve_specific_watchlist, @@ -90,7 +343,21 @@ def get_shodan_search(self, contract_name, inject_content): ShodanContractId.HOST_ENUMERATION: self._get_host_enumeration, } + contract = contract_handler.get(contract_id) + if not contract: + raise ValueError( + f"{LOG_PREFIX} - The contract ID is invalid.", + {"contract_id": contract_id} + ) + + results = contract(inject_content) + + # Todo Include the user_info call in the _process_request shodan_credit_user = self._get_user_info() + self.helper.injector_logger.info( + f"{LOG_PREFIX} - Finalization of the Shodan search process....", + ) + return results, shodan_credit_user + - return [] diff --git a/shodan/shodan/services/utils.py b/shodan/shodan/services/utils.py index e69de29b..573120e3 100644 --- a/shodan/shodan/services/utils.py +++ b/shodan/shodan/services/utils.py @@ -0,0 +1,677 @@ +from typing import Any +from datetime import datetime +from enum import Enum, StrEnum +import json + +from rich import box +from rich.align import Align +from rich.console import Console, Group +from rich.json import JSON +from rich.panel import Panel +from rich.table import Table +from rich.text import Text +from rich.tree import Tree +from rich.rule import Rule + +class OutputIcons(Enum): + SUCCESS = "✅" + FAILED = "❌" + SEARCH = "🔍" + CONFIG = "⚙️" + INFO = "ℹ️" + API = "🌐" + +class Utils: + + @staticmethod + def _get_trace_config(output_trace_config: dict, path: str, default = None): + keys = path.split(".") + try: + for key in keys: + output_trace_config = output_trace_config[key] + return output_trace_config + except (KeyError, TypeError): + return default + + + def _add_value_to_tree(self, tree: Tree, key: str, value: Any) -> None: + if isinstance(value, list): + node = tree.add(f"{key}:") + for item in value: + if isinstance(item, dict): + for sub_key, sub_value in item.items(): + if sub_value is None or sub_value == "": + continue + self._add_value_to_tree(node, sub_key, sub_value) + else: + node.add(str(item)) + elif isinstance(value, dict): + node = tree.add(f"{key}:") + for sub_key, sub_value in value.items(): + if sub_value is None or sub_value == "": + continue + self._add_value_to_tree(node, sub_key, sub_value) + else: + tree.add(f"{key}: {value}") + + + def _build_tree( + self, + tree: Tree, + data: list[dict[str, Any]], + keys_to_exclude: list[str], + keys_list_to_string: list[str], + ): + keys_to_exclude = keys_to_exclude or [] + for section in data: + if not isinstance(section, dict): + continue + + for key, value in section.items(): + if key in keys_to_exclude or value is None or not value: + continue + + if key in keys_list_to_string and isinstance(value, list): + parts = [] + + for item in value: + if isinstance(item, dict): + parts.extend(f"{k}:{v}" for k, v in item.items()) + else: + parts.append(str(item)) + value = ",".join(parts) + + self._add_value_to_tree(tree, key, value) + + + @staticmethod + def _get_output_icon(icon_name: str = "") -> str: + if not icon_name: + return "" + + icon = icon_name.upper() + if icon in OutputIcons.__members__: + return OutputIcons[icon].value + return "" + + + # MAKE JSON + def _make_json(self, data, output_trace_config: dict): + options_show_json_is_active = self._get_trace_config(output_trace_config, "options.show_json.is_active", False) + if not options_show_json_is_active: + return None + + options_show_json_indent = self._get_trace_config(output_trace_config, "options.show_json.indent", 2) + options_show_json_sort_keys = self._get_trace_config(output_trace_config, "options.show_json.sort_keys", False) + + if isinstance(data, (dict, list)): + return JSON.from_data( + data, + indent=options_show_json_indent, + sort_keys=options_show_json_sort_keys, + ) + + if isinstance(data, str): + return JSON.from_data(data) + + return JSON.from_data({"data": str(data)}) + + + @staticmethod + def _make_header(output_trace_config: dict): + options = output_trace_config.get("options", {}) + show_header = options.get("show_header", {}) + + if not show_header.get("is_active", True): + return None + + header = output_trace_config.get("header", {}) + title = header.get("title", "") + show_subtitle = show_header.get("show_subtitle", True) + now = datetime.now().isoformat(sep=" ", timespec="seconds") + subtitle = header.get("subtitle", now) or now + + return Panel( + renderable=Align.center(title), + padding=(1, 1), + subtitle=subtitle if show_subtitle else None, + subtitle_align="center", + ) + + + # MAKE SEPARATOR + def _make_separator(self, output_trace_config: dict) -> Rule | None: + options_show_separator = self._get_trace_config(output_trace_config, "options.show_separator.is_active", default=False) + if not options_show_separator: + return None + return Rule(characters="─") + + + # MAKE SECTION CONFIG + def _make_section_config(self, output_trace_config: dict, data_sections_config: list[dict]): + options_show_sections = self._get_trace_config(output_trace_config, "options.show_sections.is_active", default=True) + options_show_sec_config = self._get_trace_config(output_trace_config, "options.show_sections.sec_config", default=True) + + if not (options_show_sections and options_show_sec_config): + return None + + sec_config_icon = self._get_output_icon( + self._get_trace_config(output_trace_config, "sections_config.header.icon", default="CONFIG") + ) + sec_config_title = self._get_trace_config(output_trace_config, "sections_config.header.title", default="[CONFIG] Summary of all configurations used for the contract.") + keys_list_to_string = self._get_trace_config(output_trace_config, "sections_config.keys_list_to_string", default=[]) + keys_to_exclude = self._get_trace_config(output_trace_config, "sections_config.keys_to_exclude", default=[]) + + section_tree = Tree(f"{sec_config_icon} {sec_config_title}".strip(), guide_style="bold") + self._build_tree( + tree=section_tree, + data=data_sections_config, + keys_to_exclude=keys_to_exclude, + keys_list_to_string=keys_list_to_string, + ) + return section_tree + + + # MAKE SECTION INFO + def _make_section_info(self, output_trace_config: dict, data_sections_info: list[dict]): + options_show_sections = self._get_trace_config(output_trace_config, "options.show_sections.is_active", default=True) + options_show_sec_info = self._get_trace_config(output_trace_config, "options.show_sections.sec_info", default=True) + + if not (options_show_sections and options_show_sec_info): + return None + + sec_info_icon = self._get_output_icon( + self._get_trace_config(output_trace_config, "sections_info.header.icon", default="INFO") + ) + sec_info_title = self._get_trace_config(output_trace_config, "sections_info.header.title", default="[INFO] The Injector information") + keys_list_to_string = self._get_trace_config(output_trace_config, "sections_info.keys_list_to_string", default=[]) + keys_to_exclude = self._get_trace_config(output_trace_config, "sections_info.keys_to_exclude", default=[]) + + section_tree = Tree(f"{sec_info_icon} {sec_info_title}".strip(), guide_style="bold") + self._build_tree( + tree=section_tree, + data=data_sections_info, + keys_to_exclude=keys_to_exclude, + keys_list_to_string=keys_list_to_string, + ) + return section_tree + + + @staticmethod + def _count_at_path(result: dict, path: str) -> int: + current = result + for key in path.split("."): + if not isinstance(current, dict): + return 0 + current = current.get(key, None) + if current is None: + return 0 + if isinstance(current, list): + return len(current) + else: + return 1 + + + def _prepare_call_details(self, output_trace_config:dict, data_sections_external_api: list[dict]): + results_success = [] + total_success_details_count = 0 + + results_failed = [] + total_call_failed_count = 0 + for result in data_sections_external_api: + + if result.get("is_error"): + request = result.get("request") + response = result.get("response") + + raw_error = response.get("error") + if isinstance(raw_error, str) and raw_error.startswith("{"): + error_message = json.loads(raw_error).get("error") + else: + error_message = raw_error + + call_failed_details = { + "data_target": result.get("target"), + "request": request.get("url"), + "error": f"{error_message} ({response.get('status_code')} - {response.get('reason')})", + } + total_call_failed_count += 1 + results_failed.append(call_failed_details) + else: + call_success_details = { + "data_target": result.get("target"), + "request": result.get("url"), + "result": result.get("result"), + } + results_success.append(call_success_details) + count_at_path = self._get_trace_config(output_trace_config, "sections_external_api.call_success.count_at_path", default="") + if count_at_path: + total_success_details_count += self._count_at_path(call_success_details.get("result"), count_at_path) + + return (results_success, total_success_details_count), (results_failed, total_call_failed_count) + + # MAKE SECTION EXTERNAL API + def _make_section_external_api(self, output_trace_config: dict, data_sections_external_api: list[dict]): + options_show_sections = self._get_trace_config(output_trace_config, "options.show_sections.is_active", default=True) + options_show_sec_external_api = self._get_trace_config(output_trace_config, "options.show_sections.sec_external_api", default=True) + + if not (options_show_sections and options_show_sec_external_api): + return None + + sec_external_api_icon = self._get_output_icon( + self._get_trace_config(output_trace_config, "sections_external_api.header.icon", default="API") + ) + sec_external_api_title = self._get_trace_config(output_trace_config, "sections_external_api.header.title", default="[INJECTOR] Call API completed") + section_tree = Tree(f"{sec_external_api_icon} {sec_external_api_title}".strip(), guide_style="bold") + + # Add node call success + call_success_icon = self._get_output_icon( + self._get_trace_config(output_trace_config, "sections_external_api.call_success.icon", default="SUCCESS") + ) + call_success_title = self._get_trace_config(output_trace_config, "sections_external_api.call_success.title", default="Call Success") + success_node = section_tree.add(f"{call_success_icon} {call_success_title}".strip()) + + # Add node call failed + call_failed_icon = self._get_output_icon( + self._get_trace_config(output_trace_config, "sections_external_api.call_failed.icon", default="FAILED") + ) + call_failed_title = self._get_trace_config(output_trace_config, "sections_external_api.call_failed.title", default="Call Failed") + failed_node = section_tree.add(f"{call_failed_icon} {call_failed_title}".strip()) + + (results_success_details, total_success_details_count), (results_failed_details, total_call_failed_count) = self._prepare_call_details(output_trace_config, data_sections_external_api) + + # Add Total results (success and failed) + success_node.add(f"Total results: {total_success_details_count}") + failed_node.add(f"Total results: {total_call_failed_count}") + + # Add Details for call Success + success_details_node = success_node.add("Details:") + for result_detail in results_success_details: + target = result_detail.get("data_target") + result_details_count = 0 + request = result_detail.get("request") + + count_at_path = self._get_trace_config(output_trace_config, "sections_external_api.call_success.count_at_path", default="") + if count_at_path: + result_details_count += self._count_at_path(result_detail.get("result"), count_at_path) + target_node = success_details_node.add(f"• {target} → {result_details_count} results") + target_node.add(f"Request: {request}") + + # Add Details for call Failed + failed_details_node = failed_node.add("Details:") + for result_detail in results_failed_details: + target = result_detail.get("data_target") + error = result_detail.get("error") + request = result_detail.get("request") + + target_node = failed_details_node.add(f"• {target}") + target_node.add(f"Error: {error}") + target_node.add(f"Request: {request}") + + return section_tree, results_success_details + + + def _extract_level(self, items: list[Any], parts_remaining: list[str], use_key: bool = False) -> list[Any]: + if not parts_remaining: + return items + + part = parts_remaining[0] + next_parts = parts_remaining[1:] + result = [] + + for item in items: + if isinstance(item, dict): + if part == "*": + # If there is a "next_parts", we retrieve the values + if next_parts: + sub_result = list(item.values()) + else: + # if we have a “*” and use_key is true, then we retrieve the keys rather than the values + sub_result = [k if use_key else v for k, v in item.items()] + + if next_parts: + sub_result = self._extract_level(sub_result, next_parts, use_key) + + result.append(sub_result) + else: + value = item.get(part, []) + result.append(self._extract_level(value, next_parts, use_key)) + elif isinstance(item, list): + result.append(self._extract_level(item, parts_remaining, use_key)) + else: + if part == "*" and next_parts: + result.extend(self._extract_level([items[item]], next_parts, use_key)) + else: + if part in items and not next_parts and isinstance(item, str): + if item == part: + result.append(items[part]) + else: + continue + else: + result.append(item) + return result + + + def _extractor(self, data: list[dict], path: str, use_key: bool = False) -> list[list[Any]]: + if not path: + return [] + + parts = path.split(".") + data_list = data if isinstance(data, list) else [data] + final_result = self._extract_level(data_list, parts, use_key) + return final_result + + + @staticmethod + def _organizer_row(tables_config_columns, show_index_is_active, index_start, column_values, column_extras): + + single_columns = [ + values + for config, values in zip(tables_config_columns, column_values) + if config.get("mode") == "single" + ] + row_count = len(single_columns[0]) + + final_rows = [] + index = index_start + + for row_idx in range(row_count): + row_cells = [] + + if show_index_is_active: + row_cells.append(str(index)) + + for table_config_index, table_config in enumerate(tables_config_columns): + mode = table_config.get("mode", "inline") + + values = column_values[table_config_index] if table_config_index < len(column_values) else [] + extras = column_extras[table_config_index] if column_extras and table_config_index < len(column_extras) else None + + cell = "-" + if mode == "single": + if isinstance(values, list) and len(values) > 0: + for value in values: + if isinstance(value, list): + filtered = [str(v) for v in value if v not in (None, "")] + cell = ",".join(filtered) if filtered else "-" + else: + val = values[row_idx] if row_idx < len(values) else None + cell = str(val) if val not in (None, "") else "-" + + elif values not in (None, ""): + cell = str(values) + else: + cell = "-" + + elif mode == "inline": + if row_idx == 0: + all_vals = [] + for val in values: + if isinstance(val, list): + all_vals.extend(str(v) for v in val if v not in (None, "")) + elif val not in (None, ""): + all_vals.append(str(val)) + cell = ", ".join(all_vals) if all_vals else "-" + + else: + cell = "" + + elif mode == "align_to_single": + if row_idx < len(values): + val = values[row_idx] + extra = extras[row_idx] if extras and row_idx < len(extras) else None + if isinstance(val, list): + if val: + if extra: + cell = ", ".join(f"{v} ({e if e is not None else '-'})" for v, e in zip(val, extra or ["-"]*len(val))) + else: + cell = ", ".join(str(v) for v in val) + else: + cell = "-" + elif val not in (None, ""): + if extra: + cell = f"{val} ({extra})" + else: + cell = str(val) + else: + cell = "-" + else: + cell = "-" + + elif mode == "repeat": + if values: + val = values + extra = extras if extras else None + + if isinstance(val, list): + if val: + val_join = ", ".join(str(v) for v in val) + cell = val_join + if extra and isinstance(extra, list): + extra_join = ", ".join(str(v) for v in extra) + cell = f"{val_join} ({extra_join})" + else: + cell = "-" + elif val not in (None, ""): + if extra: + cell = f"{val} ({extra})" + else: + cell = str(val) + else: + cell = "-" + else: + cell = "-" + + row_cells.append(cell) + final_rows.append(row_cells) + index += 1 + + return final_rows + + @staticmethod + def _rows_with_limit_cells(rows:list[list[str]], max_display_by_cell:int | None): + rows_with_limit = [] + + for row in rows: + main_row = [] + hidden_row = [] + has_hidden = False + + for cell in row: + if isinstance(cell, str) and "," in cell and max_display_by_cell is not None: + items = [c.strip() for c in cell.split(",") if c.strip()] + if len(items) > max_display_by_cell: + shown = items[:max_display_by_cell] + hidden = len(items) - max_display_by_cell + + main_row.append(",".join(shown)) + hidden_row.append(f"...(+{hidden} hidden)") + has_hidden = True + else: + main_row.append(",".join(items)) + hidden_row.append("") + else: + main_row.append(cell) + hidden_row.append("") + + rows_with_limit.append(main_row) + if has_hidden: + rows_with_limit.append(hidden_row) + return rows_with_limit + + # MAKE TABLES + def _make_tables(self, output_trace_config: dict, data_tables: list[dict], auto_create_assets:bool | None): + + options_show_tables_is_active = self._get_trace_config(output_trace_config, "options.show_tables.is_active", default=True) + if not options_show_tables_is_active: + return [] + + show_index_is_active = self._get_trace_config(output_trace_config, "options.show_tables.show_index.is_active", default=False) + index_start = self._get_trace_config(output_trace_config, "options.show_tables.show_index.index_start", default=0) + show_lines = self._get_trace_config(output_trace_config, "options.show_tables.show_lines", default=True) + max_display_by_cell = self._get_trace_config(output_trace_config, "options.show_tables.max_display_by_cell", default=10) + + tables = self._get_trace_config(output_trace_config, "tables", default=[]) + for table_config in tables: + header_icon = self._get_output_icon( + self._get_trace_config(table_config, "header.icon", default="SEARCH") + ) + header_title = self._get_trace_config(table_config, "header.title", default="") + search_entity = self._get_trace_config(table_config, "config.search_entity", default=None) + tables_final = [] + for data_table in data_tables: + result = data_table.get("result") + if header_title: + table_title = header_title.format(**result) + else: + table_title = ( + f"Asset(s) " + f"{'Created' if auto_create_assets else 'Not Created'} " + f"for {result.get(search_entity) or data_table.get("data_target")}" + ) + + table = Table( + title=f"{header_icon} {table_title}".strip(), + title_justify="left", + show_lines=show_lines, + box=box.HEAVY_HEAD if show_lines else None, + expand=True, + ) + + tables_config_columns = self._get_trace_config(table_config, "config.columns") + + if show_index_is_active: + table.add_column("#") + + column_values = [] + column_extras = [] + for config in tables_config_columns: + title = config.get("title", "-") + table.add_column(title, overflow="fold") + + path = config.get("path") + use_key = config.get("use_key", False) + values = self._extractor(result, path, use_key) + column_values.append(values[0]) + + extra_path = config.get("extra") + if extra_path: + extras = self._extractor(result, extra_path) + column_extras.append(extras[0]) + else: + column_extras.append(None) + + final_rows = self._organizer_row( + tables_config_columns, + show_index_is_active, + index_start, + column_values, + column_extras, + ) + + rows_with_limit_cell = self._rows_with_limit_cells(final_rows, max_display_by_cell) + + for row_with_limit in rows_with_limit_cell: + table.add_row(*row_with_limit) + + tables_final.append(table) + + return tables_final + return [] + + + def generate_output_message( + self, + output_trace_config: dict, + data_sections_config: list[dict], + data_sections_info: list[dict], + data_sections_external_api: list[dict], + auto_create_assets: bool | None, + ): + # Todo: Make split mode + + renderables = [] + + # Output Header + output_header = self._make_header(output_trace_config=output_trace_config) + if output_header: + renderables.append(Text("")) + separator = self._make_separator(output_trace_config=output_trace_config) + if separator: + renderables.append(separator) + renderables.append(output_header) + + # Output Sections Config + output_sections_config = self._make_section_config( + output_trace_config=output_trace_config, + data_sections_config=data_sections_config, + ) + if output_sections_config: + renderables.append(Text("")) + separator = self._make_separator(output_trace_config=output_trace_config) + if separator: + renderables.append(separator) + renderables.append(output_sections_config) + + # Output Sections Info + output_sections_info = self._make_section_info( + output_trace_config=output_trace_config, + data_sections_info=data_sections_info, + ) + if output_sections_info: + renderables.append(Text("")) + separator = self._make_separator(output_trace_config=output_trace_config) + if separator: + renderables.append(separator) + renderables.append(output_sections_info) + + # Output Sections Client API + output_sections_client_api, results_success_details = self._make_section_external_api( + output_trace_config=output_trace_config, + data_sections_external_api=data_sections_external_api, + ) + if output_sections_client_api: + renderables.append(Text("")) + separator = self._make_separator(output_trace_config=output_trace_config) + if separator: + renderables.append(separator) + renderables.append(output_sections_client_api) + + # Output Tables + output_tables = self._make_tables( + output_trace_config=output_trace_config, + auto_create_assets=auto_create_assets, + data_tables=results_success_details, + ) + for output_table in output_tables: + if output_table: + renderables.append(Text("")) + separator = self._make_separator(output_trace_config=output_trace_config) + if separator: + renderables.append(separator) + renderables.append(output_table) + + + # Output JSON + output_json = self._make_json( + data=data_sections_external_api, + output_trace_config=output_trace_config, + ) + + if output_json: + renderables.append(Text("")) + separator = self._make_separator(output_trace_config=output_trace_config) + if separator: + renderables.append(separator) + renderables.append(output_json) + + group = Group(*renderables) + console = Console( + color_system=None, + force_terminal=False, + width=150, + ) + with console.capture() as capture: + console.print(group) + output_str = capture.get() + return output_str From 848f8ba872d220063848bbacd91ae7cc8a416deb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20BASLER?= Date: Tue, 27 Jan 2026 17:11:39 +0100 Subject: [PATCH 04/13] [Shodan] Apply black and isort formatting --- shodan/shodan/__init__.py | 2 +- shodan/shodan/__main__.py | 14 +- shodan/shodan/contracts/__init__.py | 8 +- .../__init__.py | 2 +- .../contract.py | 29 +- .../__init__.py | 2 +- .../contract.py | 50 ++- .../shodan/contracts/custom_query/__init__.py | 2 +- .../shodan/contracts/custom_query/contract.py | 34 +- .../contracts/cve_enumeration/__init__.py | 2 +- .../contracts/cve_enumeration/contract.py | 24 +- .../cve_specific_watchlist/__init__.py | 2 +- .../cve_specific_watchlist/contract.py | 24 +- .../contracts/domain_discovery/__init__.py | 2 +- .../contracts/domain_discovery/contract.py | 24 +- .../contracts/host_enumeration/__init__.py | 2 +- .../contracts/host_enumeration/contract.py | 25 +- shodan/shodan/contracts/shodan_contracts.py | 96 +++-- shodan/shodan/injector/__init__.py | 2 +- shodan/shodan/injector/openaev_shodan.py | 34 +- shodan/shodan/models/__init__.py | 2 +- shodan/shodan/models/configs/config_loader.py | 27 +- .../shodan/models/configs/injector_configs.py | 14 +- .../shodan/models/configs/shodan_configs.py | 10 +- shodan/shodan/services/client_api.py | 89 +++-- shodan/shodan/services/utils.py | 348 +++++++++++++----- 26 files changed, 553 insertions(+), 317 deletions(-) diff --git a/shodan/shodan/__init__.py b/shodan/shodan/__init__.py index e1d414cc..ccd9759b 100644 --- a/shodan/shodan/__init__.py +++ b/shodan/shodan/__init__.py @@ -1,3 +1,3 @@ from shodan.models import ConfigLoader -__all__ = ["ConfigLoader"] \ No newline at end of file +__all__ = ["ConfigLoader"] diff --git a/shodan/shodan/__main__.py b/shodan/shodan/__main__.py index 7abec3cd..599acd73 100644 --- a/shodan/shodan/__main__.py +++ b/shodan/shodan/__main__.py @@ -4,15 +4,19 @@ import os import sys from pathlib import Path + from pydantic import ValidationError from pyoaev.helpers import OpenAEVInjectorHelper -from shodan.models import ConfigLoader -from shodan.injector.openaev_shodan import ShodanInjector + from shodan.contracts.shodan_contracts import ShodanContracts +from shodan.injector.openaev_shodan import ShodanInjector +from shodan.models import ConfigLoader + # from shodan.injector.exception import InjectorConfigError LOG_PREFIX = "[SHODAN_MAIN]" + def main() -> None: """Define the main function to run the injector.""" logger = logging.getLogger(__name__) @@ -25,7 +29,9 @@ def main() -> None: # Prepare config for Helper shodan_contracts = ShodanContracts(config).contracts() - config_helper_adapter = config.to_config_injector_helper_adapter(contracts=shodan_contracts) + config_helper_adapter = config.to_config_injector_helper_adapter( + contracts=shodan_contracts + ) icon_bytes = Path("shodan/img/icon-shodan.png").read_bytes() helper = OpenAEVInjectorHelper(config=config_helper_adapter, icon=icon_bytes) @@ -51,4 +57,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/shodan/shodan/contracts/__init__.py b/shodan/shodan/contracts/__init__.py index ab18d29d..a7aa439c 100644 --- a/shodan/shodan/contracts/__init__.py +++ b/shodan/shodan/contracts/__init__.py @@ -1,5 +1,7 @@ from .cloud_provider_asset_discovery import CloudProviderAssetDiscovery -from .critical_ports_and_exposed_admin_interface import CriticalPortsAndExposedAdminInterface +from .critical_ports_and_exposed_admin_interface import ( + CriticalPortsAndExposedAdminInterface, +) from .custom_query import CustomQuery from .cve_enumeration import CVEEnumeration from .cve_specific_watchlist import CVESpecificWatchlist @@ -16,5 +18,5 @@ "DomainDiscovery", "HostEnumeration", "InjectorKey", - "ShodanContractId" -] \ No newline at end of file + "ShodanContractId", +] diff --git a/shodan/shodan/contracts/cloud_provider_asset_discovery/__init__.py b/shodan/shodan/contracts/cloud_provider_asset_discovery/__init__.py index 4d814cb8..8f0dd5b9 100644 --- a/shodan/shodan/contracts/cloud_provider_asset_discovery/__init__.py +++ b/shodan/shodan/contracts/cloud_provider_asset_discovery/__init__.py @@ -1,3 +1,3 @@ from .contract import CloudProviderAssetDiscovery -__all__ = ["CloudProviderAssetDiscovery"] \ No newline at end of file +__all__ = ["CloudProviderAssetDiscovery"] diff --git a/shodan/shodan/contracts/cloud_provider_asset_discovery/contract.py b/shodan/shodan/contracts/cloud_provider_asset_discovery/contract.py index 97f890ac..f5419487 100644 --- a/shodan/shodan/contracts/cloud_provider_asset_discovery/contract.py +++ b/shodan/shodan/contracts/cloud_provider_asset_discovery/contract.py @@ -9,6 +9,7 @@ SupportedLanguage, ) + class CloudProviderAssetDiscovery: @staticmethod @@ -42,7 +43,7 @@ def output_trace_config(): "call_success": { "icon": "SUCCESS", "title": "Call Success", - "count_at_path": "matches" + "count_at_path": "matches", }, "call_failed": { "icon": "FAILED", @@ -128,9 +129,9 @@ def output_trace_config(): @staticmethod def contract_with_specific_fields( - base_fields: list[ContractElement], - source_selector_key:str, - target_selector_field:str, + base_fields: list[ContractElement], + source_selector_key: str, + target_selector_field: str, ) -> list[ContractElement]: mandatory_conditions = dict( @@ -147,7 +148,7 @@ def contract_with_specific_fields( ContractTuple( key="cloud_provider", label="Cloud Provider", - defaultValue=["Google","Microsoft","Amazon","Azure"], + defaultValue=["Google", "Microsoft", "Amazon", "Azure"], **(mandatory_conditions | visible_conditions), ), ContractText( @@ -163,14 +164,14 @@ def contract_with_specific_fields( ] contract_fields = ( - ContractBuilder() - .add_fields(base_fields + specific_fields) - .build_fields() + ContractBuilder().add_fields(base_fields + specific_fields).build_fields() ) return contract_fields @staticmethod - def contract_with_specific_outputs(base_outputs: list[ContractOutputElement]) -> list[ContractOutputElement]: + def contract_with_specific_outputs( + base_outputs: list[ContractOutputElement], + ) -> list[ContractOutputElement]: specific_outputs = [] contract_outputs = ( ContractBuilder() @@ -181,10 +182,10 @@ def contract_with_specific_outputs(base_outputs: list[ContractOutputElement]) -> @staticmethod def contract( - contract_id: str, - contract_config: ContractConfig, - contract_with_specific_fields: list[ContractElement], - contract_with_specific_outputs: list[ContractOutputElement] + contract_id: str, + contract_config: ContractConfig, + contract_with_specific_fields: list[ContractElement], + contract_with_specific_outputs: list[ContractOutputElement], ) -> Contract: return Contract( contract_id=contract_id, @@ -196,4 +197,4 @@ def contract( fields=contract_with_specific_fields, outputs=contract_with_specific_outputs, manual=False, - ) \ No newline at end of file + ) diff --git a/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/__init__.py b/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/__init__.py index 71548d99..d01e15ca 100644 --- a/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/__init__.py +++ b/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/__init__.py @@ -1,3 +1,3 @@ from .contract import CriticalPortsAndExposedAdminInterface -__all__ = ["CriticalPortsAndExposedAdminInterface"] \ No newline at end of file +__all__ = ["CriticalPortsAndExposedAdminInterface"] diff --git a/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/contract.py b/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/contract.py index 336ee831..93e821aa 100644 --- a/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/contract.py +++ b/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/contract.py @@ -43,7 +43,7 @@ def output_trace_config(): "call_success": { "icon": "SUCCESS", "title": "Call Success", - "count_at_path": "matches" + "count_at_path": "matches", }, "call_failed": { "icon": "FAILED", @@ -119,9 +119,9 @@ def output_trace_config(): @staticmethod def contract_with_specific_fields( - base_fields: list[ContractElement], - source_selector_key:str, - target_selector_field:str + base_fields: list[ContractElement], + source_selector_key: str, + target_selector_field: str, ) -> list[ContractElement]: mandatory_conditions = dict( @@ -138,7 +138,29 @@ def contract_with_specific_fields( ContractTuple( key="port", label="Port", - defaultValue=["20","21","22","23","25","53","80","110","111","135","139","143","443","445","993","995","1723","3306","3389","5900","8080"], + defaultValue=[ + "20", + "21", + "22", + "23", + "25", + "53", + "80", + "110", + "111", + "135", + "139", + "143", + "443", + "445", + "993", + "995", + "1723", + "3306", + "3389", + "5900", + "8080", + ], **(mandatory_conditions | visible_conditions), ), ContractText( @@ -154,14 +176,14 @@ def contract_with_specific_fields( ] contract_fields = ( - ContractBuilder() - .add_fields(base_fields + specific_fields) - .build_fields() + ContractBuilder().add_fields(base_fields + specific_fields).build_fields() ) return contract_fields @staticmethod - def contract_with_specific_outputs(base_outputs: list[ContractOutputElement]) -> list[ContractOutputElement]: + def contract_with_specific_outputs( + base_outputs: list[ContractOutputElement], + ) -> list[ContractOutputElement]: specific_outputs = [] contract_outputs = ( ContractBuilder() @@ -172,10 +194,10 @@ def contract_with_specific_outputs(base_outputs: list[ContractOutputElement]) -> @staticmethod def contract( - contract_id: str, - contract_config: ContractConfig, - contract_with_specific_fields: list[ContractElement], - contract_with_specific_outputs: list[ContractOutputElement] + contract_id: str, + contract_config: ContractConfig, + contract_with_specific_fields: list[ContractElement], + contract_with_specific_outputs: list[ContractOutputElement], ) -> Contract: return Contract( contract_id=contract_id, @@ -187,4 +209,4 @@ def contract( fields=contract_with_specific_fields, outputs=contract_with_specific_outputs, manual=False, - ) \ No newline at end of file + ) diff --git a/shodan/shodan/contracts/custom_query/__init__.py b/shodan/shodan/contracts/custom_query/__init__.py index 9e40dd73..dd608594 100644 --- a/shodan/shodan/contracts/custom_query/__init__.py +++ b/shodan/shodan/contracts/custom_query/__init__.py @@ -1,3 +1,3 @@ from .contract import CustomQuery -__all__ = ["CustomQuery"] \ No newline at end of file +__all__ = ["CustomQuery"] diff --git a/shodan/shodan/contracts/custom_query/contract.py b/shodan/shodan/contracts/custom_query/contract.py index 119ecd5c..c18cf815 100644 --- a/shodan/shodan/contracts/custom_query/contract.py +++ b/shodan/shodan/contracts/custom_query/contract.py @@ -4,8 +4,8 @@ ContractConfig, ContractElement, ContractOutputElement, - ContractText, ContractSelect, + ContractText, SupportedLanguage, ) @@ -41,7 +41,7 @@ def output_trace_config(): "call_success": { "icon": "SUCCESS", "title": "Call Success", - "count_at_path": "matches" + "count_at_path": "matches", }, "call_failed": { "icon": "FAILED", @@ -94,9 +94,9 @@ def output_trace_config(): @staticmethod def contract_with_specific_fields( - base_fields: list[ContractElement], - source_selector_key:str, - target_selector_field:str, + base_fields: list[ContractElement], + source_selector_key: str, + target_selector_field: str, ) -> list[ContractElement]: mandatory_conditions = dict( @@ -115,10 +115,10 @@ def contract_with_specific_fields( label="HTTP Method", defaultValue=["get"], choices={ - "get":"GET", - "post":"POST", - "put":"PUT", - "delete":"DELETE", + "get": "GET", + "post": "POST", + "put": "PUT", + "delete": "DELETE", }, **(mandatory_conditions | visible_conditions), ), @@ -130,14 +130,14 @@ def contract_with_specific_fields( ] contract_fields = ( - ContractBuilder() - .add_fields(base_fields + specific_fields) - .build_fields() + ContractBuilder().add_fields(base_fields + specific_fields).build_fields() ) return contract_fields @staticmethod - def contract_with_specific_outputs(base_outputs: list[ContractOutputElement]) -> list[ContractOutputElement]: + def contract_with_specific_outputs( + base_outputs: list[ContractOutputElement], + ) -> list[ContractOutputElement]: specific_outputs = [] contract_outputs = ( ContractBuilder() @@ -148,10 +148,10 @@ def contract_with_specific_outputs(base_outputs: list[ContractOutputElement]) -> @staticmethod def contract( - contract_id: str, - contract_config: ContractConfig, - contract_with_specific_fields: list[ContractElement], - contract_with_specific_outputs: list[ContractOutputElement] + contract_id: str, + contract_config: ContractConfig, + contract_with_specific_fields: list[ContractElement], + contract_with_specific_outputs: list[ContractOutputElement], ) -> Contract: return Contract( contract_id=contract_id, diff --git a/shodan/shodan/contracts/cve_enumeration/__init__.py b/shodan/shodan/contracts/cve_enumeration/__init__.py index 8a5deeb8..4f7378e1 100644 --- a/shodan/shodan/contracts/cve_enumeration/__init__.py +++ b/shodan/shodan/contracts/cve_enumeration/__init__.py @@ -1,3 +1,3 @@ from .contract import CVEEnumeration -__all__ = ["CVEEnumeration"] \ No newline at end of file +__all__ = ["CVEEnumeration"] diff --git a/shodan/shodan/contracts/cve_enumeration/contract.py b/shodan/shodan/contracts/cve_enumeration/contract.py index 092aa678..ece5f12b 100644 --- a/shodan/shodan/contracts/cve_enumeration/contract.py +++ b/shodan/shodan/contracts/cve_enumeration/contract.py @@ -42,7 +42,7 @@ def output_trace_config(): "call_success": { "icon": "SUCCESS", "title": "Call Success", - "count_at_path": "matches" + "count_at_path": "matches", }, "call_failed": { "icon": "FAILED", @@ -118,9 +118,9 @@ def output_trace_config(): @staticmethod def contract_with_specific_fields( - base_fields: list[ContractElement], - source_selector_key:str, - target_selector_field:str, + base_fields: list[ContractElement], + source_selector_key: str, + target_selector_field: str, ) -> list[ContractElement]: mandatory_conditions = dict( @@ -147,14 +147,14 @@ def contract_with_specific_fields( ] contract_fields = ( - ContractBuilder() - .add_fields(base_fields + specific_fields) - .build_fields() + ContractBuilder().add_fields(base_fields + specific_fields).build_fields() ) return contract_fields @staticmethod - def contract_with_specific_outputs(base_outputs: list[ContractOutputElement]) -> list[ContractOutputElement]: + def contract_with_specific_outputs( + base_outputs: list[ContractOutputElement], + ) -> list[ContractOutputElement]: specific_outputs = [] contract_outputs = ( ContractBuilder() @@ -165,10 +165,10 @@ def contract_with_specific_outputs(base_outputs: list[ContractOutputElement]) -> @staticmethod def contract( - contract_id: str, - contract_config: ContractConfig, - contract_with_specific_fields: list[ContractElement], - contract_with_specific_outputs: list[ContractOutputElement] + contract_id: str, + contract_config: ContractConfig, + contract_with_specific_fields: list[ContractElement], + contract_with_specific_outputs: list[ContractOutputElement], ) -> Contract: return Contract( contract_id=contract_id, diff --git a/shodan/shodan/contracts/cve_specific_watchlist/__init__.py b/shodan/shodan/contracts/cve_specific_watchlist/__init__.py index 72e01d20..8808845c 100644 --- a/shodan/shodan/contracts/cve_specific_watchlist/__init__.py +++ b/shodan/shodan/contracts/cve_specific_watchlist/__init__.py @@ -1,3 +1,3 @@ from .contract import CVESpecificWatchlist -__all__ = ["CVESpecificWatchlist"] \ No newline at end of file +__all__ = ["CVESpecificWatchlist"] diff --git a/shodan/shodan/contracts/cve_specific_watchlist/contract.py b/shodan/shodan/contracts/cve_specific_watchlist/contract.py index 71b9d4d0..991bd69c 100644 --- a/shodan/shodan/contracts/cve_specific_watchlist/contract.py +++ b/shodan/shodan/contracts/cve_specific_watchlist/contract.py @@ -42,7 +42,7 @@ def output_trace_config(): "call_success": { "icon": "SUCCESS", "title": "Call Success", - "count_at_path": "data" + "count_at_path": "data", }, "call_failed": { "icon": "FAILED", @@ -118,9 +118,9 @@ def output_trace_config(): @staticmethod def contract_with_specific_fields( - base_fields: list[ContractElement], - source_selector_key:str, - target_selector_field:str, + base_fields: list[ContractElement], + source_selector_key: str, + target_selector_field: str, ) -> list[ContractElement]: mandatory_conditions = dict( @@ -152,14 +152,14 @@ def contract_with_specific_fields( ] contract_fields = ( - ContractBuilder() - .add_fields(base_fields + specific_fields) - .build_fields() + ContractBuilder().add_fields(base_fields + specific_fields).build_fields() ) return contract_fields @staticmethod - def contract_with_specific_outputs(base_outputs: list[ContractOutputElement]) -> list[ContractOutputElement]: + def contract_with_specific_outputs( + base_outputs: list[ContractOutputElement], + ) -> list[ContractOutputElement]: specific_outputs = [] contract_outputs = ( ContractBuilder() @@ -170,10 +170,10 @@ def contract_with_specific_outputs(base_outputs: list[ContractOutputElement]) -> @staticmethod def contract( - contract_id: str, - contract_config: ContractConfig, - contract_with_specific_fields: list[ContractElement], - contract_with_specific_outputs: list[ContractOutputElement] + contract_id: str, + contract_config: ContractConfig, + contract_with_specific_fields: list[ContractElement], + contract_with_specific_outputs: list[ContractOutputElement], ) -> Contract: return Contract( contract_id=contract_id, diff --git a/shodan/shodan/contracts/domain_discovery/__init__.py b/shodan/shodan/contracts/domain_discovery/__init__.py index 28fb9169..0b42e416 100644 --- a/shodan/shodan/contracts/domain_discovery/__init__.py +++ b/shodan/shodan/contracts/domain_discovery/__init__.py @@ -1,3 +1,3 @@ from .contract import DomainDiscovery -__all__ = ["DomainDiscovery"] \ No newline at end of file +__all__ = ["DomainDiscovery"] diff --git a/shodan/shodan/contracts/domain_discovery/contract.py b/shodan/shodan/contracts/domain_discovery/contract.py index 141626ec..939b86de 100644 --- a/shodan/shodan/contracts/domain_discovery/contract.py +++ b/shodan/shodan/contracts/domain_discovery/contract.py @@ -42,7 +42,7 @@ def output_trace_config(): "call_success": { "icon": "SUCCESS", "title": "Call Success", - "count_at_path": "matches" + "count_at_path": "matches", }, "call_failed": { "icon": "FAILED", @@ -118,9 +118,9 @@ def output_trace_config(): @staticmethod def contract_with_specific_fields( - base_fields: list[ContractElement], - source_selector_key:str, - target_selector_field:str, + base_fields: list[ContractElement], + source_selector_key: str, + target_selector_field: str, ) -> list[ContractElement]: mandatory_conditions = dict( @@ -147,14 +147,14 @@ def contract_with_specific_fields( ] contract_fields = ( - ContractBuilder() - .add_fields(base_fields + specific_fields) - .build_fields() + ContractBuilder().add_fields(base_fields + specific_fields).build_fields() ) return contract_fields @staticmethod - def contract_with_specific_outputs(base_outputs: list[ContractOutputElement]) -> list[ContractOutputElement]: + def contract_with_specific_outputs( + base_outputs: list[ContractOutputElement], + ) -> list[ContractOutputElement]: specific_outputs = [] contract_outputs = ( ContractBuilder() @@ -165,10 +165,10 @@ def contract_with_specific_outputs(base_outputs: list[ContractOutputElement]) -> @staticmethod def contract( - contract_id: str, - contract_config: ContractConfig, - contract_with_specific_fields: list[ContractElement], - contract_with_specific_outputs: list[ContractOutputElement] + contract_id: str, + contract_config: ContractConfig, + contract_with_specific_fields: list[ContractElement], + contract_with_specific_outputs: list[ContractOutputElement], ) -> Contract: return Contract( contract_id=contract_id, diff --git a/shodan/shodan/contracts/host_enumeration/__init__.py b/shodan/shodan/contracts/host_enumeration/__init__.py index 7928d665..b0dc1c90 100644 --- a/shodan/shodan/contracts/host_enumeration/__init__.py +++ b/shodan/shodan/contracts/host_enumeration/__init__.py @@ -1,3 +1,3 @@ from .contract import HostEnumeration -__all__ = ["HostEnumeration"] \ No newline at end of file +__all__ = ["HostEnumeration"] diff --git a/shodan/shodan/contracts/host_enumeration/contract.py b/shodan/shodan/contracts/host_enumeration/contract.py index 8921eb07..0e09ccf6 100644 --- a/shodan/shodan/contracts/host_enumeration/contract.py +++ b/shodan/shodan/contracts/host_enumeration/contract.py @@ -42,7 +42,7 @@ def output_trace_config(): "call_success": { "icon": "SUCCESS", "title": "Call Success", - "count_at_path": "data" + "count_at_path": "data", }, "call_failed": { "icon": "FAILED", @@ -111,12 +111,11 @@ def output_trace_config(): }, } - @staticmethod def contract_with_specific_fields( - base_fields: list[ContractElement], - source_selector_key:str, - target_selector_field:str, + base_fields: list[ContractElement], + source_selector_key: str, + target_selector_field: str, ) -> list[ContractElement]: mandatory_conditions = dict( @@ -138,14 +137,14 @@ def contract_with_specific_fields( ] contract_fields = ( - ContractBuilder() - .add_fields(base_fields + specific_fields) - .build_fields() + ContractBuilder().add_fields(base_fields + specific_fields).build_fields() ) return contract_fields @staticmethod - def contract_with_specific_outputs(base_outputs: list[ContractOutputElement]) -> list[ContractOutputElement]: + def contract_with_specific_outputs( + base_outputs: list[ContractOutputElement], + ) -> list[ContractOutputElement]: specific_outputs = [] contract_outputs = ( ContractBuilder() @@ -156,10 +155,10 @@ def contract_with_specific_outputs(base_outputs: list[ContractOutputElement]) -> @staticmethod def contract( - contract_id: str, - contract_config: ContractConfig, - contract_with_specific_fields: list[ContractElement], - contract_with_specific_outputs: list[ContractOutputElement] + contract_id: str, + contract_config: ContractConfig, + contract_with_specific_fields: list[ContractElement], + contract_with_specific_outputs: list[ContractOutputElement], ) -> Contract: return Contract( contract_id=contract_id, diff --git a/shodan/shodan/contracts/shodan_contracts.py b/shodan/shodan/contracts/shodan_contracts.py index 42652150..e548055a 100644 --- a/shodan/shodan/contracts/shodan_contracts.py +++ b/shodan/shodan/contracts/shodan_contracts.py @@ -1,14 +1,14 @@ from dataclasses import dataclass from enum import Enum, StrEnum -from shodan.models import ConfigLoader + from pyoaev.contracts.contract_config import ( Contract, ContractAsset, ContractAssetGroup, ContractCardinality, - ContractElement, - ContractConfig, ContractCheckbox, + ContractConfig, + ContractElement, ContractExpectations, ContractOutputElement, ContractOutputType, @@ -28,24 +28,28 @@ DomainDiscovery, HostEnumeration, ) +from shodan.models import ConfigLoader TYPE = "openaev_shodan" + class InjectorKey(StrEnum): - TARGETS_KEY="targets" - TARGET_SELECTOR_KEY="target_selector" - TARGET_PROPERTY_SELECTOR_KEY="target_property_selector" - AUTO_CREATE_ASSETS="auto_create_assets" - EXPECTATIONS_KEY="expectations" + TARGETS_KEY = "targets" + TARGET_SELECTOR_KEY = "target_selector" + TARGET_PROPERTY_SELECTOR_KEY = "target_property_selector" + AUTO_CREATE_ASSETS = "auto_create_assets" + EXPECTATIONS_KEY = "expectations" + class ShodanContractId(StrEnum): - CLOUD_PROVIDER_ASSET_DISCOVERY="1887b988-1553-4e46-bac1-ceeee0483f3a" - CRITICAL_PORTS_AND_EXPOSED_ADMIN_INTERFACE="5349febd-ec73-408b-aa61-24a86a1ba0a7" - CUSTOM_QUERY="c7640b4c-8a12-458a-b761-6a1287501a58" - CVE_ENUMERATION="8cdccd58-78ed-4e17-be2e-7683ec611569" - CVE_SPECIFIC_WATCHLIST="462087b4-8012-4e21-9575-b9c854ef5811" - DOMAIN_DISCOVERY="faf73809-1128-4192-aa90-a08828f8ace5" - HOST_ENUMERATION="dc6b8b73-09dd-4388-b7cc-108bf16d26cd" + CLOUD_PROVIDER_ASSET_DISCOVERY = "1887b988-1553-4e46-bac1-ceeee0483f3a" + CRITICAL_PORTS_AND_EXPOSED_ADMIN_INTERFACE = "5349febd-ec73-408b-aa61-24a86a1ba0a7" + CUSTOM_QUERY = "c7640b4c-8a12-458a-b761-6a1287501a58" + CVE_ENUMERATION = "8cdccd58-78ed-4e17-be2e-7683ec611569" + CVE_SPECIFIC_WATCHLIST = "462087b4-8012-4e21-9575-b9c854ef5811" + DOMAIN_DISCOVERY = "faf73809-1128-4192-aa90-a08828f8ace5" + HOST_ENUMERATION = "dc6b8b73-09dd-4388-b7cc-108bf16d26cd" + @dataclass class FieldDefinition: @@ -54,6 +58,7 @@ class FieldDefinition: label: str mandatory: bool = False + class TypeOfFields(Enum): ASSETS = FieldDefinition( name="field_assets", @@ -71,11 +76,13 @@ class TypeOfFields(Enum): label="Targeted assets property", ) + @dataclass class SelectorFieldDefinition: key: str label: str + class TargetSelectorField(Enum): ASSETS = SelectorFieldDefinition( key="assets", @@ -98,6 +105,7 @@ def key(self) -> str: def label(self) -> str: return self.value.label + class TargetProperty(Enum): AUTOMATIC = "Automatic" HOSTNAME = "Hostname" @@ -108,6 +116,7 @@ class TargetProperty(Enum): def default_value(value: str = "automatic"): return value.lower() + class ShodanContracts: def __init__(self, config: ConfigLoader): # Load configuration file @@ -129,7 +138,7 @@ def _base_contract_config(): # -- BUILDER CONTRACT FIELDS -- @staticmethod - def _build_target_selector(selector_default_value:str) -> ContractSelect: + def _build_target_selector(selector_default_value: str) -> ContractSelect: prefix = "only_" @@ -142,9 +151,7 @@ def _build_target_selector(selector_default_value:str) -> ContractSelect: if default_start_with_only: effective_default = selector_default_value.removeprefix(prefix) - choices = { - effective_default: choices[effective_default] - } + choices = {effective_default: choices[effective_default]} return ContractSelect( key=InjectorKey.TARGET_SELECTOR_KEY, @@ -159,12 +166,10 @@ def _build_field(field_type: TypeOfFields) -> ContractElement: builder_contract_fields_mapping = { "field_assets": lambda **kwargs: ContractAsset( - cardinality=ContractCardinality.Multiple, - **kwargs + cardinality=ContractCardinality.Multiple, **kwargs ), "field_asset_groups": lambda **kwargs: ContractAssetGroup( - cardinality=ContractCardinality.Multiple, - **kwargs + cardinality=ContractCardinality.Multiple, **kwargs ), "field_assets_property": lambda **kwargs: ContractSelect( key=InjectorKey.TARGET_PROPERTY_SELECTOR_KEY, @@ -184,9 +189,13 @@ def _build_field(field_type: TypeOfFields) -> ContractElement: label=field_type_config.label, mandatory=field_type_config.mandatory, mandatoryConditionFields=[InjectorKey.TARGET_SELECTOR_KEY], - mandatoryConditionValues={InjectorKey.TARGET_SELECTOR_KEY: field_type_config.target}, + mandatoryConditionValues={ + InjectorKey.TARGET_SELECTOR_KEY: field_type_config.target + }, visibleConditionFields=[InjectorKey.TARGET_SELECTOR_KEY], - visibleConditionValues={InjectorKey.TARGET_SELECTOR_KEY: field_type_config.target}, + visibleConditionValues={ + InjectorKey.TARGET_SELECTOR_KEY: field_type_config.target + }, ) @staticmethod @@ -216,7 +225,7 @@ def _build_expectations(): ], ) - def _base_fields(self, selector_default_value:str) -> list[ContractElement]: + def _base_fields(self, selector_default_value: str) -> list[ContractElement]: # Build Target Selector (Assets, Asset Groups, Manual) target_selector = self._build_target_selector(selector_default_value) @@ -254,11 +263,18 @@ def _base_outputs(): return [] def _build_contract( - self, - contract_id:str, - contract_cls: CloudProviderAssetDiscovery | CriticalPortsAndExposedAdminInterface | CustomQuery | - CVEEnumeration | CVESpecificWatchlist | DomainDiscovery | HostEnumeration, - contract_selector_default: str, + self, + contract_id: str, + contract_cls: ( + CloudProviderAssetDiscovery + | CriticalPortsAndExposedAdminInterface + | CustomQuery + | CVEEnumeration + | CVESpecificWatchlist + | DomainDiscovery + | HostEnumeration + ), + contract_selector_default: str, ) -> Contract: return contract_cls.contract( contract_id=contract_id, @@ -278,11 +294,23 @@ def contracts(self): selector_default = TargetSelectorField.ASSET_GROUPS.key shodan_contract_definitions = [ - (ShodanContractId.CLOUD_PROVIDER_ASSET_DISCOVERY, CloudProviderAssetDiscovery, selector_default), - (ShodanContractId.CRITICAL_PORTS_AND_EXPOSED_ADMIN_INTERFACE, CriticalPortsAndExposedAdminInterface, selector_default), + ( + ShodanContractId.CLOUD_PROVIDER_ASSET_DISCOVERY, + CloudProviderAssetDiscovery, + selector_default, + ), + ( + ShodanContractId.CRITICAL_PORTS_AND_EXPOSED_ADMIN_INTERFACE, + CriticalPortsAndExposedAdminInterface, + selector_default, + ), (ShodanContractId.CUSTOM_QUERY, CustomQuery, "only_manual"), (ShodanContractId.CVE_ENUMERATION, CVEEnumeration, selector_default), - (ShodanContractId.CVE_SPECIFIC_WATCHLIST, CVESpecificWatchlist, selector_default), + ( + ShodanContractId.CVE_SPECIFIC_WATCHLIST, + CVESpecificWatchlist, + selector_default, + ), (ShodanContractId.DOMAIN_DISCOVERY, DomainDiscovery, selector_default), (ShodanContractId.HOST_ENUMERATION, HostEnumeration, selector_default), ] diff --git a/shodan/shodan/injector/__init__.py b/shodan/shodan/injector/__init__.py index f871b0ff..929d68ad 100644 --- a/shodan/shodan/injector/__init__.py +++ b/shodan/shodan/injector/__init__.py @@ -16,4 +16,4 @@ "CVESpecificWatchlist", "DomainDiscovery", "HostEnumeration", -] \ No newline at end of file +] diff --git a/shodan/shodan/injector/openaev_shodan.py b/shodan/shodan/injector/openaev_shodan.py index b9ae92f5..710ed800 100644 --- a/shodan/shodan/injector/openaev_shodan.py +++ b/shodan/shodan/injector/openaev_shodan.py @@ -1,5 +1,8 @@ import time from datetime import datetime, timezone + +from pyoaev.helpers import OpenAEVInjectorHelper + from shodan.contracts import ( CloudProviderAssetDiscovery, CriticalPortsAndExposedAdminInterface, @@ -11,12 +14,12 @@ InjectorKey, ShodanContractId, ) -from shodan.services import ShodanClientAPI, Utils from shodan.models import ConfigLoader -from pyoaev.helpers import OpenAEVInjectorHelper +from shodan.services import ShodanClientAPI, Utils LOG_PREFIX = "[SHODAN_INJECTOR]" + class ShodanInjector: def __init__(self, config: ConfigLoader, helper: OpenAEVInjectorHelper): """Initialize the Injector with necessary configurations""" @@ -27,8 +30,9 @@ def __init__(self, config: ConfigLoader, helper: OpenAEVInjectorHelper): self.shodan_client_api = ShodanClientAPI(self.config, self.helper) self.utils = Utils() - - def _prepare_output_message(self, contract_name:str, inject_content:dict, results, user_info:dict): + def _prepare_output_message( + self, contract_name: str, inject_content: dict, results, user_info: dict + ): self.helper.injector_logger.info( f"{LOG_PREFIX} - Start preparing the output message rendering.", ) @@ -45,7 +49,7 @@ def _prepare_output_message(self, contract_name:str, inject_content:dict, result if contract_name not in output_trace_config: raise ValueError( f"{LOG_PREFIX} - The contract name is unknown.", - {"contract_name": contract_name} + {"contract_name": contract_name}, ) contract_output_trace_config = output_trace_config.get(contract_name) @@ -73,11 +77,12 @@ def _prepare_output_message(self, contract_name:str, inject_content:dict, result ) return rendering_output_message - def _shodan_execution(self, data): # Contract information - contract_id = data["injection"]["inject_injector_contract"]["convertedContent"]["contract_id"] + contract_id = data["injection"]["inject_injector_contract"]["convertedContent"][ + "contract_id" + ] contract_name = ShodanContractId(contract_id).name inject_content = data["injection"]["inject_content"] @@ -88,12 +93,14 @@ def _shodan_execution(self, data): { "contract_id": contract_id, "contract_name": contract_name, - "type_of_target" : selector_key + "type_of_target": selector_key, }, ) if selector_key == "manual": - shodan_results, shodan_credit_user = self.shodan_client_api.process_shodan_search( - contract_id, inject_content + shodan_results, shodan_credit_user = ( + self.shodan_client_api.process_shodan_search( + contract_id, inject_content + ) ) output_json = "" @@ -115,7 +122,6 @@ def _shodan_execution(self, data): else: return None, None - def process_message(self, data: dict) -> None: # Initialization to get the current start utc iso format. start_utc_isoformat = datetime.now(timezone.utc).isoformat(timespec="seconds") @@ -150,9 +156,7 @@ def process_message(self, data: dict) -> None: ) self.helper.injector_logger.info( f"{LOG_PREFIX} - The injector has completed its execution.", - { - "execution_duration": f"{execution_duration} s" - } + {"execution_duration": f"{execution_duration} s"}, ) except Exception as err: @@ -171,5 +175,3 @@ def process_message(self, data: dict) -> None: def start(self): self.helper.listen(message_callback=self.process_message) - - diff --git a/shodan/shodan/models/__init__.py b/shodan/shodan/models/__init__.py index b2e90438..f8751160 100644 --- a/shodan/shodan/models/__init__.py +++ b/shodan/shodan/models/__init__.py @@ -1,3 +1,3 @@ from shodan.models.configs.config_loader import ConfigLoader -__all__ = ["ConfigLoader"] \ No newline at end of file +__all__ = ["ConfigLoader"] diff --git a/shodan/shodan/models/configs/config_loader.py b/shodan/shodan/models/configs/config_loader.py index 50e53486..054a397b 100644 --- a/shodan/shodan/models/configs/config_loader.py +++ b/shodan/shodan/models/configs/config_loader.py @@ -1,7 +1,8 @@ """Base class for global config models.""" + from pathlib import Path -from pydantic import Field, BaseModel +from pydantic import BaseModel, Field from pydantic_settings import ( BaseSettings, DotEnvSettingsSource, @@ -11,12 +12,13 @@ ) from shodan.models.configs import ( - _SettingsLoader, - _BaseOpenAEVConfig, _BaseInjectorConfig, + _BaseOpenAEVConfig, _ConfigLoaderShodan, + _SettingsLoader, ) + class _BaseInjectorConfigHelperAdapter: def __init__(self, data: dict): self.data = data @@ -27,6 +29,7 @@ def get_conf(self, key, default=None): value = value["data"] return value + class _BaseInjectorConfigUtils: def to_flatten(self, contracts=None) -> dict: @@ -42,7 +45,9 @@ def to_flatten(self, contracts=None) -> dict: flatten_config["injector_contracts"] = contracts return flatten_config - def to_config_injector_helper_adapter(self, contracts) -> _BaseInjectorConfigHelperAdapter: + def to_config_injector_helper_adapter( + self, contracts + ) -> _BaseInjectorConfigHelperAdapter: """Returns an OpenAEVInjectorHelper-compatible object""" flatten_dict = self.to_flatten(contracts) return _BaseInjectorConfigHelperAdapter(flatten_dict) @@ -66,12 +71,12 @@ class ConfigLoader(_BaseInjectorConfigUtils, _SettingsLoader): @classmethod def settings_customise_sources( - cls, - settings_cls: type[BaseSettings], - init_settings: PydanticBaseSettingsSource, - env_settings: PydanticBaseSettingsSource, - dotenv_settings: PydanticBaseSettingsSource, - file_secret_settings: PydanticBaseSettingsSource, + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, ) -> tuple[PydanticBaseSettingsSource]: """Pydantic settings customisation sources. @@ -117,4 +122,4 @@ def settings_customise_sources( settings_cls, env_ignore_empty=True, ), - ) \ No newline at end of file + ) diff --git a/shodan/shodan/models/configs/injector_configs.py b/shodan/shodan/models/configs/injector_configs.py index dfaa44d2..9e4c9c40 100644 --- a/shodan/shodan/models/configs/injector_configs.py +++ b/shodan/shodan/models/configs/injector_configs.py @@ -1,8 +1,8 @@ """Base class for global config models.""" from abc import ABC - from typing import Literal + from pydantic import ( BaseModel, ConfigDict, @@ -10,11 +10,16 @@ HttpUrl, ) + class BaseConfigModel(BaseModel, ABC): """Base class for global config models To prevent attributes from being modified after initialization. """ - model_config = ConfigDict(extra="allow", str_min_length=1, frozen=True, validate_default=True) + + model_config = ConfigDict( + extra="allow", str_min_length=1, frozen=True, validate_default=True + ) + class _BaseOpenAEVConfig(BaseConfigModel, ABC): url: HttpUrl = Field( @@ -24,6 +29,7 @@ class _BaseOpenAEVConfig(BaseConfigModel, ABC): description="The API token to connect to OpenAEV.", ) + class _BaseInjectorConfig(BaseConfigModel, ABC): """Base class for connector configuration.""" @@ -37,9 +43,9 @@ class _BaseInjectorConfig(BaseConfigModel, ABC): ) type: str = Field( default="openaev_shodan", - description="Identifies the functional type of the injector in OpenAEV" + description="Identifies the functional type of the injector in OpenAEV", ) log_level: Literal["debug", "info", "warning", "error"] = Field( default="error", description="The minimum level of logs to display.", - ) \ No newline at end of file + ) diff --git a/shodan/shodan/models/configs/shodan_configs.py b/shodan/shodan/models/configs/shodan_configs.py index 80d4cb45..a28f2cdc 100644 --- a/shodan/shodan/models/configs/shodan_configs.py +++ b/shodan/shodan/models/configs/shodan_configs.py @@ -1,11 +1,13 @@ """Configuration for Shodan injector.""" from datetime import timedelta + from pydantic import ( Field, - SecretStr, PositiveInt, + SecretStr, ) + from shodan.models.configs import _SettingsLoader @@ -22,12 +24,12 @@ class _ConfigLoaderShodan(_SettingsLoader): api_leaky_bucket_rate: PositiveInt = Field( default=10, description="Bucket refill rate (in tokens per second). Controls the rate at which API calls are allowed. " - "For example, a rate of 10 means that 10 calls can be made per second, if the bucket is not empty.", + "For example, a rate of 10 means that 10 calls can be made per second, if the bucket is not empty.", ) api_leaky_bucket_capacity: PositiveInt = Field( default=10, description="Maximum bucket capacity (in tokens). Defines the number of calls that can be made immediately in a " - "burst. Once the bucket is empty, it refills at the rate defined by 'api_leaky_bucket_rate'.", + "burst. Once the bucket is empty, it refills at the rate defined by 'api_leaky_bucket_rate'.", ) api_retry: PositiveInt = Field( default=5, @@ -36,4 +38,4 @@ class _ConfigLoaderShodan(_SettingsLoader): api_backoff: timedelta = Field( default="PT30S", description="Maximum exponential backoff delay between retry attempts (ISO 8601 duration format).", - ) \ No newline at end of file + ) diff --git a/shodan/shodan/services/client_api.py b/shodan/shodan/services/client_api.py index beb646b7..db8b0f95 100644 --- a/shodan/shodan/services/client_api.py +++ b/shodan/shodan/services/client_api.py @@ -1,25 +1,30 @@ -import requests import re -from limiter import Limiter -from tenacity import retry, stop_after_attempt, wait_exponential_jitter, RetryError from dataclasses import dataclass from enum import Enum -from urllib.parse import urljoin, quote_plus +from urllib.parse import quote_plus, urljoin + +import requests +from limiter import Limiter +from pyoaev.helpers import OpenAEVInjectorHelper +from tenacity import RetryError, retry, stop_after_attempt, wait_exponential_jitter + from shodan.contracts import ShodanContractId from shodan.models import ConfigLoader -from pyoaev.helpers import OpenAEVInjectorHelper LOG_PREFIX = "[SHODAN_CLIENT_API]" + class MissingRequiredFieldError(ValueError): pass + @dataclass class ShodanRestAPIDefinition: http_method: str endpoint: str path_parameter: bool = False + class ShodanRestAPI(Enum): SEARCH_SHODAN = ShodanRestAPIDefinition( http_method="GET", @@ -31,8 +36,7 @@ class ShodanRestAPI(Enum): endpoint="shodan/host/{target}", ) API_PLAN_INFORMATION = ShodanRestAPIDefinition( - http_method="GET", - endpoint="api-info" + http_method="GET", endpoint="api-info" ) @property @@ -65,7 +69,6 @@ def __init__(self, config: ConfigLoader, helper: OpenAEVInjectorHelper): bucket="shodan", ) - @staticmethod def _split_target(raw_input: str) -> list[str]: return [target for target in re.split(r"[,\s]+", raw_input or "") if target] @@ -79,8 +82,8 @@ def _secure_url(url: str) -> str: return url.split("?key=")[0] @staticmethod - def _build_query(query_params: dict[str,tuple[str,str]]) -> str: - query=["query="] + def _build_query(query_params: dict[str, tuple[str, str]]) -> str: + query = ["query="] for key, (value, operator) in query_params.items(): if operator == "and": query.append(f"{key}:{value} ") @@ -91,7 +94,12 @@ def _build_query(query_params: dict[str,tuple[str,str]]) -> str: break return "".join(query).strip() - def _build_url(self, endpoint: str, query_params: str | None = None, is_custom_query:bool=False) -> str: + def _build_url( + self, + endpoint: str, + query_params: str | None = None, + is_custom_query: bool = False, + ) -> str: url = urljoin(self.base_url, endpoint) api_key = f"key={self.api_key.get_secret_value()}" if not query_params and "?query=" not in url: @@ -100,7 +108,6 @@ def _build_url(self, endpoint: str, query_params: str | None = None, is_custom_q return f"{url}&{api_key}" return f"{url}?{query_params}&{api_key}" - def _get_user_info(self): self.helper.injector_logger.info( f"{LOG_PREFIX} - Preparation for user quota recovery....", @@ -239,19 +246,19 @@ def _get_host_enumeration(self, inject_content): ) def _process_request( - self, - raw_input:str, - request_api: ShodanRestAPI | None, - filters_template:dict=None, - is_custom_query:bool=False, - http_method_custom_query:str|None=None + self, + raw_input: str, + request_api: ShodanRestAPI | None, + filters_template: dict = None, + is_custom_query: bool = False, + http_method_custom_query: str | None = None, ): if is_custom_query: targets = [raw_input] - http_method=http_method_custom_query - endpoint_template=raw_input - path_parameter=None + http_method = http_method_custom_query + endpoint_template = raw_input + path_parameter = None else: targets = self._split_target(raw_input) http_method = request_api.value.http_method @@ -284,11 +291,13 @@ def _process_request( method=http_method, url=target_url, ) - results.append({ - "target": target, - "url": f"{http_method} {self._secure_url(target_url)}", - "result": result, - }) + results.append( + { + "target": target, + "url": f"{http_method} {self._secure_url(target_url)}", + "result": result, + } + ) except RetryError as retry_exc: inner_exception = retry_exc.last_attempt.exception() @@ -305,14 +314,15 @@ def _process_request( "error": response.text, } - results.append({ - "target": target, - "is_error": True, - "request": request_filtered, - "response": response_filtered, - }) - return {"targets":targets, "data": results} - + results.append( + { + "target": target, + "is_error": True, + "request": request_filtered, + "response": response_filtered, + } + ) + return {"targets": targets, "data": results} def _request_data(self, method: str, url: str): @retry( @@ -322,14 +332,16 @@ def _request_data(self, method: str, url: str): ), ) def _retry_wrapped(): - response = requests.request(method=method, url=url) + response = requests.request(method=method, url=url) response.raise_for_status() return response.json() with self.rate_limiter: return _retry_wrapped() - def process_shodan_search(self, contract_id: ShodanContractId, inject_content:dict): + def process_shodan_search( + self, contract_id: ShodanContractId, inject_content: dict + ): self.helper.injector_logger.info( f"{LOG_PREFIX} - Starting the Shodan search process...", ) @@ -347,7 +359,7 @@ def process_shodan_search(self, contract_id: ShodanContractId, inject_content:di if not contract: raise ValueError( f"{LOG_PREFIX} - The contract ID is invalid.", - {"contract_id": contract_id} + {"contract_id": contract_id}, ) results = contract(inject_content) @@ -358,6 +370,3 @@ def process_shodan_search(self, contract_id: ShodanContractId, inject_content:di f"{LOG_PREFIX} - Finalization of the Shodan search process....", ) return results, shodan_credit_user - - - diff --git a/shodan/shodan/services/utils.py b/shodan/shodan/services/utils.py index 573120e3..9e2a8253 100644 --- a/shodan/shodan/services/utils.py +++ b/shodan/shodan/services/utils.py @@ -1,17 +1,18 @@ -from typing import Any +import json from datetime import datetime from enum import Enum, StrEnum -import json +from typing import Any from rich import box from rich.align import Align from rich.console import Console, Group from rich.json import JSON from rich.panel import Panel +from rich.rule import Rule from rich.table import Table from rich.text import Text from rich.tree import Tree -from rich.rule import Rule + class OutputIcons(Enum): SUCCESS = "✅" @@ -21,10 +22,11 @@ class OutputIcons(Enum): INFO = "ℹ️" API = "🌐" + class Utils: @staticmethod - def _get_trace_config(output_trace_config: dict, path: str, default = None): + def _get_trace_config(output_trace_config: dict, path: str, default=None): keys = path.split(".") try: for key in keys: @@ -33,7 +35,6 @@ def _get_trace_config(output_trace_config: dict, path: str, default = None): except (KeyError, TypeError): return default - def _add_value_to_tree(self, tree: Tree, key: str, value: Any) -> None: if isinstance(value, list): node = tree.add(f"{key}:") @@ -54,13 +55,12 @@ def _add_value_to_tree(self, tree: Tree, key: str, value: Any) -> None: else: tree.add(f"{key}: {value}") - def _build_tree( - self, - tree: Tree, - data: list[dict[str, Any]], - keys_to_exclude: list[str], - keys_list_to_string: list[str], + self, + tree: Tree, + data: list[dict[str, Any]], + keys_to_exclude: list[str], + keys_list_to_string: list[str], ): keys_to_exclude = keys_to_exclude or [] for section in data: @@ -83,7 +83,6 @@ def _build_tree( self._add_value_to_tree(tree, key, value) - @staticmethod def _get_output_icon(icon_name: str = "") -> str: if not icon_name: @@ -94,15 +93,20 @@ def _get_output_icon(icon_name: str = "") -> str: return OutputIcons[icon].value return "" - # MAKE JSON def _make_json(self, data, output_trace_config: dict): - options_show_json_is_active = self._get_trace_config(output_trace_config, "options.show_json.is_active", False) + options_show_json_is_active = self._get_trace_config( + output_trace_config, "options.show_json.is_active", False + ) if not options_show_json_is_active: return None - options_show_json_indent = self._get_trace_config(output_trace_config, "options.show_json.indent", 2) - options_show_json_sort_keys = self._get_trace_config(output_trace_config, "options.show_json.sort_keys", False) + options_show_json_indent = self._get_trace_config( + output_trace_config, "options.show_json.indent", 2 + ) + options_show_json_sort_keys = self._get_trace_config( + output_trace_config, "options.show_json.sort_keys", False + ) if isinstance(data, (dict, list)): return JSON.from_data( @@ -116,7 +120,6 @@ def _make_json(self, data, output_trace_config: dict): return JSON.from_data({"data": str(data)}) - @staticmethod def _make_header(output_trace_config: dict): options = output_trace_config.get("options", {}) @@ -138,31 +141,49 @@ def _make_header(output_trace_config: dict): subtitle_align="center", ) - # MAKE SEPARATOR def _make_separator(self, output_trace_config: dict) -> Rule | None: - options_show_separator = self._get_trace_config(output_trace_config, "options.show_separator.is_active", default=False) + options_show_separator = self._get_trace_config( + output_trace_config, "options.show_separator.is_active", default=False + ) if not options_show_separator: return None return Rule(characters="─") - # MAKE SECTION CONFIG - def _make_section_config(self, output_trace_config: dict, data_sections_config: list[dict]): - options_show_sections = self._get_trace_config(output_trace_config, "options.show_sections.is_active", default=True) - options_show_sec_config = self._get_trace_config(output_trace_config, "options.show_sections.sec_config", default=True) + def _make_section_config( + self, output_trace_config: dict, data_sections_config: list[dict] + ): + options_show_sections = self._get_trace_config( + output_trace_config, "options.show_sections.is_active", default=True + ) + options_show_sec_config = self._get_trace_config( + output_trace_config, "options.show_sections.sec_config", default=True + ) if not (options_show_sections and options_show_sec_config): return None sec_config_icon = self._get_output_icon( - self._get_trace_config(output_trace_config, "sections_config.header.icon", default="CONFIG") + self._get_trace_config( + output_trace_config, "sections_config.header.icon", default="CONFIG" + ) + ) + sec_config_title = self._get_trace_config( + output_trace_config, + "sections_config.header.title", + default="[CONFIG] Summary of all configurations used for the contract.", + ) + keys_list_to_string = self._get_trace_config( + output_trace_config, "sections_config.keys_list_to_string", default=[] + ) + keys_to_exclude = self._get_trace_config( + output_trace_config, "sections_config.keys_to_exclude", default=[] ) - sec_config_title = self._get_trace_config(output_trace_config, "sections_config.header.title", default="[CONFIG] Summary of all configurations used for the contract.") - keys_list_to_string = self._get_trace_config(output_trace_config, "sections_config.keys_list_to_string", default=[]) - keys_to_exclude = self._get_trace_config(output_trace_config, "sections_config.keys_to_exclude", default=[]) - section_tree = Tree(f"{sec_config_icon} {sec_config_title}".strip(), guide_style="bold") + section_tree = Tree( + f"{sec_config_icon} {sec_config_title}".strip(), guide_style="bold" + ) self._build_tree( tree=section_tree, data=data_sections_config, @@ -171,23 +192,40 @@ def _make_section_config(self, output_trace_config: dict, data_sections_config: ) return section_tree - # MAKE SECTION INFO - def _make_section_info(self, output_trace_config: dict, data_sections_info: list[dict]): - options_show_sections = self._get_trace_config(output_trace_config, "options.show_sections.is_active", default=True) - options_show_sec_info = self._get_trace_config(output_trace_config, "options.show_sections.sec_info", default=True) + def _make_section_info( + self, output_trace_config: dict, data_sections_info: list[dict] + ): + options_show_sections = self._get_trace_config( + output_trace_config, "options.show_sections.is_active", default=True + ) + options_show_sec_info = self._get_trace_config( + output_trace_config, "options.show_sections.sec_info", default=True + ) if not (options_show_sections and options_show_sec_info): return None sec_info_icon = self._get_output_icon( - self._get_trace_config(output_trace_config, "sections_info.header.icon", default="INFO") + self._get_trace_config( + output_trace_config, "sections_info.header.icon", default="INFO" + ) + ) + sec_info_title = self._get_trace_config( + output_trace_config, + "sections_info.header.title", + default="[INFO] The Injector information", + ) + keys_list_to_string = self._get_trace_config( + output_trace_config, "sections_info.keys_list_to_string", default=[] + ) + keys_to_exclude = self._get_trace_config( + output_trace_config, "sections_info.keys_to_exclude", default=[] ) - sec_info_title = self._get_trace_config(output_trace_config, "sections_info.header.title", default="[INFO] The Injector information") - keys_list_to_string = self._get_trace_config(output_trace_config, "sections_info.keys_list_to_string", default=[]) - keys_to_exclude = self._get_trace_config(output_trace_config, "sections_info.keys_to_exclude", default=[]) - section_tree = Tree(f"{sec_info_icon} {sec_info_title}".strip(), guide_style="bold") + section_tree = Tree( + f"{sec_info_icon} {sec_info_title}".strip(), guide_style="bold" + ) self._build_tree( tree=section_tree, data=data_sections_info, @@ -196,7 +234,6 @@ def _make_section_info(self, output_trace_config: dict, data_sections_info: list ) return section_tree - @staticmethod def _count_at_path(result: dict, path: str) -> int: current = result @@ -211,8 +248,9 @@ def _count_at_path(result: dict, path: str) -> int: else: return 1 - - def _prepare_call_details(self, output_trace_config:dict, data_sections_external_api: list[dict]): + def _prepare_call_details( + self, output_trace_config: dict, data_sections_external_api: list[dict] + ): results_success = [] total_success_details_count = 0 @@ -244,41 +282,88 @@ def _prepare_call_details(self, output_trace_config:dict, data_sections_external "result": result.get("result"), } results_success.append(call_success_details) - count_at_path = self._get_trace_config(output_trace_config, "sections_external_api.call_success.count_at_path", default="") + count_at_path = self._get_trace_config( + output_trace_config, + "sections_external_api.call_success.count_at_path", + default="", + ) if count_at_path: - total_success_details_count += self._count_at_path(call_success_details.get("result"), count_at_path) + total_success_details_count += self._count_at_path( + call_success_details.get("result"), count_at_path + ) - return (results_success, total_success_details_count), (results_failed, total_call_failed_count) + return (results_success, total_success_details_count), ( + results_failed, + total_call_failed_count, + ) # MAKE SECTION EXTERNAL API - def _make_section_external_api(self, output_trace_config: dict, data_sections_external_api: list[dict]): - options_show_sections = self._get_trace_config(output_trace_config, "options.show_sections.is_active", default=True) - options_show_sec_external_api = self._get_trace_config(output_trace_config, "options.show_sections.sec_external_api", default=True) + def _make_section_external_api( + self, output_trace_config: dict, data_sections_external_api: list[dict] + ): + options_show_sections = self._get_trace_config( + output_trace_config, "options.show_sections.is_active", default=True + ) + options_show_sec_external_api = self._get_trace_config( + output_trace_config, "options.show_sections.sec_external_api", default=True + ) if not (options_show_sections and options_show_sec_external_api): return None sec_external_api_icon = self._get_output_icon( - self._get_trace_config(output_trace_config, "sections_external_api.header.icon", default="API") + self._get_trace_config( + output_trace_config, "sections_external_api.header.icon", default="API" + ) + ) + sec_external_api_title = self._get_trace_config( + output_trace_config, + "sections_external_api.header.title", + default="[INJECTOR] Call API completed", + ) + section_tree = Tree( + f"{sec_external_api_icon} {sec_external_api_title}".strip(), + guide_style="bold", ) - sec_external_api_title = self._get_trace_config(output_trace_config, "sections_external_api.header.title", default="[INJECTOR] Call API completed") - section_tree = Tree(f"{sec_external_api_icon} {sec_external_api_title}".strip(), guide_style="bold") # Add node call success call_success_icon = self._get_output_icon( - self._get_trace_config(output_trace_config, "sections_external_api.call_success.icon", default="SUCCESS") + self._get_trace_config( + output_trace_config, + "sections_external_api.call_success.icon", + default="SUCCESS", + ) + ) + call_success_title = self._get_trace_config( + output_trace_config, + "sections_external_api.call_success.title", + default="Call Success", + ) + success_node = section_tree.add( + f"{call_success_icon} {call_success_title}".strip() ) - call_success_title = self._get_trace_config(output_trace_config, "sections_external_api.call_success.title", default="Call Success") - success_node = section_tree.add(f"{call_success_icon} {call_success_title}".strip()) # Add node call failed call_failed_icon = self._get_output_icon( - self._get_trace_config(output_trace_config, "sections_external_api.call_failed.icon", default="FAILED") + self._get_trace_config( + output_trace_config, + "sections_external_api.call_failed.icon", + default="FAILED", + ) + ) + call_failed_title = self._get_trace_config( + output_trace_config, + "sections_external_api.call_failed.title", + default="Call Failed", + ) + failed_node = section_tree.add( + f"{call_failed_icon} {call_failed_title}".strip() ) - call_failed_title = self._get_trace_config(output_trace_config, "sections_external_api.call_failed.title", default="Call Failed") - failed_node = section_tree.add(f"{call_failed_icon} {call_failed_title}".strip()) - (results_success_details, total_success_details_count), (results_failed_details, total_call_failed_count) = self._prepare_call_details(output_trace_config, data_sections_external_api) + (results_success_details, total_success_details_count), ( + results_failed_details, + total_call_failed_count, + ) = self._prepare_call_details(output_trace_config, data_sections_external_api) # Add Total results (success and failed) success_node.add(f"Total results: {total_success_details_count}") @@ -291,10 +376,18 @@ def _make_section_external_api(self, output_trace_config: dict, data_sections_ex result_details_count = 0 request = result_detail.get("request") - count_at_path = self._get_trace_config(output_trace_config, "sections_external_api.call_success.count_at_path", default="") + count_at_path = self._get_trace_config( + output_trace_config, + "sections_external_api.call_success.count_at_path", + default="", + ) if count_at_path: - result_details_count += self._count_at_path(result_detail.get("result"), count_at_path) - target_node = success_details_node.add(f"• {target} → {result_details_count} results") + result_details_count += self._count_at_path( + result_detail.get("result"), count_at_path + ) + target_node = success_details_node.add( + f"• {target} → {result_details_count} results" + ) target_node.add(f"Request: {request}") # Add Details for call Failed @@ -310,8 +403,9 @@ def _make_section_external_api(self, output_trace_config: dict, data_sections_ex return section_tree, results_success_details - - def _extract_level(self, items: list[Any], parts_remaining: list[str], use_key: bool = False) -> list[Any]: + def _extract_level( + self, items: list[Any], parts_remaining: list[str], use_key: bool = False + ) -> list[Any]: if not parts_remaining: return items @@ -330,7 +424,9 @@ def _extract_level(self, items: list[Any], parts_remaining: list[str], use_key: sub_result = [k if use_key else v for k, v in item.items()] if next_parts: - sub_result = self._extract_level(sub_result, next_parts, use_key) + sub_result = self._extract_level( + sub_result, next_parts, use_key + ) result.append(sub_result) else: @@ -340,7 +436,9 @@ def _extract_level(self, items: list[Any], parts_remaining: list[str], use_key: result.append(self._extract_level(item, parts_remaining, use_key)) else: if part == "*" and next_parts: - result.extend(self._extract_level([items[item]], next_parts, use_key)) + result.extend( + self._extract_level([items[item]], next_parts, use_key) + ) else: if part in items and not next_parts and isinstance(item, str): if item == part: @@ -351,8 +449,9 @@ def _extract_level(self, items: list[Any], parts_remaining: list[str], use_key: result.append(item) return result - - def _extractor(self, data: list[dict], path: str, use_key: bool = False) -> list[list[Any]]: + def _extractor( + self, data: list[dict], path: str, use_key: bool = False + ) -> list[list[Any]]: if not path: return [] @@ -361,9 +460,14 @@ def _extractor(self, data: list[dict], path: str, use_key: bool = False) -> list final_result = self._extract_level(data_list, parts, use_key) return final_result - @staticmethod - def _organizer_row(tables_config_columns, show_index_is_active, index_start, column_values, column_extras): + def _organizer_row( + tables_config_columns, + show_index_is_active, + index_start, + column_values, + column_extras, + ): single_columns = [ values @@ -384,15 +488,25 @@ def _organizer_row(tables_config_columns, show_index_is_active, index_start, col for table_config_index, table_config in enumerate(tables_config_columns): mode = table_config.get("mode", "inline") - values = column_values[table_config_index] if table_config_index < len(column_values) else [] - extras = column_extras[table_config_index] if column_extras and table_config_index < len(column_extras) else None + values = ( + column_values[table_config_index] + if table_config_index < len(column_values) + else [] + ) + extras = ( + column_extras[table_config_index] + if column_extras and table_config_index < len(column_extras) + else None + ) cell = "-" if mode == "single": if isinstance(values, list) and len(values) > 0: for value in values: if isinstance(value, list): - filtered = [str(v) for v in value if v not in (None, "")] + filtered = [ + str(v) for v in value if v not in (None, "") + ] cell = ",".join(filtered) if filtered else "-" else: val = values[row_idx] if row_idx < len(values) else None @@ -408,7 +522,9 @@ def _organizer_row(tables_config_columns, show_index_is_active, index_start, col all_vals = [] for val in values: if isinstance(val, list): - all_vals.extend(str(v) for v in val if v not in (None, "")) + all_vals.extend( + str(v) for v in val if v not in (None, "") + ) elif val not in (None, ""): all_vals.append(str(val)) cell = ", ".join(all_vals) if all_vals else "-" @@ -419,11 +535,18 @@ def _organizer_row(tables_config_columns, show_index_is_active, index_start, col elif mode == "align_to_single": if row_idx < len(values): val = values[row_idx] - extra = extras[row_idx] if extras and row_idx < len(extras) else None + extra = ( + extras[row_idx] + if extras and row_idx < len(extras) + else None + ) if isinstance(val, list): if val: if extra: - cell = ", ".join(f"{v} ({e if e is not None else '-'})" for v, e in zip(val, extra or ["-"]*len(val))) + cell = ", ".join( + f"{v} ({e if e is not None else '-'})" + for v, e in zip(val, extra or ["-"] * len(val)) + ) else: cell = ", ".join(str(v) for v in val) else: @@ -469,7 +592,7 @@ def _organizer_row(tables_config_columns, show_index_is_active, index_start, col return final_rows @staticmethod - def _rows_with_limit_cells(rows:list[list[str]], max_display_by_cell:int | None): + def _rows_with_limit_cells(rows: list[list[str]], max_display_by_cell: int | None): rows_with_limit = [] for row in rows: @@ -478,7 +601,11 @@ def _rows_with_limit_cells(rows:list[list[str]], max_display_by_cell:int | None) has_hidden = False for cell in row: - if isinstance(cell, str) and "," in cell and max_display_by_cell is not None: + if ( + isinstance(cell, str) + and "," in cell + and max_display_by_cell is not None + ): items = [c.strip() for c in cell.split(",") if c.strip()] if len(items) > max_display_by_cell: shown = items[:max_display_by_cell] @@ -500,24 +627,45 @@ def _rows_with_limit_cells(rows:list[list[str]], max_display_by_cell:int | None) return rows_with_limit # MAKE TABLES - def _make_tables(self, output_trace_config: dict, data_tables: list[dict], auto_create_assets:bool | None): + def _make_tables( + self, + output_trace_config: dict, + data_tables: list[dict], + auto_create_assets: bool | None, + ): - options_show_tables_is_active = self._get_trace_config(output_trace_config, "options.show_tables.is_active", default=True) + options_show_tables_is_active = self._get_trace_config( + output_trace_config, "options.show_tables.is_active", default=True + ) if not options_show_tables_is_active: return [] - show_index_is_active = self._get_trace_config(output_trace_config, "options.show_tables.show_index.is_active", default=False) - index_start = self._get_trace_config(output_trace_config, "options.show_tables.show_index.index_start", default=0) - show_lines = self._get_trace_config(output_trace_config, "options.show_tables.show_lines", default=True) - max_display_by_cell = self._get_trace_config(output_trace_config, "options.show_tables.max_display_by_cell", default=10) + show_index_is_active = self._get_trace_config( + output_trace_config, + "options.show_tables.show_index.is_active", + default=False, + ) + index_start = self._get_trace_config( + output_trace_config, "options.show_tables.show_index.index_start", default=0 + ) + show_lines = self._get_trace_config( + output_trace_config, "options.show_tables.show_lines", default=True + ) + max_display_by_cell = self._get_trace_config( + output_trace_config, "options.show_tables.max_display_by_cell", default=10 + ) tables = self._get_trace_config(output_trace_config, "tables", default=[]) for table_config in tables: header_icon = self._get_output_icon( self._get_trace_config(table_config, "header.icon", default="SEARCH") ) - header_title = self._get_trace_config(table_config, "header.title", default="") - search_entity = self._get_trace_config(table_config, "config.search_entity", default=None) + header_title = self._get_trace_config( + table_config, "header.title", default="" + ) + search_entity = self._get_trace_config( + table_config, "config.search_entity", default=None + ) tables_final = [] for data_table in data_tables: result = data_table.get("result") @@ -538,7 +686,9 @@ def _make_tables(self, output_trace_config: dict, data_tables: list[dict], auto_ expand=True, ) - tables_config_columns = self._get_trace_config(table_config, "config.columns") + tables_config_columns = self._get_trace_config( + table_config, "config.columns" + ) if show_index_is_active: table.add_column("#") @@ -569,7 +719,9 @@ def _make_tables(self, output_trace_config: dict, data_tables: list[dict], auto_ column_extras, ) - rows_with_limit_cell = self._rows_with_limit_cells(final_rows, max_display_by_cell) + rows_with_limit_cell = self._rows_with_limit_cells( + final_rows, max_display_by_cell + ) for row_with_limit in rows_with_limit_cell: table.add_row(*row_with_limit) @@ -579,14 +731,13 @@ def _make_tables(self, output_trace_config: dict, data_tables: list[dict], auto_ return tables_final return [] - def generate_output_message( - self, - output_trace_config: dict, - data_sections_config: list[dict], - data_sections_info: list[dict], - data_sections_external_api: list[dict], - auto_create_assets: bool | None, + self, + output_trace_config: dict, + data_sections_config: list[dict], + data_sections_info: list[dict], + data_sections_external_api: list[dict], + auto_create_assets: bool | None, ): # Todo: Make split mode @@ -626,9 +777,11 @@ def generate_output_message( renderables.append(output_sections_info) # Output Sections Client API - output_sections_client_api, results_success_details = self._make_section_external_api( - output_trace_config=output_trace_config, - data_sections_external_api=data_sections_external_api, + output_sections_client_api, results_success_details = ( + self._make_section_external_api( + output_trace_config=output_trace_config, + data_sections_external_api=data_sections_external_api, + ) ) if output_sections_client_api: renderables.append(Text("")) @@ -646,12 +799,13 @@ def generate_output_message( for output_table in output_tables: if output_table: renderables.append(Text("")) - separator = self._make_separator(output_trace_config=output_trace_config) + separator = self._make_separator( + output_trace_config=output_trace_config + ) if separator: renderables.append(separator) renderables.append(output_table) - # Output JSON output_json = self._make_json( data=data_sections_external_api, From 6de103ab8251784a5133e9ab5e3fb1bd14254368 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20BASLER?= Date: Tue, 27 Jan 2026 17:19:03 +0100 Subject: [PATCH 05/13] [Shodan] Fix import element unused --- shodan/shodan/contracts/shodan_contracts.py | 9 --------- shodan/shodan/services/utils.py | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/shodan/shodan/contracts/shodan_contracts.py b/shodan/shodan/contracts/shodan_contracts.py index e548055a..8ef1b1e1 100644 --- a/shodan/shodan/contracts/shodan_contracts.py +++ b/shodan/shodan/contracts/shodan_contracts.py @@ -10,8 +10,6 @@ ContractConfig, ContractElement, ContractExpectations, - ContractOutputElement, - ContractOutputType, ContractSelect, Expectation, ExpectationType, @@ -253,13 +251,6 @@ def _base_fields(self, selector_default_value: str) -> list[ContractElement]: # -- OUTPUTS -- @staticmethod def _base_outputs(): - # output_assets = ContractOutputElement( - # type=ContractOutputType.ASSET, - # field="found_assets", - # isMultiple=True, - # isFindingCompatible=False, - # labels=["shodan"], - # ) return [] def _build_contract( diff --git a/shodan/shodan/services/utils.py b/shodan/shodan/services/utils.py index 9e2a8253..672cf342 100644 --- a/shodan/shodan/services/utils.py +++ b/shodan/shodan/services/utils.py @@ -1,6 +1,6 @@ import json from datetime import datetime -from enum import Enum, StrEnum +from enum import Enum from typing import Any from rich import box From 673fcd79a9b77fcb50950f369c005f66684ee289 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20BASLER?= Date: Tue, 27 Jan 2026 17:32:57 +0100 Subject: [PATCH 06/13] [Shodan] Fix client_api --- shodan/shodan/services/client_api.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/shodan/shodan/services/client_api.py b/shodan/shodan/services/client_api.py index db8b0f95..67fb9246 100644 --- a/shodan/shodan/services/client_api.py +++ b/shodan/shodan/services/client_api.py @@ -122,7 +122,9 @@ def _get_user_info(self): def _get_cve_enumeration(self, inject_content): hostname = inject_content.get("hostname") if not hostname: - return None + raise MissingRequiredFieldError( + f"{LOG_PREFIX} - The 'Hostname' field is required and cannot be empty." + ) filters = { "has_vuln": ("true", "and"), @@ -139,7 +141,9 @@ def _get_cve_enumeration(self, inject_content): def _get_cve_specific_watchlist(self, inject_content): hostname = inject_content.get("hostname") if not hostname: - return None + raise MissingRequiredFieldError( + f"{LOG_PREFIX} - The 'Hostname' field is required and cannot be empty." + ) filters = { "vuln": (inject_content.get("vulnerability"), "and"), @@ -156,7 +160,9 @@ def _get_cve_specific_watchlist(self, inject_content): def _get_cloud_provider_asset_discovery(self, inject_content): hostname = inject_content.get("hostname") if not hostname: - return None + raise MissingRequiredFieldError( + f"{LOG_PREFIX} - The 'Hostname' field is required and cannot be empty." + ) cloud_provider = inject_content.get("cloud_provider") # Please note that the default values are by nature a list. @@ -179,7 +185,7 @@ def _get_critical_ports_and_exposed_admin_interface(self, inject_content): hostname = inject_content.get("hostname") if not hostname: raise MissingRequiredFieldError( - f"{LOG_PREFIX} - The 'custom_request' field is required and cannot be empty." + f"{LOG_PREFIX} - The 'Hostname' field is required and cannot be empty." ) port = inject_content.get("port") @@ -200,15 +206,15 @@ def _get_critical_ports_and_exposed_admin_interface(self, inject_content): # CONTRACT - CUSTOM QUERY def _get_custom_query(self, inject_content): - custom_request = inject_content.get("custom_request") + custom_query = inject_content.get("custom_query") http_method = inject_content.get("http_method") - if not custom_request: + if not custom_query: raise MissingRequiredFieldError( - f"{LOG_PREFIX} - The 'Custom Request' field is required and cannot be empty." + f"{LOG_PREFIX} - The 'Custom Query' field is required and cannot be empty." ) return self._process_request( - raw_input=custom_request, + raw_input=custom_query, request_api=None, filters_template=None, is_custom_query=True, From 93ef4e65ad2cccebf2f4a184e1ab88b8a292b89f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20BASLER?= Date: Tue, 27 Jan 2026 17:44:19 +0100 Subject: [PATCH 07/13] [Shodan] Update icon_path --- shodan/shodan/__main__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shodan/shodan/__main__.py b/shodan/shodan/__main__.py index 599acd73..a771e9fc 100644 --- a/shodan/shodan/__main__.py +++ b/shodan/shodan/__main__.py @@ -32,7 +32,8 @@ def main() -> None: config_helper_adapter = config.to_config_injector_helper_adapter( contracts=shodan_contracts ) - icon_bytes = Path("shodan/img/icon-shodan.png").read_bytes() + icon_path = Path(__file__).parent / "img" / "icon-shodan.png" + icon_bytes = icon_path.read_bytes() helper = OpenAEVInjectorHelper(config=config_helper_adapter, icon=icon_bytes) From 92d002adac8b4e55a7e5148aa1659b9ba6df060d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20BASLER?= Date: Thu, 29 Jan 2026 14:17:43 +0100 Subject: [PATCH 08/13] [Shodan] Update Readme, Fix get_user_info, Fix dockerfile, Add .env.sample --- shodan/.dockerignore | 2 +- shodan/Dockerfile | 17 ++++-- shodan/README.md | 58 +++++++++++-------- shodan/pyproject.toml | 3 +- shodan/shodan/.env.sample | 20 +++++++ .../shodan/contracts/custom_query/contract.py | 2 + shodan/shodan/services/client_api.py | 16 ++--- 7 files changed, 78 insertions(+), 40 deletions(-) create mode 100644 shodan/shodan/.env.sample diff --git a/shodan/.dockerignore b/shodan/.dockerignore index 8d8716b4..15e202d9 100644 --- a/shodan/.dockerignore +++ b/shodan/.dockerignore @@ -1,4 +1,4 @@ config.yml -src/__pycache__ +shodan/__pycache__ __pycache__ diff --git a/shodan/Dockerfile b/shodan/Dockerfile index ec661071..52773207 100644 --- a/shodan/Dockerfile +++ b/shodan/Dockerfile @@ -1,13 +1,20 @@ FROM python:3.13-alpine AS build WORKDIR /opt/openaev-injector-shodan -COPY shodan /opt/openaev-injector-shodan -# Build dependencies + pip install +# Copy dependency files (pyproject.toml / poetry.lock / README.md) +COPY pyproject.toml poetry.lock* README.md ./ + +# Build dependencies RUN apk add --no-cache --virtual .build-dependencies \ - build-base git libffi-dev libxml2-dev libxslt-dev && \ - pip install --no-cache-dir -r shodan/requirements.txt && \ - apk del .build-dependencies + build-base git libffi-dev libxml2-dev libxslt-dev \ + && pip3 install --no-cache-dir poetry==2.2.1 \ + && poetry config virtualenvs.create false \ + && poetry install --only main --extras prod --no-interaction --no-ansi --no-root \ + && apk del .build-dependencies + +# Copy the injector +COPY shodan ./shodan FROM python:3.13-alpine AS runner diff --git a/shodan/README.md b/shodan/README.md index 094f8ef6..1fdd92ee 100644 --- a/shodan/README.md +++ b/shodan/README.md @@ -2,18 +2,20 @@ ## Table of Contents -- [OpenAEV SHODAN Injector](#openaev-nuclei-injector) +- [OpenAEV SHODAN Injector](#openaev-shodan-injector) - [Prerequisites](#prerequisites) - [Configuration variables](#configuration-variables) - - [OpenAEV environment variables](#openaev-environment-variables) + - [Base OpenAEV environment variables](#base-openaev-environment-variables) - [Base injector environment variables](#base-injector-environment-variables) - [Base Shodan environment variables](#base-shodan-environment-variables) - [Deployment](#deployment) - [Docker Deployment](#docker-deployment) - [Manual Deployment](#manual-deployment) - [Behavior](#behavior) - - [Template Selection](#template-selection) - - [Target Selection](#target-selection) + - [Supported Contracts](#supported-contracts) + - [Fields available in manual mode by contract](#fields-available-in-manual-mode-by-contract) + - [Output Trace Message](#output-trace-message) + - [Auto-Create Assets](#auto-create-assets) - [Resources](#resources) --- @@ -44,7 +46,9 @@ Depending on your deployment method: Configuration is provided either through environment variables (Docker) or a config file (`config.yml`, manual). -### OpenAEV environment variables +--- + +### Base OpenAEV environment variables | Parameter | config.yml | Docker environment variable | Mandatory | Description | |---------------|------------|-----------------------------|-----------|------------------------------------------------------| @@ -75,6 +79,8 @@ file (`config.yml`, manual). ## Deployment +--- + ### Docker Deployment Build the Docker image using the provided `Dockerfile`: @@ -157,10 +163,10 @@ Use Poetry for production or current releases, or to manage dependencies automat ```bash # Install prod dependencies (stable Pyoaev release) -poetry install --with prod +poetry install --extras prod # Install current release from Git / latest -poetry install --with current +poetry install --extras current # Tips For development/testing with local Pyoaev (client-python) poetry run pip install -r requirements.txt @@ -186,12 +192,14 @@ ShodanInjector #### Commands Summary: The table below summarizes the installation and run commands for different workflows (pip or Poetry, prod/current/dev). -| Installation | Install Command | Run Command | Notes | -|-----------------|----------------------------------------------|-----------------------------------------|------------------------------------------------------------| -| Pip Dev/Test | `pip install -r requirements.txt` | `python -m shodan` / `ShodanInjector` | Requires venv active, handles local editable client-python | -| Poetry Prod | `poetry install --with prod` | `poetry run ShodanInjector` | Installs stable dependencies, venv managed automatically | -| Poetry Current | `poetry install --with current` | `poetry run ShodanInjector` | Installs latest release from Git | -| Poetry Dev/Test | `poetry run pip install -r requirements.txt` | `poetry run ShodanInjector` | Handles local editable client-python + dev/test extras | +| Installation | Install Command | Run Command | Notes | +|-----------------|----------------------------------------------|-----------------------------------------------------------------------|----------------------------------------------------------| +| Pip Prod | `pip install .[prod]` | `python -m shodan` / `ShodanInjector` (Requires venv active) | Installs stable dependencies, venv managed automatically | +| Pip Current | `pip install .[current]` | `python -m shodan` / `ShodanInjector` (Requires venv active) | Installs latest release from Git | +| Pip Dev/Test | `pip install -r requirements.txt` | `python -m shodan` / `ShodanInjector` (Requires venv active) | Handles local editable client-python + dev/test extras | +| Poetry Prod | `poetry install --extras prod` | `poetry run ShodanInjector` / `ShodanInjector` (Requires venv active) | Installs stable dependencies, venv managed automatically | +| Poetry Current | `poetry install --extras current` | `poetry run ShodanInjector` / `ShodanInjector` (Requires venv active) | Installs latest release from Git | +| Poetry Dev/Test | `poetry run pip install -r requirements.txt` | `poetry run ShodanInjector` / `ShodanInjector` (Requires venv active) | Handles local editable client-python + dev/test extras | --- @@ -228,38 +236,38 @@ Direct values separated by commas or spaces between IP addresses or hostnames ar ### Fields available in manual mode by contract -- Cloud Provider Asset Discovery +- Cloud Provider Asset Discovery (Search Shodan Endpoint: `/shodan/host/search`) - Cloud Provider (default: `Google,Microsoft,Amazon,Azure`) - mandatory - Hostname - mandatory - Organization - Optional (default: `hostname value is used`) -- Critical Ports And Exposed Admin Interface +- Critical Ports And Exposed Admin Interface (Search Shodan Endpoint: `/shodan/host/search`) - Port (default: `20,21,22,23,25,53,80,110,111,135,139,143,443,445,993,995,1723,3306,3389,5900,8080`) - mandatory - Hostname - mandatory - Organization - Optional (default: `hostname value is used`) -- Custom Query +- Custom Query (Endpoint: `your custom endpoint`) - HTTP Method (choices: `GET, POST, PUT, DELETE`, default: `GET`) - mandatory - Custom Query - mandatory -- CVE Enumeration +- CVE Enumeration (Search Shodan Endpoint: `/shodan/host/search`) - Hostname - mandatory - Organization - Optional (default: `hostname value is used`) -- CVE Specific Watchlist +- CVE Specific Watchlist (Search Shodan Endpoint: `/shodan/host/search`) - Vulnerability - mandatory - Hostname - mandatory - Organization - Optional (default: `hostname value is used`) -- Domain Discovery - - Hostname - mandatory - - Organization - Optional (default: `hostname value is used`) +- Domain Discovery (Search Shodan Endpoint: `/shodan/host/search`) + - Hostname - mandatory + - Organization - Optional (default: `hostname value is used`) -- Host Enumeration +- Host Enumeration (Host Information Endpoint: `/shodan/host/{ip}`) - Host - mandatory -### Output Parsing +### Output Trace Message The injector captures the fields filled in by the user and analyzes the JSON output of the Shodan response, then returns several sections in the report if successful: @@ -267,8 +275,8 @@ then returns several sections in the report if successful: - **Section Config** – Groups all the configuration information entered by the user for the current contract. - **Section Info** – Makes an additional final call to the Shodan API to determine the user's remaining quota and plan. - **Section External API** – Processes information from Shodan API responses based on the contract and fields filled in. This section also includes: - - Call Success – List of successful calls + As well as various other relevant information related to API calls - - Call Failed – List of failed calls + As well as various other relevant information related to API calls + - Call Success – List of successful calls + As well as various other relevant information related to API calls. + - Call Failed – List of failed calls + As well as various other relevant information related to API calls. - **Section Table** – Visually displays the details of each call, based on the configuration defined for the contract in the injector. - **OPTIONAL** - JSON return of the response directly (In the case of a "custom query", we return the JSON directly rather than the table section.) diff --git a/shodan/pyproject.toml b/shodan/pyproject.toml index 8fcf7508..45631ef7 100644 --- a/shodan/pyproject.toml +++ b/shodan/pyproject.toml @@ -15,12 +15,13 @@ description = "An injector for running with Shodan" readme = "README.md" authors = [{ name = "Filigran", email = "contact@filigran.io" }] license = "Apache-2.0" -requires-python = ">=3.11, <3.13" +requires-python = ">=3.11, <3.14" classifiers = [ "Development Status :: 5 - Production/Stable", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ] dependencies = [ "pydantic~=2.11.7", diff --git a/shodan/shodan/.env.sample b/shodan/shodan/.env.sample new file mode 100644 index 00000000..1c55f139 --- /dev/null +++ b/shodan/shodan/.env.sample @@ -0,0 +1,20 @@ +# OPENAEV Environment Variables +# base URL to reach the OpenAEV server +# note this URL must be routable from inside the container +# so `localhost` will most likely not work +OPENAEV_URL=ChangeMe +# admin account API token from the OpenAEV server +OPENAEV_TOKEN=ChangeMe + +# INJECTOR Environment Variables +#INJECTOR_ID=shodan--a87488ad-2c72-4592-b429-69259d7bcef1 +#INJECTOR_NAME=Shodan +#INJECTOR_LOG_LEVEL=error + +# SHODAN Environment Variables +SHODAN_API_KEY=ChangeMe +#SHODAN_BASE_URL=https://api.shodan.io +#SHODAN_API_LEAKY_BUCKET_RATE=10 +#SHODAN_API_LEAKY_BUCKET_CAPACITY=10 +#SHODAN_API_RETRY=5 +#SHODAN_API_BACKOFF=PT30S \ No newline at end of file diff --git a/shodan/shodan/contracts/custom_query/contract.py b/shodan/shodan/contracts/custom_query/contract.py index c18cf815..c2914176 100644 --- a/shodan/shodan/contracts/custom_query/contract.py +++ b/shodan/shodan/contracts/custom_query/contract.py @@ -24,6 +24,7 @@ def output_trace_config(): "icon": "CONFIG", "title": "[CONFIG] Summary of all configurations used for the contract.", }, + "keys_list_to_string": [], "keys_to_exclude": [], }, "sections_info": { @@ -31,6 +32,7 @@ def output_trace_config(): "icon": "INFO", "title": "[INFO] The Shodan information for the remaining credits and the user's plan.", }, + "keys_list_to_string": [], "keys_to_exclude": [], }, "sections_external_api": { diff --git a/shodan/shodan/services/client_api.py b/shodan/shodan/services/client_api.py index 67fb9246..e9eff10e 100644 --- a/shodan/shodan/services/client_api.py +++ b/shodan/shodan/services/client_api.py @@ -108,14 +108,15 @@ def _build_url( return f"{url}&{api_key}" return f"{url}?{query_params}&{api_key}" + # SECTION INFO def _get_user_info(self): self.helper.injector_logger.info( - f"{LOG_PREFIX} - Preparation for user quota recovery....", + f"{LOG_PREFIX} - Preparation for user quota recovery...", ) - target_url = self._build_url(ShodanRestAPI.API_PLAN_INFORMATION.endpoint) - return self._request_data( - method=ShodanRestAPI.API_PLAN_INFORMATION.http_method, - url=target_url, + + return self._process_request( + raw_input="user_info", + request_api=ShodanRestAPI.API_PLAN_INFORMATION, ) # CONTRACT - CVE ENUMERATION @@ -369,10 +370,9 @@ def process_shodan_search( ) results = contract(inject_content) - - # Todo Include the user_info call in the _process_request shodan_credit_user = self._get_user_info() + self.helper.injector_logger.info( - f"{LOG_PREFIX} - Finalization of the Shodan search process....", + f"{LOG_PREFIX} - Finalization of the Shodan search process.", ) return results, shodan_credit_user From 01143cf06c157daba2c14ceff37eb15333193f33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20BASLER?= Date: Fri, 30 Jan 2026 18:09:44 +0100 Subject: [PATCH 09/13] =?UTF-8?q?[Shodan]=20Minor=20updates=20and=20minor?= =?UTF-8?q?=20fixes=C3=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- shodan/.dockerignore | 37 +++++++- shodan/.gitignore | 32 ++++++- shodan/README.md | 87 +++++++++++++++---- shodan/pyproject.toml | 1 - shodan/shodan/__main__.py | 12 +-- .../contract.py | 2 +- .../contract.py | 2 +- .../shodan/contracts/custom_query/contract.py | 2 +- .../contracts/cve_enumeration/contract.py | 4 +- .../cve_specific_watchlist/contract.py | 4 +- .../contracts/domain_discovery/contract.py | 4 +- .../contracts/host_enumeration/contract.py | 2 +- shodan/shodan/contracts/shodan_contracts.py | 4 +- shodan/shodan/injector/openaev_shodan.py | 11 ++- shodan/shodan/services/client_api.py | 3 +- shodan/shodan/services/utils.py | 46 +++++----- 16 files changed, 189 insertions(+), 64 deletions(-) diff --git a/shodan/.dockerignore b/shodan/.dockerignore index 15e202d9..8fe47e02 100644 --- a/shodan/.dockerignore +++ b/shodan/.dockerignore @@ -1,4 +1,37 @@ +# Git +.git +.git* +.gitignore + +# Python cache +__pycache__/ +*.py[cod] + +# Virtual environments +.venv/ +venv/ +env/ + +# Build / packaging +build/ +dist/ +*.egg-info/ + +# Environment files +.env config.yml -shodan/__pycache__ -__pycache__ +# Logs +logs/ + +# Tests & coverage +.pytest_cache/ +.coverage +htmlcov/ + +# IDE +.idea/ +.vscode/ + +# OS +.DS_Store \ No newline at end of file diff --git a/shodan/.gitignore b/shodan/.gitignore index b93baaf2..f42e47d8 100644 --- a/shodan/.gitignore +++ b/shodan/.gitignore @@ -1,2 +1,32 @@ +# Python cache +__pycache__/ +*.py[cod] + +# Virtual environments +.venv/ +venv/ +env/ + +# Build / packaging +build/ +dist/ +*.egg-info/ + +# Environment files +.env config.yml -__pycache__ \ No newline at end of file + +# Logs +logs/ + +# Tests & coverage +.pytest_cache/ +.coverage +htmlcov/ + +# IDE +.idea/ +.vscode/ + +# OS +.DS_Store \ No newline at end of file diff --git a/shodan/README.md b/shodan/README.md index 1fdd92ee..2eda38ba 100644 --- a/shodan/README.md +++ b/shodan/README.md @@ -16,6 +16,7 @@ - [Fields available in manual mode by contract](#fields-available-in-manual-mode-by-contract) - [Output Trace Message](#output-trace-message) - [Auto-Create Assets](#auto-create-assets) + - [Rate Limiting and Retry](#rate-limiting-and-retry) - [Resources](#resources) --- @@ -207,13 +208,20 @@ The table below summarizes the installation and run commands for different workf For the Shodan injector, we have 7 contracts available. +- Specific behavior for all contracts: + - For each contract that has the hostnames and organization fields + - For each hostname entered in the hostname field, a dedicated API call will be made with its associated wildcard. + - Example: `hostname:filigran.io,*.filigran.io` (wildcard) + - If left empty, the organization is automatically derived from each hostname. Otherwise, the specified organization is applied to all hostnames. + - Once all calls have been made, a final API call is made to retrieve user information, including the remaining quota. + ### Supported Contracts - Cloud Provider Asset Discovery - Critical Ports And Exposed Admin Interface - Custom Query - CVE Enumeration -- CVE Specific Watchlist (The only contract that requires a plan only available to academic users, Small Business API subscribers, and higher.) +- CVE Specific Watchlist (The only contract that requires a plan Shodan only available to academic users, Small Business API subscribers, and higher.) - Domain Discovery - Host Enumeration @@ -221,7 +229,7 @@ For the Shodan injector, we have 7 contracts available. Targets are selected based on the `target_selector` field. -#### If target type is **Assets**: +#### If target type is **Assets** / **Asset-Groups**: (Currently disabled) | Selected Property | Uses Asset Field | |-------------------|----------------------------| @@ -237,35 +245,73 @@ Direct values separated by commas or spaces between IP addresses or hostnames ar ### Fields available in manual mode by contract - Cloud Provider Asset Discovery (Search Shodan Endpoint: `/shodan/host/search`) - - Cloud Provider (default: `Google,Microsoft,Amazon,Azure`) - mandatory - - Hostname - mandatory - - Organization - Optional (default: `hostname value is used`) + - The `Cloud Provider` field must contain one or more cloud providers, separated by commas. + - The `Hostname` field must contain one or more hostnames, separated by commas. + - The `Organization` field must contain one or more organizations, separated by commas. + + | Field | Mandatory | Default / Notes | + |----------------|-----------|---------------------------------------| + | Cloud Provider | Yes | `Google,Microsoft,Amazon,Azure` | + | Hostname | Yes | / | + | Organization | No | If empty, the hostname value is used. | - Critical Ports And Exposed Admin Interface (Search Shodan Endpoint: `/shodan/host/search`) - - Port (default: `20,21,22,23,25,53,80,110,111,135,139,143,443,445,993,995,1723,3306,3389,5900,8080`) - mandatory - - Hostname - mandatory - - Organization - Optional (default: `hostname value is used`) + - The `Port` field must contain one or more ports, separated by commas. + - The `Hostname` field must contain one or more hostnames, separated by commas. + - The `Organization` field must contain one or more organizations, separated by commas. + + | Field | Mandatory | Default / Notes | + |--------------|-------------|-------------------------------------------------------------------------------------| + | Port | Yes | `20,21,22,23,25,53,80,110,111,135,139,143,443,445,993,995,1723,3306,3389,5900,8080` | + | Hostname | Yes | / | + | Organization | No | If empty, the hostname value is used. | - Custom Query (Endpoint: `your custom endpoint`) - - HTTP Method (choices: `GET, POST, PUT, DELETE`, default: `GET`) - mandatory - - Custom Query - mandatory + - See the Shodan documentation: https://developer.shodan.io/api + + | Field | Mandatory | Default / Notes | + |--------------|-----------|-----------------------------------------| + | HTTP Method | Yes | `GET` (choices: GET, POST, PUT, DELETE) | + | Custom Query | Yes | / | - CVE Enumeration (Search Shodan Endpoint: `/shodan/host/search`) - - Hostname - mandatory - - Organization - Optional (default: `hostname value is used`) + - The `Hostname` field must contain one or more hostnames, separated by commas. + - The `Organization` field must contain one or more organizations, separated by commas. + + | Field | Mandatory | Default / Notes | + |--------------|-----------|---------------------------------------| + | Hostname | Yes | / | + | Organization | No | If empty, the hostname value is used. | - CVE Specific Watchlist (Search Shodan Endpoint: `/shodan/host/search`) - - Vulnerability - mandatory - - Hostname - mandatory - - Organization - Optional (default: `hostname value is used`) + - Warning : The only contract that requires a plan Shodan only available to academic users, Small Business API subscribers, and higher. + - The `Vulnerability` field must contain one or more specific CVEs. + - The `Hostname` field must contain one or more hostnames, separated by commas. + - The `Organization` field must contain one or more organizations, separated by commas. + + | Field | Mandatory | Default / Notes | + |---------------|-----------|---------------------------------------| + | Vulnerability | Yes | / | + | Hostname | Yes | / | + | Organization | No | If empty, the hostname value is used. | - Domain Discovery (Search Shodan Endpoint: `/shodan/host/search`) - - Hostname - mandatory - - Organization - Optional (default: `hostname value is used`) + - The `Hostname` field must contain one or more hostnames, separated by commas. + - The `Organization` field must contain one or more organizations, separated by commas. + + | Field | Mandatory | Default / Notes | + |--------------|-----------|---------------------------------------| + | Hostname | Yes | / | + | Organization | No | If empty, the hostname value is used. | - Host Enumeration (Host Information Endpoint: `/shodan/host/{ip}`) - - Host - mandatory + - The `Host` field must contain one or more valid IPv4 addresses. + + | Field | Mandatory | Default / Notes | + |---------|-----------|-----------------| + | Host | Yes | / | +--- ### Output Trace Message The injector captures the fields filled in by the user and analyzes the JSON output of the Shodan response, @@ -278,11 +324,14 @@ then returns several sections in the report if successful: - Call Success – List of successful calls + As well as various other relevant information related to API calls. - Call Failed – List of failed calls + As well as various other relevant information related to API calls. - **Section Table** – Visually displays the details of each call, based on the configuration defined for the contract in the injector. -- **OPTIONAL** - JSON return of the response directly (In the case of a "custom query", we return the JSON directly rather than the table section.) +- **Section JSON** – JSON return of the response directly (In the case of a "custom query", we return the JSON directly rather than the table section.) ### Auto-Create Assets - Feature currently under development +### Rate Limiting and Retry +- WIP + --- ## Resources diff --git a/shodan/pyproject.toml b/shodan/pyproject.toml index 45631ef7..d2aa6179 100644 --- a/shodan/pyproject.toml +++ b/shodan/pyproject.toml @@ -46,7 +46,6 @@ dev = [ "mypy~=1.18.2", # Type validator "pip_audit~=2.9.0", # Security checker "pre-commit~=4.3.0", # Git hooks - "flake8~=7.3.0", # Linter "types-PyYAML~=6.0.12", # stubs for untyped module ] test = [ diff --git a/shodan/shodan/__main__.py b/shodan/shodan/__main__.py index a771e9fc..76673fce 100644 --- a/shodan/shodan/__main__.py +++ b/shodan/shodan/__main__.py @@ -22,25 +22,27 @@ def main() -> None: logger = logging.getLogger(__name__) try: - logger.info(f"{LOG_PREFIX} Starting Shodan Injector...") - - # Injector Config + # Loading injector configuration config = ConfigLoader() - # Prepare config for Helper + # Build Shodan contracts and adapt config for the helper shodan_contracts = ShodanContracts(config).contracts() config_helper_adapter = config.to_config_injector_helper_adapter( contracts=shodan_contracts ) + + # Load the injector icon for the helper icon_path = Path(__file__).parent / "img" / "icon-shodan.png" icon_bytes = icon_path.read_bytes() + # Instantiate the OpenAEV injector helper helper = OpenAEVInjectorHelper(config=config_helper_adapter, icon=icon_bytes) logger.info( - f"{LOG_PREFIX} The initialization of the Shodan injector configuration was successful." + f"{LOG_PREFIX} - Shodan injector configuration initialized successfully." ) + # Start the Shodan injector injector = ShodanInjector(config, helper) injector.start() diff --git a/shodan/shodan/contracts/cloud_provider_asset_discovery/contract.py b/shodan/shodan/contracts/cloud_provider_asset_discovery/contract.py index f5419487..b700a05f 100644 --- a/shodan/shodan/contracts/cloud_provider_asset_discovery/contract.py +++ b/shodan/shodan/contracts/cloud_provider_asset_discovery/contract.py @@ -110,7 +110,7 @@ def output_trace_config(): "show_tables": { "is_active": True, "show_lines": True, - "max_display_by_cell": 4, + "max_display_by_cell": 10, "show_index": { "is_active": False, "index_start": 1, diff --git a/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/contract.py b/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/contract.py index 93e821aa..08100cb9 100644 --- a/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/contract.py +++ b/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/contract.py @@ -100,7 +100,7 @@ def output_trace_config(): "show_tables": { "is_active": True, "show_lines": True, - "max_display_by_cell": 4, + "max_display_by_cell": 10, "show_index": { "is_active": False, "index_start": 1, diff --git a/shodan/shodan/contracts/custom_query/contract.py b/shodan/shodan/contracts/custom_query/contract.py index c2914176..a244f713 100644 --- a/shodan/shodan/contracts/custom_query/contract.py +++ b/shodan/shodan/contracts/custom_query/contract.py @@ -77,7 +77,7 @@ def output_trace_config(): "show_tables": { "is_active": False, "show_lines": True, - "max_display_by_cell": 4, + "max_display_by_cell": 10, "show_index": { "is_active": False, "index_start": 1, diff --git a/shodan/shodan/contracts/cve_enumeration/contract.py b/shodan/shodan/contracts/cve_enumeration/contract.py index ece5f12b..b7d85189 100644 --- a/shodan/shodan/contracts/cve_enumeration/contract.py +++ b/shodan/shodan/contracts/cve_enumeration/contract.py @@ -99,14 +99,14 @@ def output_trace_config(): "show_tables": { "is_active": True, "show_lines": True, - "max_display_by_cell": 4, + "max_display_by_cell": 10, "show_index": { "is_active": False, "index_start": 1, }, }, "show_separator": { - "is_active": True, + "is_active": False, }, "show_json": { "is_active": False, diff --git a/shodan/shodan/contracts/cve_specific_watchlist/contract.py b/shodan/shodan/contracts/cve_specific_watchlist/contract.py index 991bd69c..926351e5 100644 --- a/shodan/shodan/contracts/cve_specific_watchlist/contract.py +++ b/shodan/shodan/contracts/cve_specific_watchlist/contract.py @@ -99,14 +99,14 @@ def output_trace_config(): "show_tables": { "is_active": True, "show_lines": True, - "max_display_by_cell": 4, + "max_display_by_cell": 10, "show_index": { "is_active": False, "index_start": 1, }, }, "show_separator": { - "is_active": True, + "is_active": False, }, "show_json": { "is_active": False, diff --git a/shodan/shodan/contracts/domain_discovery/contract.py b/shodan/shodan/contracts/domain_discovery/contract.py index 939b86de..797f4bb9 100644 --- a/shodan/shodan/contracts/domain_discovery/contract.py +++ b/shodan/shodan/contracts/domain_discovery/contract.py @@ -99,14 +99,14 @@ def output_trace_config(): "show_tables": { "is_active": True, "show_lines": True, - "max_display_by_cell": 4, + "max_display_by_cell": 10, "show_index": { "is_active": False, "index_start": 1, }, }, "show_separator": { - "is_active": True, + "is_active": False, }, "show_json": { "is_active": False, diff --git a/shodan/shodan/contracts/host_enumeration/contract.py b/shodan/shodan/contracts/host_enumeration/contract.py index 0e09ccf6..a81e3690 100644 --- a/shodan/shodan/contracts/host_enumeration/contract.py +++ b/shodan/shodan/contracts/host_enumeration/contract.py @@ -94,7 +94,7 @@ def output_trace_config(): "show_tables": { "is_active": True, "show_lines": True, - "max_display_by_cell": 4, + "max_display_by_cell": 10, "show_index": { "is_active": False, "index_start": 1, diff --git a/shodan/shodan/contracts/shodan_contracts.py b/shodan/shodan/contracts/shodan_contracts.py index 8ef1b1e1..a1b4a02b 100644 --- a/shodan/shodan/contracts/shodan_contracts.py +++ b/shodan/shodan/contracts/shodan_contracts.py @@ -282,7 +282,7 @@ def _build_contract( def contracts(self): - selector_default = TargetSelectorField.ASSET_GROUPS.key + selector_default = "only_manual" shodan_contract_definitions = [ ( @@ -295,7 +295,7 @@ def contracts(self): CriticalPortsAndExposedAdminInterface, selector_default, ), - (ShodanContractId.CUSTOM_QUERY, CustomQuery, "only_manual"), + (ShodanContractId.CUSTOM_QUERY, CustomQuery, selector_default), (ShodanContractId.CVE_ENUMERATION, CVEEnumeration, selector_default), ( ShodanContractId.CVE_SPECIFIC_WATCHLIST, diff --git a/shodan/shodan/injector/openaev_shodan.py b/shodan/shodan/injector/openaev_shodan.py index 710ed800..27d97792 100644 --- a/shodan/shodan/injector/openaev_shodan.py +++ b/shodan/shodan/injector/openaev_shodan.py @@ -47,10 +47,11 @@ def _prepare_output_message( "HOST_ENUMERATION": HostEnumeration.output_trace_config(), } if contract_name not in output_trace_config: - raise ValueError( + self.helper.injector_logger.error( f"{LOG_PREFIX} - The contract name is unknown.", {"contract_name": contract_name}, ) + raise ValueError(f"{LOG_PREFIX} - The contract name is unknown.") contract_output_trace_config = output_trace_config.get(contract_name) @@ -120,7 +121,13 @@ def _shodan_execution(self, data): return output_json, output_message else: - return None, None + self.helper.injector_logger.error( + f"{LOG_PREFIX} - Invalid selector key, expected keys 'manual', 'assets', or 'asset-groups", + {"selector_key": selector_key}, + ) + raise ValueError( + f"{LOG_PREFIX} - Invalid selector key, expected keys 'manual', 'assets', or 'asset-groups'." + ) def process_message(self, data: dict) -> None: # Initialization to get the current start utc iso format. diff --git a/shodan/shodan/services/client_api.py b/shodan/shodan/services/client_api.py index e9eff10e..41cca127 100644 --- a/shodan/shodan/services/client_api.py +++ b/shodan/shodan/services/client_api.py @@ -364,10 +364,11 @@ def process_shodan_search( contract = contract_handler.get(contract_id) if not contract: - raise ValueError( + self.helper.injector_logger.error( f"{LOG_PREFIX} - The contract ID is invalid.", {"contract_id": contract_id}, ) + raise ValueError(f"{LOG_PREFIX} - The contract ID is invalid.") results = contract(inject_content) shodan_credit_user = self._get_user_info() diff --git a/shodan/shodan/services/utils.py b/shodan/shodan/services/utils.py index 672cf342..2fc4ab0d 100644 --- a/shodan/shodan/services/utils.py +++ b/shodan/shodan/services/utils.py @@ -309,7 +309,7 @@ def _make_section_external_api( ) if not (options_show_sections and options_show_sec_external_api): - return None + return None, None sec_external_api_icon = self._get_output_icon( self._get_trace_config( @@ -507,7 +507,7 @@ def _organizer_row( filtered = [ str(v) for v in value if v not in (None, "") ] - cell = ",".join(filtered) if filtered else "-" + cell = ", ".join(filtered) if filtered else "-" else: val = values[row_idx] if row_idx < len(values) else None cell = str(val) if val not in (None, "") else "-" @@ -656,6 +656,8 @@ def _make_tables( ) tables = self._get_trace_config(output_trace_config, "tables", default=[]) + + tables_rendering = [] for table_config in tables: header_icon = self._get_output_icon( self._get_trace_config(table_config, "header.icon", default="SEARCH") @@ -666,7 +668,8 @@ def _make_tables( search_entity = self._get_trace_config( table_config, "config.search_entity", default=None ) - tables_final = [] + + table_rendering = [] for data_table in data_tables: result = data_table.get("result") if header_title: @@ -726,10 +729,10 @@ def _make_tables( for row_with_limit in rows_with_limit_cell: table.add_row(*row_with_limit) - tables_final.append(table) + table_rendering.append(table) - return tables_final - return [] + tables_rendering.extend(table_rendering) + return tables_rendering def generate_output_message( self, @@ -790,21 +793,22 @@ def generate_output_message( renderables.append(separator) renderables.append(output_sections_client_api) - # Output Tables - output_tables = self._make_tables( - output_trace_config=output_trace_config, - auto_create_assets=auto_create_assets, - data_tables=results_success_details, - ) - for output_table in output_tables: - if output_table: - renderables.append(Text("")) - separator = self._make_separator( - output_trace_config=output_trace_config - ) - if separator: - renderables.append(separator) - renderables.append(output_table) + if results_success_details: + # Output Tables + output_tables = self._make_tables( + output_trace_config=output_trace_config, + auto_create_assets=auto_create_assets, + data_tables=results_success_details, + ) + for output_table in output_tables: + if output_table: + renderables.append(Text("")) + separator = self._make_separator( + output_trace_config=output_trace_config + ) + if separator: + renderables.append(separator) + renderables.append(output_table) # Output JSON output_json = self._make_json( From dc2ce7ea396a6a3ecbeee0e2ccba7836d2aa14fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20BASLER?= Date: Tue, 3 Feb 2026 10:02:36 +0100 Subject: [PATCH 10/13] [Shodan] Change of contract name (Host Enumeration -> IP Enumeration), addition of the rate limiting and retry section in the README. --- shodan/README.md | 58 ++++++++++++++++--- shodan/shodan/contracts/__init__.py | 4 +- .../contracts/host_enumeration/__init__.py | 3 - .../contracts/ip_enumeration/__init__.py | 3 + .../contract.py | 24 ++++---- shodan/shodan/contracts/shodan_contracts.py | 8 +-- shodan/shodan/injector/__init__.py | 4 +- shodan/shodan/injector/openaev_shodan.py | 4 +- shodan/shodan/services/client_api.py | 32 +++++----- shodan/shodan/services/utils.py | 4 +- 10 files changed, 92 insertions(+), 52 deletions(-) delete mode 100644 shodan/shodan/contracts/host_enumeration/__init__.py create mode 100644 shodan/shodan/contracts/ip_enumeration/__init__.py rename shodan/shodan/contracts/{host_enumeration => ip_enumeration}/contract.py (90%) diff --git a/shodan/README.md b/shodan/README.md index 2eda38ba..57436357 100644 --- a/shodan/README.md +++ b/shodan/README.md @@ -223,7 +223,7 @@ For the Shodan injector, we have 7 contracts available. - CVE Enumeration - CVE Specific Watchlist (The only contract that requires a plan Shodan only available to academic users, Small Business API subscribers, and higher.) - Domain Discovery -- Host Enumeration +- IP Enumeration ### Target Selection @@ -304,12 +304,12 @@ Direct values separated by commas or spaces between IP addresses or hostnames ar | Hostname | Yes | / | | Organization | No | If empty, the hostname value is used. | -- Host Enumeration (Host Information Endpoint: `/shodan/host/{ip}`) - - The `Host` field must contain one or more valid IPv4 addresses. +- IP Enumeration (Search Shodan Endpoint: `/shodan/host/search`) + - The `IP` field must contain one or more valid IPv4 addresses. - | Field | Mandatory | Default / Notes | - |---------|-----------|-----------------| - | Host | Yes | / | + | Field | Mandatory | Default / Notes | + |-------|-----------|-----------------| + | IP | Yes | / | --- @@ -330,7 +330,51 @@ then returns several sections in the report if successful: - Feature currently under development ### Rate Limiting and Retry -- WIP + +#### Overview + +The Shodan API does not publicly document strict rate-limiting rules or request thresholds. +However, to ensure stable and reliable interactions with the API, a rate-limiting and retry mechanism is implemented. + +These mechanisms aim to: +- Smooth the flow of outgoing requests, +- Reduce the risk of HTTP 429 Too Many Requests responses, +- Handle transient failures, +- Improve the clarity of error reporting in output traces message. + +#### Rate limiting + +A rate-limiting mechanism is applied to control the frequency of API calls. +Requests are deliberately paced to avoid sending bursts of traffic that could exceed Shodan’s internal limits or trigger protective measures. + +This approach helps: +- Prevent temporary blocking or throttling, +- Ensure consistent behavior across multiple API calls, +- Maintain predictable execution when processing multiple hostnames or IPs. + +There are two environment variables for controlling the flow: +- `SHODAN_API_LEAKY_BUCKET_RATE` - Controls the rate at which API requests are allowed to be executed. +- `SHODAN_API_LEAKY_BUCKET_CAPACITY` - Defines the maximum burst size allowed before requests are delayed. + +#### Retry strategy + +A retry mechanism is applied to handle failed API requests. +Requests are retried after a short delay to mitigate transient failures and improve execution stability. + +This approach helps: +- Automatically retry failed requests up to a limited number of attempts. +- Introduce delays between retries to avoid repeated immediate failures. + +There are two environment variables for controlling the flow: +- `SHODAN_API_RETRY` - Defines the maximum number of retry attempts for a failed request. +- `SHODAN_API_BACKOFF` - Specifies the maximum delay (in seconds) applied between retry attempts, using an exponential backoff strategy. + +#### Execution trace and feedback + +Each API call is tracked individually. +The success or failure of each request is reported in the external API section of the output trace message. + +Once all API calls related to the host name or IP address have been made, a final API call is executed to retrieve user information, including the remaining API quota. --- diff --git a/shodan/shodan/contracts/__init__.py b/shodan/shodan/contracts/__init__.py index a7aa439c..dd5329c7 100644 --- a/shodan/shodan/contracts/__init__.py +++ b/shodan/shodan/contracts/__init__.py @@ -6,7 +6,7 @@ from .cve_enumeration import CVEEnumeration from .cve_specific_watchlist import CVESpecificWatchlist from .domain_discovery import DomainDiscovery -from .host_enumeration import HostEnumeration +from .ip_enumeration import IPEnumeration from .shodan_contracts import InjectorKey, ShodanContractId __all__ = [ @@ -16,7 +16,7 @@ "CVEEnumeration", "CVESpecificWatchlist", "DomainDiscovery", - "HostEnumeration", + "IPEnumeration", "InjectorKey", "ShodanContractId", ] diff --git a/shodan/shodan/contracts/host_enumeration/__init__.py b/shodan/shodan/contracts/host_enumeration/__init__.py deleted file mode 100644 index b0dc1c90..00000000 --- a/shodan/shodan/contracts/host_enumeration/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .contract import HostEnumeration - -__all__ = ["HostEnumeration"] diff --git a/shodan/shodan/contracts/ip_enumeration/__init__.py b/shodan/shodan/contracts/ip_enumeration/__init__.py new file mode 100644 index 00000000..a9144ff9 --- /dev/null +++ b/shodan/shodan/contracts/ip_enumeration/__init__.py @@ -0,0 +1,3 @@ +from .contract import IPEnumeration + +__all__ = ["IPEnumeration"] diff --git a/shodan/shodan/contracts/host_enumeration/contract.py b/shodan/shodan/contracts/ip_enumeration/contract.py similarity index 90% rename from shodan/shodan/contracts/host_enumeration/contract.py rename to shodan/shodan/contracts/ip_enumeration/contract.py index a81e3690..0b26fcb8 100644 --- a/shodan/shodan/contracts/host_enumeration/contract.py +++ b/shodan/shodan/contracts/ip_enumeration/contract.py @@ -9,13 +9,13 @@ ) -class HostEnumeration: +class IPEnumeration: @staticmethod def output_trace_config(): return { "header": { - "title": "SHODAN - HOST ENUMERATION", + "title": "SHODAN - IP ENUMERATION", "subtitle": None, }, "sections_config": { @@ -42,7 +42,7 @@ def output_trace_config(): "call_success": { "icon": "SUCCESS", "title": "Call Success", - "count_at_path": "data", + "count_at_path": "matches", }, "call_failed": { "icon": "FAILED", @@ -60,19 +60,19 @@ def output_trace_config(): "columns": [ { "title": "Port", - "path": "data.port", + "path": "matches.port", "mode": "single", }, { "title": "Hostnames", - "path": "hostnames", - "mode": "repeat", + "path": "matches.hostnames", + "mode": "align_to_single", }, { "title": "Vulnerabilities (score)", - "path": "data.vulns.*", + "path": "matches.vulns.*", "use_key": True, - "extra": "data.vulns.*.cvss", + "extra": "matches.vulns.*.cvss", "mode": "align_to_single", }, ], @@ -130,8 +130,8 @@ def contract_with_specific_fields( specific_fields = [ ContractText( - key="host", - label="Host", + key="ip", + label="IP", **(mandatory_conditions | visible_conditions), ), ] @@ -164,8 +164,8 @@ def contract( contract_id=contract_id, config=contract_config, label={ - SupportedLanguage.en: "Shodan - Host Enumeration", - SupportedLanguage.fr: "Shodan - Énumération des hôtes", + SupportedLanguage.en: "Shodan - IP Enumeration", + SupportedLanguage.fr: "Shodan - Énumération des IP", }, fields=contract_with_specific_fields, outputs=contract_with_specific_outputs, diff --git a/shodan/shodan/contracts/shodan_contracts.py b/shodan/shodan/contracts/shodan_contracts.py index a1b4a02b..941db96f 100644 --- a/shodan/shodan/contracts/shodan_contracts.py +++ b/shodan/shodan/contracts/shodan_contracts.py @@ -24,7 +24,7 @@ CVEEnumeration, CVESpecificWatchlist, DomainDiscovery, - HostEnumeration, + IPEnumeration, ) from shodan.models import ConfigLoader @@ -46,7 +46,7 @@ class ShodanContractId(StrEnum): CVE_ENUMERATION = "8cdccd58-78ed-4e17-be2e-7683ec611569" CVE_SPECIFIC_WATCHLIST = "462087b4-8012-4e21-9575-b9c854ef5811" DOMAIN_DISCOVERY = "faf73809-1128-4192-aa90-a08828f8ace5" - HOST_ENUMERATION = "dc6b8b73-09dd-4388-b7cc-108bf16d26cd" + IP_ENUMERATION = "dc6b8b73-09dd-4388-b7cc-108bf16d26cd" @dataclass @@ -263,7 +263,7 @@ def _build_contract( | CVEEnumeration | CVESpecificWatchlist | DomainDiscovery - | HostEnumeration + | IPEnumeration ), contract_selector_default: str, ) -> Contract: @@ -303,7 +303,7 @@ def contracts(self): selector_default, ), (ShodanContractId.DOMAIN_DISCOVERY, DomainDiscovery, selector_default), - (ShodanContractId.HOST_ENUMERATION, HostEnumeration, selector_default), + (ShodanContractId.IP_ENUMERATION, IPEnumeration, selector_default), ] contracts = [ diff --git a/shodan/shodan/injector/__init__.py b/shodan/shodan/injector/__init__.py index 929d68ad..b8d32ab3 100644 --- a/shodan/shodan/injector/__init__.py +++ b/shodan/shodan/injector/__init__.py @@ -5,7 +5,7 @@ CVEEnumeration, CVESpecificWatchlist, DomainDiscovery, - HostEnumeration, + IPEnumeration, ) __all__ = [ @@ -15,5 +15,5 @@ "CVEEnumeration", "CVESpecificWatchlist", "DomainDiscovery", - "HostEnumeration", + "IPEnumeration", ] diff --git a/shodan/shodan/injector/openaev_shodan.py b/shodan/shodan/injector/openaev_shodan.py index 27d97792..49e4c176 100644 --- a/shodan/shodan/injector/openaev_shodan.py +++ b/shodan/shodan/injector/openaev_shodan.py @@ -10,7 +10,7 @@ CVEEnumeration, CVESpecificWatchlist, DomainDiscovery, - HostEnumeration, + IPEnumeration, InjectorKey, ShodanContractId, ) @@ -44,7 +44,7 @@ def _prepare_output_message( "CVE_ENUMERATION": CVEEnumeration.output_trace_config(), "CVE_SPECIFIC_WATCHLIST": CVESpecificWatchlist.output_trace_config(), "DOMAIN_DISCOVERY": DomainDiscovery.output_trace_config(), - "HOST_ENUMERATION": HostEnumeration.output_trace_config(), + "IP_ENUMERATION": IPEnumeration.output_trace_config(), } if contract_name not in output_trace_config: self.helper.injector_logger.error( diff --git a/shodan/shodan/services/client_api.py b/shodan/shodan/services/client_api.py index 41cca127..7cf47d01 100644 --- a/shodan/shodan/services/client_api.py +++ b/shodan/shodan/services/client_api.py @@ -22,7 +22,6 @@ class MissingRequiredFieldError(ValueError): class ShodanRestAPIDefinition: http_method: str endpoint: str - path_parameter: bool = False class ShodanRestAPI(Enum): @@ -30,11 +29,6 @@ class ShodanRestAPI(Enum): http_method="GET", endpoint="shodan/host/search", ) - HOST_INFORMATION = ShodanRestAPIDefinition( - http_method="GET", - path_parameter=True, - endpoint="shodan/host/{target}", - ) API_PLAN_INFORMATION = ShodanRestAPIDefinition( http_method="GET", endpoint="api-info" ) @@ -240,16 +234,22 @@ def _get_domain_discovery(self, inject_content): filters_template=filters, ) - # CONTRACT - HOST ENUMERATION - def _get_host_enumeration(self, inject_content): - host = inject_content.get("host") - if not host: + # CONTRACT - IP ENUMERATION + def _get_ip_enumeration(self, inject_content): + ip = inject_content.get("ip") + if not ip: raise MissingRequiredFieldError( - f"{LOG_PREFIX} - The 'host' field is required and cannot be empty." + f"{LOG_PREFIX} - The 'ip' field is required and cannot be empty." ) + + filters = { + "ip": ("{target}", "end"), + } + return self._process_request( - raw_input=host, - request_api=ShodanRestAPI.HOST_INFORMATION, + raw_input=ip, + request_api=ShodanRestAPI.SEARCH_SHODAN, + filters_template=filters, ) def _process_request( @@ -265,20 +265,16 @@ def _process_request( targets = [raw_input] http_method = http_method_custom_query endpoint_template = raw_input - path_parameter = None else: targets = self._split_target(raw_input) http_method = request_api.value.http_method endpoint_template = request_api.value.endpoint - path_parameter = request_api.value.path_parameter results = [] for target in targets: new_endpoint = endpoint_template query_params = None encoded_for_shodan = None - if path_parameter: - new_endpoint = endpoint_template.format(target=target) if filters_template: filters = {} @@ -359,7 +355,7 @@ def process_shodan_search( ShodanContractId.CRITICAL_PORTS_AND_EXPOSED_ADMIN_INTERFACE: self._get_critical_ports_and_exposed_admin_interface, ShodanContractId.CUSTOM_QUERY: self._get_custom_query, ShodanContractId.DOMAIN_DISCOVERY: self._get_domain_discovery, - ShodanContractId.HOST_ENUMERATION: self._get_host_enumeration, + ShodanContractId.IP_ENUMERATION: self._get_ip_enumeration, } contract = contract_handler.get(contract_id) diff --git a/shodan/shodan/services/utils.py b/shodan/shodan/services/utils.py index 2fc4ab0d..beccbb23 100644 --- a/shodan/shodan/services/utils.py +++ b/shodan/shodan/services/utils.py @@ -611,11 +611,11 @@ def _rows_with_limit_cells(rows: list[list[str]], max_display_by_cell: int | Non shown = items[:max_display_by_cell] hidden = len(items) - max_display_by_cell - main_row.append(",".join(shown)) + main_row.append(", ".join(shown)) hidden_row.append(f"...(+{hidden} hidden)") has_hidden = True else: - main_row.append(",".join(items)) + main_row.append(", ".join(items)) hidden_row.append("") else: main_row.append(cell) From 7b3f9d6a9b177cfd526d0a28515eef1f77bf1f18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20BASLER?= Date: Tue, 3 Feb 2026 10:09:18 +0100 Subject: [PATCH 11/13] [Shodan] Up linter --- shodan/shodan/injector/openaev_shodan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shodan/shodan/injector/openaev_shodan.py b/shodan/shodan/injector/openaev_shodan.py index 49e4c176..66984141 100644 --- a/shodan/shodan/injector/openaev_shodan.py +++ b/shodan/shodan/injector/openaev_shodan.py @@ -10,8 +10,8 @@ CVEEnumeration, CVESpecificWatchlist, DomainDiscovery, - IPEnumeration, InjectorKey, + IPEnumeration, ShodanContractId, ) from shodan.models import ConfigLoader From 82a987214a7e9d875ac927a0193b149702cac3dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20BASLER?= Date: Tue, 3 Feb 2026 17:34:08 +0100 Subject: [PATCH 12/13] [Shodan] Fix get_user_info, Add a no data found visual --- shodan/shodan/config.yml.sample | 2 +- shodan/shodan/services/client_api.py | 3 ++- shodan/shodan/services/utils.py | 10 ++++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/shodan/shodan/config.yml.sample b/shodan/shodan/config.yml.sample index 5371ea15..80ae8e99 100644 --- a/shodan/shodan/config.yml.sample +++ b/shodan/shodan/config.yml.sample @@ -2,7 +2,7 @@ openaev: url: 'ChangeMe' token: 'ChangeMe' -injector: +#injector: # id: 'shodan--a87488ad-2c72-4592-b429-69259d7bcef1' # name: 'Shodan' # log_level: 'error' diff --git a/shodan/shodan/services/client_api.py b/shodan/shodan/services/client_api.py index 7cf47d01..eb8663c4 100644 --- a/shodan/shodan/services/client_api.py +++ b/shodan/shodan/services/client_api.py @@ -270,6 +270,7 @@ def _process_request( http_method = request_api.value.http_method endpoint_template = request_api.value.endpoint + result=None results = [] for target in targets: new_endpoint = endpoint_template @@ -325,7 +326,7 @@ def _process_request( "response": response_filtered, } ) - return {"targets": targets, "data": results} + return result if targets == ["user_info"] else {"targets": targets, "data": results} def _request_data(self, method: str, url: str): @retry( diff --git a/shodan/shodan/services/utils.py b/shodan/shodan/services/utils.py index beccbb23..8bcdbd49 100644 --- a/shodan/shodan/services/utils.py +++ b/shodan/shodan/services/utils.py @@ -722,6 +722,16 @@ def _make_tables( column_extras, ) + if not final_rows: + tables_rendering.append( + Panel( + renderable="No data found", + title=f"Result for {result.get(search_entity) or data_table.get("data_target")}", + title_align="left", + ) + ) + continue + rows_with_limit_cell = self._rows_with_limit_cells( final_rows, max_display_by_cell ) From 903d363afe8bfaec3a061047af9e2af98c6f47bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20BASLER?= Date: Tue, 3 Feb 2026 17:45:36 +0100 Subject: [PATCH 13/13] [Shodan] Up linter --- shodan/shodan/services/client_api.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/shodan/shodan/services/client_api.py b/shodan/shodan/services/client_api.py index eb8663c4..6af23155 100644 --- a/shodan/shodan/services/client_api.py +++ b/shodan/shodan/services/client_api.py @@ -270,7 +270,7 @@ def _process_request( http_method = request_api.value.http_method endpoint_template = request_api.value.endpoint - result=None + result = None results = [] for target in targets: new_endpoint = endpoint_template @@ -326,7 +326,11 @@ def _process_request( "response": response_filtered, } ) - return result if targets == ["user_info"] else {"targets": targets, "data": results} + return ( + result + if targets == ["user_info"] + else {"targets": targets, "data": results} + ) def _request_data(self, method: str, url: str): @retry(