diff --git a/shodan/.dockerignore b/shodan/.dockerignore new file mode 100644 index 00000000..8fe47e02 --- /dev/null +++ b/shodan/.dockerignore @@ -0,0 +1,37 @@ +# Git +.git +.git* +.gitignore + +# Python cache +__pycache__/ +*.py[cod] + +# Virtual environments +.venv/ +venv/ +env/ + +# Build / packaging +build/ +dist/ +*.egg-info/ + +# Environment files +.env +config.yml + +# Logs +logs/ + +# Tests & coverage +.pytest_cache/ +.coverage +htmlcov/ + +# IDE +.idea/ +.vscode/ + +# OS +.DS_Store \ No newline at end of file diff --git a/shodan/.gitignore b/shodan/.gitignore new file mode 100644 index 00000000..f42e47d8 --- /dev/null +++ b/shodan/.gitignore @@ -0,0 +1,32 @@ +# Python cache +__pycache__/ +*.py[cod] + +# Virtual environments +.venv/ +venv/ +env/ + +# Build / packaging +build/ +dist/ +*.egg-info/ + +# Environment files +.env +config.yml + +# Logs +logs/ + +# Tests & coverage +.pytest_cache/ +.coverage +htmlcov/ + +# IDE +.idea/ +.vscode/ + +# OS +.DS_Store \ No newline at end of file diff --git a/shodan/CONTRIBUTING.md b/shodan/CONTRIBUTING.md new file mode 100644 index 00000000..6d3f6659 --- /dev/null +++ b/shodan/CONTRIBUTING.md @@ -0,0 +1 @@ +WIP \ No newline at end of file diff --git a/shodan/Dockerfile b/shodan/Dockerfile new file mode 100644 index 00000000..52773207 --- /dev/null +++ b/shodan/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.13-alpine AS build + +WORKDIR /opt/openaev-injector-shodan + +# Copy dependency files (pyproject.toml / poetry.lock / README.md) +COPY pyproject.toml poetry.lock* README.md ./ + +# Build dependencies +RUN apk add --no-cache --virtual .build-dependencies \ + build-base git libffi-dev libxml2-dev libxslt-dev \ + && pip3 install --no-cache-dir poetry==2.2.1 \ + && poetry config virtualenvs.create false \ + && poetry install --only main --extras prod --no-interaction --no-ansi --no-root \ + && apk del .build-dependencies + +# Copy the injector +COPY shodan ./shodan + +FROM python:3.13-alpine AS runner + +WORKDIR /opt/openaev-injector-shodan + +# Runtime libraries +RUN apk add --no-cache libmagic libffi libxml2 libxslt + +COPY --from=build /opt/openaev-injector-shodan/shodan ./shodan +COPY --from=build /usr/local/lib/python3.13/site-packages /usr/local/lib/python3.13/site-packages + +CMD ["python3", "-m", "shodan"] \ No newline at end of file diff --git a/shodan/README.md b/shodan/README.md new file mode 100644 index 00000000..57436357 --- /dev/null +++ b/shodan/README.md @@ -0,0 +1,392 @@ +# OpenAEV Shodan Injector + +## Table of Contents + +- [OpenAEV SHODAN Injector](#openaev-shodan-injector) + - [Prerequisites](#prerequisites) + - [Configuration variables](#configuration-variables) + - [Base OpenAEV environment variables](#base-openaev-environment-variables) + - [Base injector environment variables](#base-injector-environment-variables) + - [Base Shodan environment variables](#base-shodan-environment-variables) + - [Deployment](#deployment) + - [Docker Deployment](#docker-deployment) + - [Manual Deployment](#manual-deployment) + - [Behavior](#behavior) + - [Supported Contracts](#supported-contracts) + - [Fields available in manual mode by contract](#fields-available-in-manual-mode-by-contract) + - [Output Trace Message](#output-trace-message) + - [Auto-Create Assets](#auto-create-assets) + - [Rate Limiting and Retry](#rate-limiting-and-retry) + - [Resources](#resources) + +--- + +## Prerequisites + +This injector uses [Shodan](https://www.shodan.io/) to collect information about exposed assets. + +To operate correctly, the injector requires: + +- A **valid Shodan API key**, which must be provided through an environment variable (`SHODAN_API_KEY`) or via the `config.yml` file. +- You can create an account on this page: https://account.shodan.io/register + +In addition, for proper integration with the OpenAEV platform: + +- It **must be able to reach the OpenAEV platform** via the URL you provide (through `OPENAEV_URL` or `config.yml`). +- It **must be able to reach the RabbitMQ broker** used by the OpenAEV platform. + +Depending on your deployment method: + +- When using the **Docker deployment**, ensure the Shodan API key is correctly injected into the container environment. +- When running manually (e.g., in development), the Shodan API key must be available in your local environment. + +--- + +## Configuration variables + +Configuration is provided either through environment variables (Docker) or a config +file (`config.yml`, manual). + +--- + +### Base OpenAEV environment variables + +| Parameter | config.yml | Docker environment variable | Mandatory | Description | +|---------------|------------|-----------------------------|-----------|------------------------------------------------------| +| OpenAEV URL | url | `OPENAEV_URL` | Yes | The URL of the OpenAEV platform. | +| OpenAEV Token | token | `OPENAEV_TOKEN` | Yes | The default admin token set in the OpenAEV platform. | + +### Base injector environment variables + +| Parameter | config.yml | Docker environment variable | Default | Mandatory | Description | +|---------------|-------------|------------------------------|------------------|-----------|----------------------------------------------------------------------------------------| +| Injector ID | id | `INJECTOR_ID` | `shodan--` | No | A unique `UUIDv4` identifier for this injector instance. | +| Injector Name | name | `INJECTOR_NAME` | `Shodan` | No | Name of the injector. | +| Log Level | log_level | `INJECTOR_LOG_LEVEL` | `error` | No | Determines the verbosity of the logs. Options: `debug`, `info`, `warning`, or `error`. | + + +### Base Shodan environment variables + +| Parameter | config.yml | Docker environment variable | Default | Mandatory | Description | +|----------------------------------|---------------------------|------------------------------------|-------------------------|-----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Shodan Base URL | token | `SHODAN_BASE_URL` | `https://api.shodan.io` | No | This is the base URL for the Shodan API. | +| Shodan API Key | api_key | `SHODAN_API_KEY` | / | Yes | This is the API key for the Shodan API. | +| Shodan API leaky bucket rate | api_leaky_bucket_rate | `SHODAN_API_LEAKY_BUCKET_RATE` | `10` | No | Bucket refill rate (in tokens per second). Controls the rate at which API calls are allowed. For example, a rate of 10 means that 10 calls can be made per second, if the bucket is not empty. | +| Shodan API leaky bucket capacity | api_leaky_bucket_capacity | `SHODAN_API_LEAKY_BUCKET_CAPACITY` | `10` | No | Maximum bucket capacity (in tokens). Defines the number of calls that can be made immediately in a burst. Once the bucket is empty, it refills at the rate defined by 'api_leaky_bucket_rate'. | +| Shodan API Retry | api_retry | `SHODAN_API_RETRY` | `5` | No | Maximum number of attempts (including the initial request) in case of API failure. | +| Shodan API backoff | api_backoff | `SHODAN_API_BACKOFF` | `PT30S` | No | Maximum exponential backoff delay between retry attempts (ISO 8601 duration format). | + +--- + +## Deployment + +--- + +### Docker Deployment + +Build the Docker image using the provided `Dockerfile`: + +```bash +docker build -t openaev/injector-shodan:latest . +``` + +Edit the `docker-compose.yml` file with your OpenAEV configuration, then start the container: + +```bash +docker compose up -d +``` + +> ✅ The Docker image already includes the Shodan injector and its runtime dependencies. +Only a valid `SHODAN_API_KEY` and access to the OpenAEV platform are required at runtime. + +--- + +### Manual Deployment + +This injector can be run manually in different modes, depending on whether you want **production**, **current**, or **development/testing**. + +--- + +#### Prerequisites + +- Python 3.11+ installed +- Poetry installed (https://python-poetry.org/) / uv / pip +- A valid `SHODAN_API_KEY` set as environment variable or in `config.yml` +- Access to the OpenAEV platform (`OPENAEV_URL` / `OPENAEV_TOKEN`) and RabbitMQ + + +#### Install and Run with Pip + +Use pip if you prefer a classic Python workflow or for **development/testing with local dependencies**. + +```bash +# Create a virtual environment +python -m venv .venv + +# Activate the venv +# Linux / macOS +source .venv/bin/activate +# Windows - Git Bash +source .venv/Scripts/activate + +# Install prod dependencies (stable Pyoaev release) +pip install .[prod] + +# Install current release from Git / latest +pip install .[current] + +# Install dependencies (Dev + Test includes local client-python) +pip install -r requirements.txt +``` + +Run the injector: +The injector can be started using either the Python module or the console entry point, depending on how it was installed. + +Run as a Python module (recommended for Docker and simple setups) +```bash +python -m shodan +``` + +Run using the console entry point +```bash +# Requires the package to be installed and the virtual environment to be active +ShodanInjector +``` + +**Why use pip:** + +- Manages local modifiable dependencies (`../../client-python`) that Poetry cannot resolve automatically while complying with PEP. +- Handles extras and entry points (ShodanInjector) defined in `pyproject.toml` +- Installs development/test extras (`.[dev,test]`) in a Poetry-compatible venv. + +#### Install and Run with Poetry +Use Poetry for production or current releases, or to manage dependencies automatically in an isolated virtual environment. + +```bash +# Install prod dependencies (stable Pyoaev release) +poetry install --extras prod + +# Install current release from Git / latest +poetry install --extras current + +# Tips For development/testing with local Pyoaev (client-python) +poetry run pip install -r requirements.txt +``` + +Run the injector: +```bash +poetry run ShodanInjector +``` + +Run using the console entry point +```bash +# Requires the package to be installed and the virtual environment to be active +ShodanInjector +``` + +**Why use Poetry:** + +- Automatically creates and manages a virtual environment +- Handles extras and entry points (ShodanInjector) defined in `pyproject.toml` +- Keeps your environment isolated and PEP 621 compliant + +#### Commands Summary: +The table below summarizes the installation and run commands for different workflows (pip or Poetry, prod/current/dev). + +| Installation | Install Command | Run Command | Notes | +|-----------------|----------------------------------------------|-----------------------------------------------------------------------|----------------------------------------------------------| +| Pip Prod | `pip install .[prod]` | `python -m shodan` / `ShodanInjector` (Requires venv active) | Installs stable dependencies, venv managed automatically | +| Pip Current | `pip install .[current]` | `python -m shodan` / `ShodanInjector` (Requires venv active) | Installs latest release from Git | +| Pip Dev/Test | `pip install -r requirements.txt` | `python -m shodan` / `ShodanInjector` (Requires venv active) | Handles local editable client-python + dev/test extras | +| Poetry Prod | `poetry install --extras prod` | `poetry run ShodanInjector` / `ShodanInjector` (Requires venv active) | Installs stable dependencies, venv managed automatically | +| Poetry Current | `poetry install --extras current` | `poetry run ShodanInjector` / `ShodanInjector` (Requires venv active) | Installs latest release from Git | +| Poetry Dev/Test | `poetry run pip install -r requirements.txt` | `poetry run ShodanInjector` / `ShodanInjector` (Requires venv active) | Handles local editable client-python + dev/test extras | + +--- + +## Behavior + +For the Shodan injector, we have 7 contracts available. + +- Specific behavior for all contracts: + - For each contract that has the hostnames and organization fields + - For each hostname entered in the hostname field, a dedicated API call will be made with its associated wildcard. + - Example: `hostname:filigran.io,*.filigran.io` (wildcard) + - If left empty, the organization is automatically derived from each hostname. Otherwise, the specified organization is applied to all hostnames. + - Once all calls have been made, a final API call is made to retrieve user information, including the remaining quota. + +### Supported Contracts + +- Cloud Provider Asset Discovery +- Critical Ports And Exposed Admin Interface +- Custom Query +- CVE Enumeration +- CVE Specific Watchlist (The only contract that requires a plan Shodan only available to academic users, Small Business API subscribers, and higher.) +- Domain Discovery +- IP Enumeration + +### Target Selection + +Targets are selected based on the `target_selector` field. + +#### If target type is **Assets** / **Asset-Groups**: (Currently disabled) + +| Selected Property | Uses Asset Field | +|-------------------|----------------------------| +| Automatic | Automatic | +| Seen IP | endpoint_seen_ip | +| Local IP | First IP from endpoint_ips | +| Hostname | endpoint_hostname | + +#### If target type is **Manual**: + +Direct values separated by commas or spaces between IP addresses or hostnames are used. + +### Fields available in manual mode by contract + +- Cloud Provider Asset Discovery (Search Shodan Endpoint: `/shodan/host/search`) + - The `Cloud Provider` field must contain one or more cloud providers, separated by commas. + - The `Hostname` field must contain one or more hostnames, separated by commas. + - The `Organization` field must contain one or more organizations, separated by commas. + + | Field | Mandatory | Default / Notes | + |----------------|-----------|---------------------------------------| + | Cloud Provider | Yes | `Google,Microsoft,Amazon,Azure` | + | Hostname | Yes | / | + | Organization | No | If empty, the hostname value is used. | + +- Critical Ports And Exposed Admin Interface (Search Shodan Endpoint: `/shodan/host/search`) + - The `Port` field must contain one or more ports, separated by commas. + - The `Hostname` field must contain one or more hostnames, separated by commas. + - The `Organization` field must contain one or more organizations, separated by commas. + + | Field | Mandatory | Default / Notes | + |--------------|-------------|-------------------------------------------------------------------------------------| + | Port | Yes | `20,21,22,23,25,53,80,110,111,135,139,143,443,445,993,995,1723,3306,3389,5900,8080` | + | Hostname | Yes | / | + | Organization | No | If empty, the hostname value is used. | + +- Custom Query (Endpoint: `your custom endpoint`) + - See the Shodan documentation: https://developer.shodan.io/api + + | Field | Mandatory | Default / Notes | + |--------------|-----------|-----------------------------------------| + | HTTP Method | Yes | `GET` (choices: GET, POST, PUT, DELETE) | + | Custom Query | Yes | / | + +- CVE Enumeration (Search Shodan Endpoint: `/shodan/host/search`) + - The `Hostname` field must contain one or more hostnames, separated by commas. + - The `Organization` field must contain one or more organizations, separated by commas. + + | Field | Mandatory | Default / Notes | + |--------------|-----------|---------------------------------------| + | Hostname | Yes | / | + | Organization | No | If empty, the hostname value is used. | + +- CVE Specific Watchlist (Search Shodan Endpoint: `/shodan/host/search`) + - Warning : The only contract that requires a plan Shodan only available to academic users, Small Business API subscribers, and higher. + - The `Vulnerability` field must contain one or more specific CVEs. + - The `Hostname` field must contain one or more hostnames, separated by commas. + - The `Organization` field must contain one or more organizations, separated by commas. + + | Field | Mandatory | Default / Notes | + |---------------|-----------|---------------------------------------| + | Vulnerability | Yes | / | + | Hostname | Yes | / | + | Organization | No | If empty, the hostname value is used. | + +- Domain Discovery (Search Shodan Endpoint: `/shodan/host/search`) + - The `Hostname` field must contain one or more hostnames, separated by commas. + - The `Organization` field must contain one or more organizations, separated by commas. + + | Field | Mandatory | Default / Notes | + |--------------|-----------|---------------------------------------| + | Hostname | Yes | / | + | Organization | No | If empty, the hostname value is used. | + +- IP Enumeration (Search Shodan Endpoint: `/shodan/host/search`) + - The `IP` field must contain one or more valid IPv4 addresses. + + | Field | Mandatory | Default / Notes | + |-------|-----------|-----------------| + | IP | Yes | / | + +--- + +### Output Trace Message +The injector captures the fields filled in by the user and analyzes the JSON output of the Shodan response, +then returns several sections in the report if successful: + +- **Section Title** – Contains the title of the report and the date and time it was created. +- **Section Config** – Groups all the configuration information entered by the user for the current contract. +- **Section Info** – Makes an additional final call to the Shodan API to determine the user's remaining quota and plan. +- **Section External API** – Processes information from Shodan API responses based on the contract and fields filled in. This section also includes: + - Call Success – List of successful calls + As well as various other relevant information related to API calls. + - Call Failed – List of failed calls + As well as various other relevant information related to API calls. +- **Section Table** – Visually displays the details of each call, based on the configuration defined for the contract in the injector. +- **Section JSON** – JSON return of the response directly (In the case of a "custom query", we return the JSON directly rather than the table section.) + +### Auto-Create Assets +- Feature currently under development + +### Rate Limiting and Retry + +#### Overview + +The Shodan API does not publicly document strict rate-limiting rules or request thresholds. +However, to ensure stable and reliable interactions with the API, a rate-limiting and retry mechanism is implemented. + +These mechanisms aim to: +- Smooth the flow of outgoing requests, +- Reduce the risk of HTTP 429 Too Many Requests responses, +- Handle transient failures, +- Improve the clarity of error reporting in output traces message. + +#### Rate limiting + +A rate-limiting mechanism is applied to control the frequency of API calls. +Requests are deliberately paced to avoid sending bursts of traffic that could exceed Shodan’s internal limits or trigger protective measures. + +This approach helps: +- Prevent temporary blocking or throttling, +- Ensure consistent behavior across multiple API calls, +- Maintain predictable execution when processing multiple hostnames or IPs. + +There are two environment variables for controlling the flow: +- `SHODAN_API_LEAKY_BUCKET_RATE` - Controls the rate at which API requests are allowed to be executed. +- `SHODAN_API_LEAKY_BUCKET_CAPACITY` - Defines the maximum burst size allowed before requests are delayed. + +#### Retry strategy + +A retry mechanism is applied to handle failed API requests. +Requests are retried after a short delay to mitigate transient failures and improve execution stability. + +This approach helps: +- Automatically retry failed requests up to a limited number of attempts. +- Introduce delays between retries to avoid repeated immediate failures. + +There are two environment variables for controlling the flow: +- `SHODAN_API_RETRY` - Defines the maximum number of retry attempts for a failed request. +- `SHODAN_API_BACKOFF` - Specifies the maximum delay (in seconds) applied between retry attempts, using an exponential backoff strategy. + +#### Execution trace and feedback + +Each API call is tracked individually. +The success or failure of each request is reported in the external API section of the output trace message. + +Once all API calls related to the host name or IP address have been made, a final API call is executed to retrieve user information, including the remaining API quota. + +--- + +## Resources + +**Filigran** +- Homepage: https://filigran.io/ +- Repository: https://github.com/OpenAEV-Platform/injectors/tree/main/shodan +- Documentation: https://github.com/OpenAEV-Platform/injectors/tree/main/shodan/README.md +- Issues: https://github.com/OpenAEV-Platform/injectors/issues + +**Shodan** +- Homepage: https://www.shodan.io/ +- Register Page: https://account.shodan.io/register +- REST API Documentation: https://developer.shodan.io/api \ No newline at end of file diff --git a/shodan/docker-compose.yml b/shodan/docker-compose.yml new file mode 100644 index 00000000..143ffd19 --- /dev/null +++ b/shodan/docker-compose.yml @@ -0,0 +1,20 @@ +version: '3.8' +services: + injector-shodan: + image: openaev/injector-shodan:2.0.14 + environment: + - OPENAEV_URL=http://localhost + - OPENAEV_TOKEN=ChangeMe +# - INJECTOR_ID=shodan--a87488ad-2c72-4592-b429-69259d7bcef1 +# - INJECTOR_NAME=Shodan +# - INJECTOR_LOG_LEVEL=error + - SHODAN_API_KEY=ChangeMe +# - SHODAN_BASE_URL=https://api.shodan.io +# - SHODAN_API_LEAKY_BUCKET_RATE=10 +# - SHODAN_API_LEAKY_BUCKET_CAPACITY=10 +# - SHODAN_API_RETRY=5 +# - SHODAN_API_BACKOFF=PT30S + restart: always + depends_on: + openaev: + condition: service_healthy diff --git a/shodan/pyproject.toml b/shodan/pyproject.toml new file mode 100644 index 00000000..d2aa6179 --- /dev/null +++ b/shodan/pyproject.toml @@ -0,0 +1,144 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[project.urls] +Homepage = "https://filigran.io/" +Repository = "https://github.com/OpenAEV-Platform/injectors/tree/main/shodan" +Documentation = "https://github.com/OpenAEV-Platform/injectors/tree/main/shodan/README.md" +Issues = "https://github.com/OpenAEV-Platform/injectors/issues" + +[project] +name = "openaev-shodan-injector" +version = "2.0.14" +description = "An injector for running with Shodan" +readme = "README.md" +authors = [{ name = "Filigran", email = "contact@filigran.io" }] +license = "Apache-2.0" +requires-python = ">=3.11, <3.14" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", +] +dependencies = [ + "pydantic~=2.11.7", + "pydantic-settings~=2.11.0", + "requests~=2.32.5", + "limiter==0.5.0", + "tenacity~=9.1.2", + "rich~=14.2.0", +] + +[project.optional-dependencies] +prod = [ + "pyoaev ~=2.0.14" +] +current = [ + "pyoaev @ git+https://github.com/OpenAEV-Platform/client-python.git@release/current" +] +dev = [ + "black~=25.1", # Code formatter + "isort~=7.0.0", # Import sorter + "ruff~=0.14.2", # linter + "mypy~=1.18.2", # Type validator + "pip_audit~=2.9.0", # Security checker + "pre-commit~=4.3.0", # Git hooks + "types-PyYAML~=6.0.12", # stubs for untyped module +] +test = [ + "pytest~=8.4.1", + "polyfactory~=2.22.2", +] + +[tool.poetry] +packages = [{ include = "shodan" }] + +[project.scripts] +ShodanInjector = "shodan.__main__:main" + +[tool.setuptools.packages.find] +where = ["."] + +[tool.isort] +profile = "black" +src_paths = ["shodan"] + +[tool.pytest.ini_options] +testpaths = ["./tests"] + +[tool.ruff] +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "site-packages", + "venv", +] + +target-version = "py312" + +[tool.ruff.lint] +# Never enforce `I001` (unsorted import). Already handle with isort +# Never enforce `E501` (line length violations). Already handle with black +# Never enforce `F821` (Undefined name `null`). incorrect issue with notebook +# Never enforce `D213` (Multi-line docstring summary should start at the second line) conflict with our docstring convention +# Never enforce `D211` (NoBlankLinesBeforeClass)` +# Never enforce `G004` (logging-f-string) Logging statement uses f-string +# Never enforce `TRY003`() Avoid specifying long messages outside the exception class not useful +# Never enforce `D104` (Missing docstring in public package) +# Never enforce `D407` (Missing dashed underline after section) +# Never enforce `D408` (Section underline should be in the line following the section’s name) +# Never enforce `D409` (Section underline should match the length of its name) +ignore = [ + "I001", + "D203", + "E501", + "F821", + "D205", + "D213", + "D211", + "G004", + "TRY003", + "D104", + "D407", + "D408", + "D409", +] +select = ["E", "F", "W", "D", "G", "T", "B", "C", "N", "I", "S"] + +[tool.mypy] +strict = true +exclude = [ + '^tests', + '^docs', + '^build', + '^dist', + '^venv', + '^site-packages', + '^__pypackages__', + '^.venv', +] +plugins = ["pydantic.mypy"] \ No newline at end of file diff --git a/shodan/requirements.txt b/shodan/requirements.txt new file mode 100644 index 00000000..2edfa69b --- /dev/null +++ b/shodan/requirements.txt @@ -0,0 +1,5 @@ +# Local dependency (client-python) +-e ../../client-python + +# Local project with extras (dev + test) +-e .[dev,test] \ No newline at end of file diff --git a/shodan/shodan/.env.sample b/shodan/shodan/.env.sample new file mode 100644 index 00000000..1c55f139 --- /dev/null +++ b/shodan/shodan/.env.sample @@ -0,0 +1,20 @@ +# OPENAEV Environment Variables +# base URL to reach the OpenAEV server +# note this URL must be routable from inside the container +# so `localhost` will most likely not work +OPENAEV_URL=ChangeMe +# admin account API token from the OpenAEV server +OPENAEV_TOKEN=ChangeMe + +# INJECTOR Environment Variables +#INJECTOR_ID=shodan--a87488ad-2c72-4592-b429-69259d7bcef1 +#INJECTOR_NAME=Shodan +#INJECTOR_LOG_LEVEL=error + +# SHODAN Environment Variables +SHODAN_API_KEY=ChangeMe +#SHODAN_BASE_URL=https://api.shodan.io +#SHODAN_API_LEAKY_BUCKET_RATE=10 +#SHODAN_API_LEAKY_BUCKET_CAPACITY=10 +#SHODAN_API_RETRY=5 +#SHODAN_API_BACKOFF=PT30S \ No newline at end of file diff --git a/shodan/shodan/__init__.py b/shodan/shodan/__init__.py new file mode 100644 index 00000000..ccd9759b --- /dev/null +++ b/shodan/shodan/__init__.py @@ -0,0 +1,3 @@ +from shodan.models import ConfigLoader + +__all__ = ["ConfigLoader"] diff --git a/shodan/shodan/__main__.py b/shodan/shodan/__main__.py new file mode 100644 index 00000000..76673fce --- /dev/null +++ b/shodan/shodan/__main__.py @@ -0,0 +1,63 @@ +"""Main entry point for the injector.""" + +import logging +import os +import sys +from pathlib import Path + +from pydantic import ValidationError +from pyoaev.helpers import OpenAEVInjectorHelper + +from shodan.contracts.shodan_contracts import ShodanContracts +from shodan.injector.openaev_shodan import ShodanInjector +from shodan.models import ConfigLoader + +# from shodan.injector.exception import InjectorConfigError + +LOG_PREFIX = "[SHODAN_MAIN]" + + +def main() -> None: + """Define the main function to run the injector.""" + logger = logging.getLogger(__name__) + + try: + # Loading injector configuration + config = ConfigLoader() + + # Build Shodan contracts and adapt config for the helper + shodan_contracts = ShodanContracts(config).contracts() + config_helper_adapter = config.to_config_injector_helper_adapter( + contracts=shodan_contracts + ) + + # Load the injector icon for the helper + icon_path = Path(__file__).parent / "img" / "icon-shodan.png" + icon_bytes = icon_path.read_bytes() + + # Instantiate the OpenAEV injector helper + helper = OpenAEVInjectorHelper(config=config_helper_adapter, icon=icon_bytes) + + logger.info( + f"{LOG_PREFIX} - Shodan injector configuration initialized successfully." + ) + + # Start the Shodan injector + injector = ShodanInjector(config, helper) + injector.start() + + except ValidationError as err: + logger.error(f"{LOG_PREFIX} Configuration error: {err}") + sys.exit(2) + + except KeyboardInterrupt: + logger.info(f"{LOG_PREFIX} Injector stopped by user (Ctrl+C)") + os._exit(0) + + except Exception as err: + logger.exception(f"{LOG_PREFIX} Fatal error starting injector: {err}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/shodan/shodan/config.yml.sample b/shodan/shodan/config.yml.sample new file mode 100644 index 00000000..80ae8e99 --- /dev/null +++ b/shodan/shodan/config.yml.sample @@ -0,0 +1,16 @@ +openaev: + url: 'ChangeMe' + token: 'ChangeMe' + +#injector: +# id: 'shodan--a87488ad-2c72-4592-b429-69259d7bcef1' +# name: 'Shodan' +# log_level: 'error' + +shodan: + api_key: 'ChangeMe' +# base_url: 'https://api.shodan.io' +# api_leaky_bucket_rate: 10 +# api_leaky_bucket_capacity: 10 +# api_retry: 5 +# api_backoff: 'PT30S' \ No newline at end of file diff --git a/shodan/shodan/contracts/__init__.py b/shodan/shodan/contracts/__init__.py new file mode 100644 index 00000000..dd5329c7 --- /dev/null +++ b/shodan/shodan/contracts/__init__.py @@ -0,0 +1,22 @@ +from .cloud_provider_asset_discovery import CloudProviderAssetDiscovery +from .critical_ports_and_exposed_admin_interface import ( + CriticalPortsAndExposedAdminInterface, +) +from .custom_query import CustomQuery +from .cve_enumeration import CVEEnumeration +from .cve_specific_watchlist import CVESpecificWatchlist +from .domain_discovery import DomainDiscovery +from .ip_enumeration import IPEnumeration +from .shodan_contracts import InjectorKey, ShodanContractId + +__all__ = [ + "CloudProviderAssetDiscovery", + "CriticalPortsAndExposedAdminInterface", + "CustomQuery", + "CVEEnumeration", + "CVESpecificWatchlist", + "DomainDiscovery", + "IPEnumeration", + "InjectorKey", + "ShodanContractId", +] diff --git a/shodan/shodan/contracts/cloud_provider_asset_discovery/__init__.py b/shodan/shodan/contracts/cloud_provider_asset_discovery/__init__.py new file mode 100644 index 00000000..8f0dd5b9 --- /dev/null +++ b/shodan/shodan/contracts/cloud_provider_asset_discovery/__init__.py @@ -0,0 +1,3 @@ +from .contract import CloudProviderAssetDiscovery + +__all__ = ["CloudProviderAssetDiscovery"] diff --git a/shodan/shodan/contracts/cloud_provider_asset_discovery/contract.py b/shodan/shodan/contracts/cloud_provider_asset_discovery/contract.py new file mode 100644 index 00000000..b700a05f --- /dev/null +++ b/shodan/shodan/contracts/cloud_provider_asset_discovery/contract.py @@ -0,0 +1,200 @@ +from pyoaev.contracts import ContractBuilder +from pyoaev.contracts.contract_config import ( + Contract, + ContractConfig, + ContractElement, + ContractOutputElement, + ContractText, + ContractTuple, + SupportedLanguage, +) + + +class CloudProviderAssetDiscovery: + + @staticmethod + def output_trace_config(): + return { + "header": { + "title": "SHODAN - CLOUD PROVIDER ASSET DISCOVERY", + "subtitle": None, + }, + "sections_config": { + "header": { + "icon": "CONFIG", + "title": "[CONFIG] Summary of all configurations used for the contract.", + }, + "keys_list_to_string": ["cloud_provider"], + "keys_to_exclude": [], + }, + "sections_info": { + "header": { + "icon": "INFO", + "title": "[INFO] The Shodan information for the remaining credits and the user's plan.", + }, + "keys_list_to_string": [], + "keys_to_exclude": [], + }, + "sections_external_api": { + "header": { + "icon": "API", + "title": "[SHODAN] Call API completed", + }, + "call_success": { + "icon": "SUCCESS", + "title": "Call Success", + "count_at_path": "matches", + }, + "call_failed": { + "icon": "FAILED", + "title": "Call Failed", + }, + }, + "tables": [ + { + "header": { + "icon": "SEARCH", + "title": None, + }, + "config": { + "search_entity": None, + "columns": [ + { + "title": "Hostnames", + "path": "matches.hostnames", + "mode": "align_to_single", + }, + { + "title": "IP", + "path": "matches.ip_str", + "mode": "single", + }, + { + "title": "Port", + "path": "matches.port", + "mode": "align_to_single", + }, + { + "title": "Cloud Provider", + "path": "matches.cloud.provider", + "mode": "align_to_single", + }, + { + "title": "OS", + "path": "matches.os", + "mode": "align_to_single", + }, + { + "title": "Vulnerabilities (score)", + "path": "matches.vulns.*", + "use_key": True, + "extra": "matches.vulns.*.cvss", + "mode": "align_to_single", + }, + ], + }, + } + ], + "options": { + # "split_output": False, + "show_header": { + "is_active": True, + "show_subtitle": True, + }, + "show_sections": { + "is_active": True, + "sec_config": True, + "sec_info": True, + "sec_external_api": True, + }, + "show_tables": { + "is_active": True, + "show_lines": True, + "max_display_by_cell": 10, + "show_index": { + "is_active": False, + "index_start": 1, + }, + }, + "show_separator": { + "is_active": False, + }, + "show_json": { + "is_active": False, + "indent": 2, + "sort_keys": False, + }, + }, + } + + @staticmethod + def contract_with_specific_fields( + base_fields: list[ContractElement], + source_selector_key: str, + target_selector_field: str, + ) -> list[ContractElement]: + + mandatory_conditions = dict( + mandatoryConditionFields=[source_selector_key], + mandatoryConditionValues={source_selector_key: target_selector_field}, + ) + + visible_conditions = dict( + visibleConditionFields=[source_selector_key], + visibleConditionValues={source_selector_key: target_selector_field}, + ) + + specific_fields = [ + ContractTuple( + key="cloud_provider", + label="Cloud Provider", + defaultValue=["Google", "Microsoft", "Amazon", "Azure"], + **(mandatory_conditions | visible_conditions), + ), + ContractText( + key="hostname", + label="Hostname", + **(mandatory_conditions | visible_conditions), + ), + ContractText( + key="organization", + label="Organization", + **visible_conditions, + ), + ] + + contract_fields = ( + ContractBuilder().add_fields(base_fields + specific_fields).build_fields() + ) + return contract_fields + + @staticmethod + def contract_with_specific_outputs( + base_outputs: list[ContractOutputElement], + ) -> list[ContractOutputElement]: + specific_outputs = [] + contract_outputs = ( + ContractBuilder() + .add_outputs(base_outputs + specific_outputs) + .build_outputs() + ) + return contract_outputs + + @staticmethod + def contract( + contract_id: str, + contract_config: ContractConfig, + contract_with_specific_fields: list[ContractElement], + contract_with_specific_outputs: list[ContractOutputElement], + ) -> Contract: + return Contract( + contract_id=contract_id, + config=contract_config, + label={ + SupportedLanguage.en: "Shodan - Cloud Provider Asset Discovery", + SupportedLanguage.fr: "Shodan - Découverte des actifs chez les fournisseurs cloud", + }, + fields=contract_with_specific_fields, + outputs=contract_with_specific_outputs, + manual=False, + ) diff --git a/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/__init__.py b/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/__init__.py new file mode 100644 index 00000000..d01e15ca --- /dev/null +++ b/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/__init__.py @@ -0,0 +1,3 @@ +from .contract import CriticalPortsAndExposedAdminInterface + +__all__ = ["CriticalPortsAndExposedAdminInterface"] diff --git a/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/contract.py b/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/contract.py new file mode 100644 index 00000000..08100cb9 --- /dev/null +++ b/shodan/shodan/contracts/critical_ports_and_exposed_admin_interface/contract.py @@ -0,0 +1,212 @@ +from pyoaev.contracts import ContractBuilder +from pyoaev.contracts.contract_config import ( + Contract, + ContractConfig, + ContractElement, + ContractOutputElement, + ContractText, + ContractTuple, + SupportedLanguage, +) + + +class CriticalPortsAndExposedAdminInterface: + + @staticmethod + def output_trace_config(): + return { + "header": { + "title": "SHODAN - CRITICAL PORTS AND EXPOSED ADMIN INTERFACE", + "subtitle": None, + }, + "sections_config": { + "header": { + "icon": "CONFIG", + "title": "[CONFIG] Summary of all configurations used for the contract.", + }, + "keys_list_to_string": ["port"], + "keys_to_exclude": [], + }, + "sections_info": { + "header": { + "icon": "INFO", + "title": "[INFO] The Shodan information for the remaining credits and the user's plan.", + }, + "keys_list_to_string": [], + "keys_to_exclude": [], + }, + "sections_external_api": { + "header": { + "icon": "API", + "title": "[SHODAN] Call API completed", + }, + "call_success": { + "icon": "SUCCESS", + "title": "Call Success", + "count_at_path": "matches", + }, + "call_failed": { + "icon": "FAILED", + "title": "Call Failed", + }, + }, + "tables": [ + { + "header": { + "icon": "SEARCH", + "title": None, + }, + "config": { + "search_entity": None, + "columns": [ + { + "title": "Port", + "path": "matches.port", + "mode": "single", + }, + { + "title": "Hostnames", + "path": "matches.hostnames", + "mode": "align_to_single", + }, + { + "title": "IP", + "path": "matches.ip_str", + "mode": "align_to_single", + }, + { + "title": "Vulnerabilities (score)", + "path": "matches.vulns.*", + "use_key": True, + "extra": "matches.vulns.*.cvss", + "mode": "align_to_single", + }, + ], + }, + } + ], + "options": { + # "split_output": False, + "show_header": { + "is_active": True, + "show_subtitle": True, + }, + "show_sections": { + "is_active": True, + "sec_config": True, + "sec_info": True, + "sec_external_api": True, + }, + "show_tables": { + "is_active": True, + "show_lines": True, + "max_display_by_cell": 10, + "show_index": { + "is_active": False, + "index_start": 1, + }, + }, + "show_separator": { + "is_active": False, + }, + "show_json": { + "is_active": False, + "indent": 2, + "sort_keys": False, + }, + }, + } + + @staticmethod + def contract_with_specific_fields( + base_fields: list[ContractElement], + source_selector_key: str, + target_selector_field: str, + ) -> list[ContractElement]: + + mandatory_conditions = dict( + mandatoryConditionFields=[source_selector_key], + mandatoryConditionValues={source_selector_key: target_selector_field}, + ) + + visible_conditions = dict( + visibleConditionFields=[source_selector_key], + visibleConditionValues={source_selector_key: target_selector_field}, + ) + + specific_fields = [ + ContractTuple( + key="port", + label="Port", + defaultValue=[ + "20", + "21", + "22", + "23", + "25", + "53", + "80", + "110", + "111", + "135", + "139", + "143", + "443", + "445", + "993", + "995", + "1723", + "3306", + "3389", + "5900", + "8080", + ], + **(mandatory_conditions | visible_conditions), + ), + ContractText( + key="hostname", + label="Hostname", + **(mandatory_conditions | visible_conditions), + ), + ContractText( + key="organization", + label="Organization", + **visible_conditions, + ), + ] + + contract_fields = ( + ContractBuilder().add_fields(base_fields + specific_fields).build_fields() + ) + return contract_fields + + @staticmethod + def contract_with_specific_outputs( + base_outputs: list[ContractOutputElement], + ) -> list[ContractOutputElement]: + specific_outputs = [] + contract_outputs = ( + ContractBuilder() + .add_outputs(base_outputs + specific_outputs) + .build_outputs() + ) + return contract_outputs + + @staticmethod + def contract( + contract_id: str, + contract_config: ContractConfig, + contract_with_specific_fields: list[ContractElement], + contract_with_specific_outputs: list[ContractOutputElement], + ) -> Contract: + return Contract( + contract_id=contract_id, + config=contract_config, + label={ + SupportedLanguage.en: "Shodan - Critical ports and exposed admin interface", + SupportedLanguage.fr: "Shodan - Ports critiques et interface d'administration exposée", + }, + fields=contract_with_specific_fields, + outputs=contract_with_specific_outputs, + manual=False, + ) diff --git a/shodan/shodan/contracts/custom_query/__init__.py b/shodan/shodan/contracts/custom_query/__init__.py new file mode 100644 index 00000000..dd608594 --- /dev/null +++ b/shodan/shodan/contracts/custom_query/__init__.py @@ -0,0 +1,3 @@ +from .contract import CustomQuery + +__all__ = ["CustomQuery"] diff --git a/shodan/shodan/contracts/custom_query/contract.py b/shodan/shodan/contracts/custom_query/contract.py new file mode 100644 index 00000000..a244f713 --- /dev/null +++ b/shodan/shodan/contracts/custom_query/contract.py @@ -0,0 +1,168 @@ +from pyoaev.contracts import ContractBuilder +from pyoaev.contracts.contract_config import ( + Contract, + ContractConfig, + ContractElement, + ContractOutputElement, + ContractSelect, + ContractText, + SupportedLanguage, +) + + +class CustomQuery: + + @staticmethod + def output_trace_config(): + return { + "header": { + "title": "SHODAN - CUSTOM QUERY", + "subtitle": None, + }, + "sections_config": { + "header": { + "icon": "CONFIG", + "title": "[CONFIG] Summary of all configurations used for the contract.", + }, + "keys_list_to_string": [], + "keys_to_exclude": [], + }, + "sections_info": { + "header": { + "icon": "INFO", + "title": "[INFO] The Shodan information for the remaining credits and the user's plan.", + }, + "keys_list_to_string": [], + "keys_to_exclude": [], + }, + "sections_external_api": { + "header": { + "icon": "API", + "title": "[SHODAN] Call API completed", + }, + "call_success": { + "icon": "SUCCESS", + "title": "Call Success", + "count_at_path": "matches", + }, + "call_failed": { + "icon": "FAILED", + "title": "Call Failed", + }, + }, + "tables": [ + { + "header": { + "icon": "SEARCH", + "title": None, + }, + "config": { + "search_entity": None, + "columns": [], + }, + } + ], + "options": { + # "split_output": False, + "show_header": { + "is_active": True, + "show_subtitle": True, + }, + "show_sections": { + "is_active": True, + "sec_config": True, + "sec_info": True, + "sec_external_api": True, + }, + "show_tables": { + "is_active": False, + "show_lines": True, + "max_display_by_cell": 10, + "show_index": { + "is_active": False, + "index_start": 1, + }, + }, + "show_separator": { + "is_active": False, + }, + "show_json": { + "is_active": True, + "indent": 2, + "sort_keys": False, + }, + }, + } + + @staticmethod + def contract_with_specific_fields( + base_fields: list[ContractElement], + source_selector_key: str, + target_selector_field: str, + ) -> list[ContractElement]: + + mandatory_conditions = dict( + mandatoryConditionFields=[source_selector_key], + mandatoryConditionValues={source_selector_key: target_selector_field}, + ) + + visible_conditions = dict( + visibleConditionFields=[source_selector_key], + visibleConditionValues={source_selector_key: target_selector_field}, + ) + + specific_fields = [ + ContractSelect( + key="http_method", + label="HTTP Method", + defaultValue=["get"], + choices={ + "get": "GET", + "post": "POST", + "put": "PUT", + "delete": "DELETE", + }, + **(mandatory_conditions | visible_conditions), + ), + ContractText( + key="custom_query", + label="Custom Query", + **(mandatory_conditions | visible_conditions), + ), + ] + + contract_fields = ( + ContractBuilder().add_fields(base_fields + specific_fields).build_fields() + ) + return contract_fields + + @staticmethod + def contract_with_specific_outputs( + base_outputs: list[ContractOutputElement], + ) -> list[ContractOutputElement]: + specific_outputs = [] + contract_outputs = ( + ContractBuilder() + .add_outputs(base_outputs + specific_outputs) + .build_outputs() + ) + return contract_outputs + + @staticmethod + def contract( + contract_id: str, + contract_config: ContractConfig, + contract_with_specific_fields: list[ContractElement], + contract_with_specific_outputs: list[ContractOutputElement], + ) -> Contract: + return Contract( + contract_id=contract_id, + config=contract_config, + label={ + SupportedLanguage.en: "Shodan - Custom query", + SupportedLanguage.fr: "Shodan - Requête personnalisée", + }, + fields=contract_with_specific_fields, + outputs=contract_with_specific_outputs, + manual=False, + ) diff --git a/shodan/shodan/contracts/cve_enumeration/__init__.py b/shodan/shodan/contracts/cve_enumeration/__init__.py new file mode 100644 index 00000000..4f7378e1 --- /dev/null +++ b/shodan/shodan/contracts/cve_enumeration/__init__.py @@ -0,0 +1,3 @@ +from .contract import CVEEnumeration + +__all__ = ["CVEEnumeration"] diff --git a/shodan/shodan/contracts/cve_enumeration/contract.py b/shodan/shodan/contracts/cve_enumeration/contract.py new file mode 100644 index 00000000..b7d85189 --- /dev/null +++ b/shodan/shodan/contracts/cve_enumeration/contract.py @@ -0,0 +1,183 @@ +from pyoaev.contracts import ContractBuilder +from pyoaev.contracts.contract_config import ( + Contract, + ContractConfig, + ContractElement, + ContractOutputElement, + ContractText, + SupportedLanguage, +) + + +class CVEEnumeration: + + @staticmethod + def output_trace_config(): + return { + "header": { + "title": "SHODAN - CVE ENUMERATION", + "subtitle": None, + }, + "sections_config": { + "header": { + "icon": "CONFIG", + "title": "[CONFIG] Summary of all configurations used for the contract.", + }, + "keys_list_to_string": [], + "keys_to_exclude": [], + }, + "sections_info": { + "header": { + "icon": "INFO", + "title": "[INFO] The Shodan information for the remaining credits and the user's plan.", + }, + "keys_list_to_string": [], + "keys_to_exclude": [], + }, + "sections_external_api": { + "header": { + "icon": "API", + "title": "[SHODAN] Call API completed", + }, + "call_success": { + "icon": "SUCCESS", + "title": "Call Success", + "count_at_path": "matches", + }, + "call_failed": { + "icon": "FAILED", + "title": "Call Failed", + }, + }, + "tables": [ + { + "header": { + "icon": "SEARCH", + "title": None, + }, + "config": { + "search_entity": None, + "columns": [ + { + "title": "Hostnames", + "path": "matches.hostnames", + "mode": "single", + }, + { + "title": "IP", + "path": "matches.ip_str", + "mode": "align_to_single", + }, + { + "title": "Port", + "path": "matches.port", + "mode": "align_to_single", + }, + { + "title": "Vulnerabilities (score)", + "path": "matches.vulns.*", + "use_key": True, + "extra": "matches.vulns.*.cvss", + "mode": "align_to_single", + }, + ], + }, + } + ], + "options": { + # "split_output": False, + "show_header": { + "is_active": True, + "show_subtitle": True, + }, + "show_sections": { + "is_active": True, + "sec_config": True, + "sec_info": True, + "sec_external_api": True, + }, + "show_tables": { + "is_active": True, + "show_lines": True, + "max_display_by_cell": 10, + "show_index": { + "is_active": False, + "index_start": 1, + }, + }, + "show_separator": { + "is_active": False, + }, + "show_json": { + "is_active": False, + "indent": 2, + "sort_keys": False, + }, + }, + } + + @staticmethod + def contract_with_specific_fields( + base_fields: list[ContractElement], + source_selector_key: str, + target_selector_field: str, + ) -> list[ContractElement]: + + mandatory_conditions = dict( + mandatoryConditionFields=[source_selector_key], + mandatoryConditionValues={source_selector_key: target_selector_field}, + ) + + visible_conditions = dict( + visibleConditionFields=[source_selector_key], + visibleConditionValues={source_selector_key: target_selector_field}, + ) + + specific_fields = [ + ContractText( + key="hostname", + label="Hostname", + **(mandatory_conditions | visible_conditions), + ), + ContractText( + key="organization", + label="Organization", + **visible_conditions, + ), + ] + + contract_fields = ( + ContractBuilder().add_fields(base_fields + specific_fields).build_fields() + ) + return contract_fields + + @staticmethod + def contract_with_specific_outputs( + base_outputs: list[ContractOutputElement], + ) -> list[ContractOutputElement]: + specific_outputs = [] + contract_outputs = ( + ContractBuilder() + .add_outputs(base_outputs + specific_outputs) + .build_outputs() + ) + return contract_outputs + + @staticmethod + def contract( + contract_id: str, + contract_config: ContractConfig, + contract_with_specific_fields: list[ContractElement], + contract_with_specific_outputs: list[ContractOutputElement], + ) -> Contract: + return Contract( + contract_id=contract_id, + config=contract_config, + label={ + SupportedLanguage.en: "Shodan - CVE Enumeration", + SupportedLanguage.fr: "Shodan - Énumération des CVE", + }, + fields=contract_with_specific_fields, + outputs=contract_with_specific_outputs, + manual=False, + ) diff --git a/shodan/shodan/contracts/cve_specific_watchlist/__init__.py b/shodan/shodan/contracts/cve_specific_watchlist/__init__.py new file mode 100644 index 00000000..8808845c --- /dev/null +++ b/shodan/shodan/contracts/cve_specific_watchlist/__init__.py @@ -0,0 +1,3 @@ +from .contract import CVESpecificWatchlist + +__all__ = ["CVESpecificWatchlist"] diff --git a/shodan/shodan/contracts/cve_specific_watchlist/contract.py b/shodan/shodan/contracts/cve_specific_watchlist/contract.py new file mode 100644 index 00000000..926351e5 --- /dev/null +++ b/shodan/shodan/contracts/cve_specific_watchlist/contract.py @@ -0,0 +1,188 @@ +from pyoaev.contracts import ContractBuilder +from pyoaev.contracts.contract_config import ( + Contract, + ContractConfig, + ContractElement, + ContractOutputElement, + ContractText, + SupportedLanguage, +) + + +class CVESpecificWatchlist: + + @staticmethod + def output_trace_config(): + return { + "header": { + "title": "SHODAN - CVE SPECIFIC WATCH LIST", + "subtitle": None, + }, + "sections_config": { + "header": { + "icon": "CONFIG", + "title": "[CONFIG] Summary of all configurations used for the contract.", + }, + "keys_list_to_string": [], + "keys_to_exclude": [], + }, + "sections_info": { + "header": { + "icon": "INFO", + "title": "[INFO] The Injector information for the remaining credits and the user's plan.", + }, + "keys_list_to_string": [], + "keys_to_exclude": [], + }, + "sections_external_api": { + "header": { + "icon": "API", + "title": "[SHODAN] Call API completed", + }, + "call_success": { + "icon": "SUCCESS", + "title": "Call Success", + "count_at_path": "data", + }, + "call_failed": { + "icon": "FAILED", + "title": "Call Failed", + }, + }, + "tables": [ + { + "header": { + "icon": "SEARCH", + "title": None, + }, + "config": { + "search_entity": "ip_str", + "columns": [ + { + "title": "Port", + "path": "data.port", + "mode": "single", + }, + { + "title": "Hostnames", + "path": "hostnames", + "mode": "align_to_single", + }, + { + "title": "IP", + "path": "ip_str", + "mode": "align_to_single", + }, + { + "title": "Vulnerabilities (score)", + "path": "data.vulns.*", + "use_key": True, + "extra": "data.vulns.*.cvss", + "mode": "align_to_single", + }, + ], + }, + } + ], + "options": { + # "split_output": False, + "show_header": { + "is_active": True, + "show_subtitle": True, + }, + "show_sections": { + "is_active": True, + "sec_config": True, + "sec_info": True, + "sec_external_api": True, + }, + "show_tables": { + "is_active": True, + "show_lines": True, + "max_display_by_cell": 10, + "show_index": { + "is_active": False, + "index_start": 1, + }, + }, + "show_separator": { + "is_active": False, + }, + "show_json": { + "is_active": False, + "indent": 2, + "sort_keys": False, + }, + }, + } + + @staticmethod + def contract_with_specific_fields( + base_fields: list[ContractElement], + source_selector_key: str, + target_selector_field: str, + ) -> list[ContractElement]: + + mandatory_conditions = dict( + mandatoryConditionFields=[source_selector_key], + mandatoryConditionValues={source_selector_key: target_selector_field}, + ) + + visible_conditions = dict( + visibleConditionFields=[source_selector_key], + visibleConditionValues={source_selector_key: target_selector_field}, + ) + + specific_fields = [ + ContractText( + key="vulnerability", + label="Vulnerability", + **(mandatory_conditions | visible_conditions), + ), + ContractText( + key="hostname", + label="Hostname", + **(mandatory_conditions | visible_conditions), + ), + ContractText( + key="organization", + label="Organization", + **visible_conditions, + ), + ] + + contract_fields = ( + ContractBuilder().add_fields(base_fields + specific_fields).build_fields() + ) + return contract_fields + + @staticmethod + def contract_with_specific_outputs( + base_outputs: list[ContractOutputElement], + ) -> list[ContractOutputElement]: + specific_outputs = [] + contract_outputs = ( + ContractBuilder() + .add_outputs(base_outputs + specific_outputs) + .build_outputs() + ) + return contract_outputs + + @staticmethod + def contract( + contract_id: str, + contract_config: ContractConfig, + contract_with_specific_fields: list[ContractElement], + contract_with_specific_outputs: list[ContractOutputElement], + ) -> Contract: + return Contract( + contract_id=contract_id, + config=contract_config, + label={ + SupportedLanguage.en: "Shodan - CVE specific watchlist", + SupportedLanguage.fr: "Shodan - Liste de surveillance spécifique aux CVE", + }, + fields=contract_with_specific_fields, + outputs=contract_with_specific_outputs, + manual=False, + ) diff --git a/shodan/shodan/contracts/domain_discovery/__init__.py b/shodan/shodan/contracts/domain_discovery/__init__.py new file mode 100644 index 00000000..0b42e416 --- /dev/null +++ b/shodan/shodan/contracts/domain_discovery/__init__.py @@ -0,0 +1,3 @@ +from .contract import DomainDiscovery + +__all__ = ["DomainDiscovery"] diff --git a/shodan/shodan/contracts/domain_discovery/contract.py b/shodan/shodan/contracts/domain_discovery/contract.py new file mode 100644 index 00000000..797f4bb9 --- /dev/null +++ b/shodan/shodan/contracts/domain_discovery/contract.py @@ -0,0 +1,183 @@ +from pyoaev.contracts import ContractBuilder +from pyoaev.contracts.contract_config import ( + Contract, + ContractConfig, + ContractElement, + ContractOutputElement, + ContractText, + SupportedLanguage, +) + + +class DomainDiscovery: + + @staticmethod + def output_trace_config(): + return { + "header": { + "title": "SHODAN - DOMAIN DISCOVERY", + "subtitle": None, + }, + "sections_config": { + "header": { + "icon": "CONFIG", + "title": "[CONFIG] Summary of all configurations used for the contract.", + }, + "keys_list_to_string": [], + "keys_to_exclude": [], + }, + "sections_info": { + "header": { + "icon": "INFO", + "title": "[INFO] The Shodan information for the remaining credits and the user's plan.", + }, + "keys_list_to_string": [], + "keys_to_exclude": [], + }, + "sections_external_api": { + "header": { + "icon": "API", + "title": "[SHODAN] Call API completed", + }, + "call_success": { + "icon": "SUCCESS", + "title": "Call Success", + "count_at_path": "matches", + }, + "call_failed": { + "icon": "FAILED", + "title": "Call Failed", + }, + }, + "tables": [ + { + "header": { + "icon": "SEARCH", + "title": None, + }, + "config": { + "search_entity": None, + "columns": [ + { + "title": "Hostnames", + "path": "matches.hostnames", + "mode": "single", + }, + { + "title": "IP", + "path": "matches.ip_str", + "mode": "align_to_single", + }, + { + "title": "Port", + "path": "matches.port", + "mode": "align_to_single", + }, + { + "title": "Vulnerabilities (score)", + "path": "matches.vulns.*", + "use_key": True, + "extra": "matches.vulns.*.cvss", + "mode": "align_to_single", + }, + ], + }, + } + ], + "options": { + # "split_output": False, + "show_header": { + "is_active": True, + "show_subtitle": True, + }, + "show_sections": { + "is_active": True, + "sec_config": True, + "sec_info": True, + "sec_external_api": True, + }, + "show_tables": { + "is_active": True, + "show_lines": True, + "max_display_by_cell": 10, + "show_index": { + "is_active": False, + "index_start": 1, + }, + }, + "show_separator": { + "is_active": False, + }, + "show_json": { + "is_active": False, + "indent": 2, + "sort_keys": False, + }, + }, + } + + @staticmethod + def contract_with_specific_fields( + base_fields: list[ContractElement], + source_selector_key: str, + target_selector_field: str, + ) -> list[ContractElement]: + + mandatory_conditions = dict( + mandatoryConditionFields=[source_selector_key], + mandatoryConditionValues={source_selector_key: target_selector_field}, + ) + + visible_conditions = dict( + visibleConditionFields=[source_selector_key], + visibleConditionValues={source_selector_key: target_selector_field}, + ) + + specific_fields = [ + ContractText( + key="hostname", + label="Hostname", + **(mandatory_conditions | visible_conditions), + ), + ContractText( + key="organization", + label="Organization", + **visible_conditions, + ), + ] + + contract_fields = ( + ContractBuilder().add_fields(base_fields + specific_fields).build_fields() + ) + return contract_fields + + @staticmethod + def contract_with_specific_outputs( + base_outputs: list[ContractOutputElement], + ) -> list[ContractOutputElement]: + specific_outputs = [] + contract_outputs = ( + ContractBuilder() + .add_outputs(base_outputs + specific_outputs) + .build_outputs() + ) + return contract_outputs + + @staticmethod + def contract( + contract_id: str, + contract_config: ContractConfig, + contract_with_specific_fields: list[ContractElement], + contract_with_specific_outputs: list[ContractOutputElement], + ) -> Contract: + return Contract( + contract_id=contract_id, + config=contract_config, + label={ + SupportedLanguage.en: "Shodan - Domain discovery", + SupportedLanguage.fr: "Shodan - Découverte de domaine", + }, + fields=contract_with_specific_fields, + outputs=contract_with_specific_outputs, + manual=False, + ) diff --git a/shodan/shodan/contracts/ip_enumeration/__init__.py b/shodan/shodan/contracts/ip_enumeration/__init__.py new file mode 100644 index 00000000..a9144ff9 --- /dev/null +++ b/shodan/shodan/contracts/ip_enumeration/__init__.py @@ -0,0 +1,3 @@ +from .contract import IPEnumeration + +__all__ = ["IPEnumeration"] diff --git a/shodan/shodan/contracts/ip_enumeration/contract.py b/shodan/shodan/contracts/ip_enumeration/contract.py new file mode 100644 index 00000000..0b26fcb8 --- /dev/null +++ b/shodan/shodan/contracts/ip_enumeration/contract.py @@ -0,0 +1,173 @@ +from pyoaev.contracts import ContractBuilder +from pyoaev.contracts.contract_config import ( + Contract, + ContractConfig, + ContractElement, + ContractOutputElement, + ContractText, + SupportedLanguage, +) + + +class IPEnumeration: + + @staticmethod + def output_trace_config(): + return { + "header": { + "title": "SHODAN - IP ENUMERATION", + "subtitle": None, + }, + "sections_config": { + "header": { + "icon": "CONFIG", + "title": "[CONFIG] Summary of all configurations used for the contract.", + }, + "keys_list_to_string": [], + "keys_to_exclude": [], + }, + "sections_info": { + "header": { + "icon": "INFO", + "title": "[INFO] The Shodan information for the remaining credits and the user's plan.", + }, + "keys_list_to_string": [], + "keys_to_exclude": [], + }, + "sections_external_api": { + "header": { + "icon": "API", + "title": "[SHODAN] Call API completed", + }, + "call_success": { + "icon": "SUCCESS", + "title": "Call Success", + "count_at_path": "matches", + }, + "call_failed": { + "icon": "FAILED", + "title": "Call Failed", + }, + }, + "tables": [ + { + "header": { + "icon": "SEARCH", + "title": None, + }, + "config": { + "search_entity": "ip_str", + "columns": [ + { + "title": "Port", + "path": "matches.port", + "mode": "single", + }, + { + "title": "Hostnames", + "path": "matches.hostnames", + "mode": "align_to_single", + }, + { + "title": "Vulnerabilities (score)", + "path": "matches.vulns.*", + "use_key": True, + "extra": "matches.vulns.*.cvss", + "mode": "align_to_single", + }, + ], + }, + } + ], + "options": { + # "split_output": False, + "show_header": { + "is_active": True, + "show_subtitle": True, + }, + "show_sections": { + "is_active": True, + "sec_config": True, + "sec_info": True, + "sec_external_api": True, + }, + "show_tables": { + "is_active": True, + "show_lines": True, + "max_display_by_cell": 10, + "show_index": { + "is_active": False, + "index_start": 1, + }, + }, + "show_separator": { + "is_active": False, + }, + "show_json": { + "is_active": False, + "indent": 2, + "sort_keys": False, + }, + }, + } + + @staticmethod + def contract_with_specific_fields( + base_fields: list[ContractElement], + source_selector_key: str, + target_selector_field: str, + ) -> list[ContractElement]: + + mandatory_conditions = dict( + mandatoryConditionFields=[source_selector_key], + mandatoryConditionValues={source_selector_key: target_selector_field}, + ) + + visible_conditions = dict( + visibleConditionFields=[source_selector_key], + visibleConditionValues={source_selector_key: target_selector_field}, + ) + + specific_fields = [ + ContractText( + key="ip", + label="IP", + **(mandatory_conditions | visible_conditions), + ), + ] + + contract_fields = ( + ContractBuilder().add_fields(base_fields + specific_fields).build_fields() + ) + return contract_fields + + @staticmethod + def contract_with_specific_outputs( + base_outputs: list[ContractOutputElement], + ) -> list[ContractOutputElement]: + specific_outputs = [] + contract_outputs = ( + ContractBuilder() + .add_outputs(base_outputs + specific_outputs) + .build_outputs() + ) + return contract_outputs + + @staticmethod + def contract( + contract_id: str, + contract_config: ContractConfig, + contract_with_specific_fields: list[ContractElement], + contract_with_specific_outputs: list[ContractOutputElement], + ) -> Contract: + return Contract( + contract_id=contract_id, + config=contract_config, + label={ + SupportedLanguage.en: "Shodan - IP Enumeration", + SupportedLanguage.fr: "Shodan - Énumération des IP", + }, + fields=contract_with_specific_fields, + outputs=contract_with_specific_outputs, + manual=False, + ) diff --git a/shodan/shodan/contracts/shodan_contracts.py b/shodan/shodan/contracts/shodan_contracts.py new file mode 100644 index 00000000..941db96f --- /dev/null +++ b/shodan/shodan/contracts/shodan_contracts.py @@ -0,0 +1,314 @@ +from dataclasses import dataclass +from enum import Enum, StrEnum + +from pyoaev.contracts.contract_config import ( + Contract, + ContractAsset, + ContractAssetGroup, + ContractCardinality, + ContractCheckbox, + ContractConfig, + ContractElement, + ContractExpectations, + ContractSelect, + Expectation, + ExpectationType, + SupportedLanguage, + prepare_contracts, +) + +from shodan.contracts import ( + CloudProviderAssetDiscovery, + CriticalPortsAndExposedAdminInterface, + CustomQuery, + CVEEnumeration, + CVESpecificWatchlist, + DomainDiscovery, + IPEnumeration, +) +from shodan.models import ConfigLoader + +TYPE = "openaev_shodan" + + +class InjectorKey(StrEnum): + TARGETS_KEY = "targets" + TARGET_SELECTOR_KEY = "target_selector" + TARGET_PROPERTY_SELECTOR_KEY = "target_property_selector" + AUTO_CREATE_ASSETS = "auto_create_assets" + EXPECTATIONS_KEY = "expectations" + + +class ShodanContractId(StrEnum): + CLOUD_PROVIDER_ASSET_DISCOVERY = "1887b988-1553-4e46-bac1-ceeee0483f3a" + CRITICAL_PORTS_AND_EXPOSED_ADMIN_INTERFACE = "5349febd-ec73-408b-aa61-24a86a1ba0a7" + CUSTOM_QUERY = "c7640b4c-8a12-458a-b761-6a1287501a58" + CVE_ENUMERATION = "8cdccd58-78ed-4e17-be2e-7683ec611569" + CVE_SPECIFIC_WATCHLIST = "462087b4-8012-4e21-9575-b9c854ef5811" + DOMAIN_DISCOVERY = "faf73809-1128-4192-aa90-a08828f8ace5" + IP_ENUMERATION = "dc6b8b73-09dd-4388-b7cc-108bf16d26cd" + + +@dataclass +class FieldDefinition: + name: str + target: str | list[str] + label: str + mandatory: bool = False + + +class TypeOfFields(Enum): + ASSETS = FieldDefinition( + name="field_assets", + target="assets", + label="Targeted assets", + ) + ASSET_GROUPS = FieldDefinition( + name="field_asset_groups", + target="asset-groups", + label="Targeted asset groups", + ) + ASSETS_PROPERTY = FieldDefinition( + name="field_assets_property", + target=["assets", "asset-groups"], + label="Targeted assets property", + ) + + +@dataclass +class SelectorFieldDefinition: + key: str + label: str + + +class TargetSelectorField(Enum): + ASSETS = SelectorFieldDefinition( + key="assets", + label="Assets", + ) + ASSET_GROUPS = SelectorFieldDefinition( + key="asset-groups", + label="Asset groups", + ) + MANUAL = SelectorFieldDefinition( + key="manual", + label="Manual", + ) + + @property + def key(self) -> str: + return self.value.key + + @property + def label(self) -> str: + return self.value.label + + +class TargetProperty(Enum): + AUTOMATIC = "Automatic" + HOSTNAME = "Hostname" + SEEN_IP = "Seen IP" + LOCAL_IP = "Local IP (first)" + + @staticmethod + def default_value(value: str = "automatic"): + return value.lower() + + +class ShodanContracts: + def __init__(self, config: ConfigLoader): + # Load configuration file + self.config = config + + # -- CONFIG -- + @staticmethod + def _base_contract_config(): + return ContractConfig( + type=TYPE, + label={ + SupportedLanguage.en: "Shodan", + SupportedLanguage.fr: "Shodan", + }, + color_dark="#ff5722", + color_light="#ff5722", + expose=True, + ) + + # -- BUILDER CONTRACT FIELDS -- + @staticmethod + def _build_target_selector(selector_default_value: str) -> ContractSelect: + + prefix = "only_" + + choices = { + item_selector.key: item_selector.label + for item_selector in TargetSelectorField + } + effective_default = selector_default_value + default_start_with_only = selector_default_value.startswith(prefix) + + if default_start_with_only: + effective_default = selector_default_value.removeprefix(prefix) + choices = {effective_default: choices[effective_default]} + + return ContractSelect( + key=InjectorKey.TARGET_SELECTOR_KEY, + label="Type of targets", + defaultValue=[effective_default], + mandatory=True, + choices=choices, + ) + + @staticmethod + def _build_field(field_type: TypeOfFields) -> ContractElement: + + builder_contract_fields_mapping = { + "field_assets": lambda **kwargs: ContractAsset( + cardinality=ContractCardinality.Multiple, **kwargs + ), + "field_asset_groups": lambda **kwargs: ContractAssetGroup( + cardinality=ContractCardinality.Multiple, **kwargs + ), + "field_assets_property": lambda **kwargs: ContractSelect( + key=InjectorKey.TARGET_PROPERTY_SELECTOR_KEY, + defaultValue=[TargetProperty.default_value()], + choices={ + item_property.name.lower(): item_property.value + for item_property in TargetProperty + }, + **kwargs + ), + } + + field_type_config = field_type.value + builder = builder_contract_fields_mapping[field_type_config.name] + + return builder( + label=field_type_config.label, + mandatory=field_type_config.mandatory, + mandatoryConditionFields=[InjectorKey.TARGET_SELECTOR_KEY], + mandatoryConditionValues={ + InjectorKey.TARGET_SELECTOR_KEY: field_type_config.target + }, + visibleConditionFields=[InjectorKey.TARGET_SELECTOR_KEY], + visibleConditionValues={ + InjectorKey.TARGET_SELECTOR_KEY: field_type_config.target + }, + ) + + @staticmethod + def _build_auto_create_assets_checkbox() -> ContractElement: + return ContractCheckbox( + key=InjectorKey.AUTO_CREATE_ASSETS, + label="Auto-Create assets", + defaultValue=False, + mandatory=False, + ) + + @staticmethod + def _build_expectations(): + return ContractExpectations( + key=InjectorKey.EXPECTATIONS_KEY, + label="Expectations", + mandatory=False, + cardinality=ContractCardinality.Multiple, + predefinedExpectations=[ + Expectation( + expectation_type=ExpectationType.vulnerability, + expectation_name="Not vulnerable", + expectation_description="", + expectation_score=100, + expectation_expectation_group=False, + ) + ], + ) + + def _base_fields(self, selector_default_value: str) -> list[ContractElement]: + + # Build Target Selector (Assets, Asset Groups, Manual) + target_selector = self._build_target_selector(selector_default_value) + + # Build all fields + target_assets = self._build_field(TypeOfFields.ASSETS) + target_asset_groups = self._build_field(TypeOfFields.ASSET_GROUPS) + target_assets_property = self._build_field(TypeOfFields.ASSETS_PROPERTY) + + # Build Checkbox Auto-Create Assets + checkbox_auto_create_assets = self._build_auto_create_assets_checkbox() + + # Build expectations + expectations = self._build_expectations() + + return [ + target_selector, + target_assets, + target_asset_groups, + target_assets_property, + checkbox_auto_create_assets, + expectations, + ] + + # -- OUTPUTS -- + @staticmethod + def _base_outputs(): + return [] + + def _build_contract( + self, + contract_id: str, + contract_cls: ( + CloudProviderAssetDiscovery + | CriticalPortsAndExposedAdminInterface + | CustomQuery + | CVEEnumeration + | CVESpecificWatchlist + | DomainDiscovery + | IPEnumeration + ), + contract_selector_default: str, + ) -> Contract: + return contract_cls.contract( + contract_id=contract_id, + contract_config=self._base_contract_config(), + contract_with_specific_fields=contract_cls.contract_with_specific_fields( + base_fields=self._base_fields(contract_selector_default), + source_selector_key=InjectorKey.TARGET_SELECTOR_KEY, + target_selector_field=TargetSelectorField.MANUAL.key, + ), + contract_with_specific_outputs=contract_cls.contract_with_specific_outputs( + self._base_outputs() + ), + ) + + def contracts(self): + + selector_default = "only_manual" + + shodan_contract_definitions = [ + ( + ShodanContractId.CLOUD_PROVIDER_ASSET_DISCOVERY, + CloudProviderAssetDiscovery, + selector_default, + ), + ( + ShodanContractId.CRITICAL_PORTS_AND_EXPOSED_ADMIN_INTERFACE, + CriticalPortsAndExposedAdminInterface, + selector_default, + ), + (ShodanContractId.CUSTOM_QUERY, CustomQuery, selector_default), + (ShodanContractId.CVE_ENUMERATION, CVEEnumeration, selector_default), + ( + ShodanContractId.CVE_SPECIFIC_WATCHLIST, + CVESpecificWatchlist, + selector_default, + ), + (ShodanContractId.DOMAIN_DISCOVERY, DomainDiscovery, selector_default), + (ShodanContractId.IP_ENUMERATION, IPEnumeration, selector_default), + ] + + contracts = [ + self._build_contract(contract_id, contract_cls, contract_selector_default) + for contract_id, contract_cls, contract_selector_default in shodan_contract_definitions + ] + + return {"data": prepare_contracts(contracts)} diff --git a/shodan/shodan/img/icon-shodan.png b/shodan/shodan/img/icon-shodan.png new file mode 100644 index 00000000..c44f0ced Binary files /dev/null and b/shodan/shodan/img/icon-shodan.png differ diff --git a/shodan/shodan/injector/__init__.py b/shodan/shodan/injector/__init__.py new file mode 100644 index 00000000..b8d32ab3 --- /dev/null +++ b/shodan/shodan/injector/__init__.py @@ -0,0 +1,19 @@ +from shodan.contracts import ( + CloudProviderAssetDiscovery, + CriticalPortsAndExposedAdminInterface, + CustomQuery, + CVEEnumeration, + CVESpecificWatchlist, + DomainDiscovery, + IPEnumeration, +) + +__all__ = [ + "CloudProviderAssetDiscovery", + "CriticalPortsAndExposedAdminInterface", + "CustomQuery", + "CVEEnumeration", + "CVESpecificWatchlist", + "DomainDiscovery", + "IPEnumeration", +] diff --git a/shodan/shodan/injector/openaev_shodan.py b/shodan/shodan/injector/openaev_shodan.py new file mode 100644 index 00000000..66984141 --- /dev/null +++ b/shodan/shodan/injector/openaev_shodan.py @@ -0,0 +1,184 @@ +import time +from datetime import datetime, timezone + +from pyoaev.helpers import OpenAEVInjectorHelper + +from shodan.contracts import ( + CloudProviderAssetDiscovery, + CriticalPortsAndExposedAdminInterface, + CustomQuery, + CVEEnumeration, + CVESpecificWatchlist, + DomainDiscovery, + InjectorKey, + IPEnumeration, + ShodanContractId, +) +from shodan.models import ConfigLoader +from shodan.services import ShodanClientAPI, Utils + +LOG_PREFIX = "[SHODAN_INJECTOR]" + + +class ShodanInjector: + def __init__(self, config: ConfigLoader, helper: OpenAEVInjectorHelper): + """Initialize the Injector with necessary configurations""" + + # Load configuration file and connection helper + self.config = config + self.helper = helper + self.shodan_client_api = ShodanClientAPI(self.config, self.helper) + self.utils = Utils() + + def _prepare_output_message( + self, contract_name: str, inject_content: dict, results, user_info: dict + ): + self.helper.injector_logger.info( + f"{LOG_PREFIX} - Start preparing the output message rendering.", + ) + # Retrieving the contract-specific output trace configuration. + output_trace_config = { + "CLOUD_PROVIDER_ASSET_DISCOVERY": CloudProviderAssetDiscovery.output_trace_config(), + "CRITICAL_PORTS_AND_EXPOSED_ADMIN_INTERFACE": CriticalPortsAndExposedAdminInterface.output_trace_config(), + "CUSTOM_QUERY": CustomQuery.output_trace_config(), + "CVE_ENUMERATION": CVEEnumeration.output_trace_config(), + "CVE_SPECIFIC_WATCHLIST": CVESpecificWatchlist.output_trace_config(), + "DOMAIN_DISCOVERY": DomainDiscovery.output_trace_config(), + "IP_ENUMERATION": IPEnumeration.output_trace_config(), + } + if contract_name not in output_trace_config: + self.helper.injector_logger.error( + f"{LOG_PREFIX} - The contract name is unknown.", + {"contract_name": contract_name}, + ) + raise ValueError(f"{LOG_PREFIX} - The contract name is unknown.") + + contract_output_trace_config = output_trace_config.get(contract_name) + + # Data Sections Info + usage_limits = user_info.get("usage_limits", {}) + data_sections_info = { + "plan": user_info.get("plan"), + "scan_credits_remaining": f"{user_info.get('scan_credits')} / {usage_limits.get('scan_credits')}", + "query_credits_remaining": f"{user_info.get('query_credits')} / {usage_limits.get('query_credits')}", + } + + # Data Section External API + results_data = results.get("data") + + rendering_output_message = self.utils.generate_output_message( + output_trace_config=contract_output_trace_config, + data_sections_config=[inject_content], + data_sections_info=[data_sections_info], + data_sections_external_api=results_data, + auto_create_assets=inject_content.get("auto_create_assets", None), + ) + self.helper.injector_logger.info( + f"{LOG_PREFIX} - Finalization of the preparation of the output message rendering.", + ) + return rendering_output_message + + def _shodan_execution(self, data): + + # Contract information + contract_id = data["injection"]["inject_injector_contract"]["convertedContent"][ + "contract_id" + ] + contract_name = ShodanContractId(contract_id).name + + inject_content = data["injection"]["inject_content"] + selector_key = inject_content[InjectorKey.TARGET_SELECTOR_KEY] + + self.helper.injector_logger.info( + f"{LOG_PREFIX} - Starting the execution of Shodan...", + { + "contract_id": contract_id, + "contract_name": contract_name, + "type_of_target": selector_key, + }, + ) + if selector_key == "manual": + shodan_results, shodan_credit_user = ( + self.shodan_client_api.process_shodan_search( + contract_id, inject_content + ) + ) + + output_json = "" + output_message = self._prepare_output_message( + contract_name, inject_content, shodan_results, shodan_credit_user + ) + return output_json, output_message + + elif selector_key == "assets": + output_json = "" + output_message = "Assets - Currently not supported" + return output_json, output_message + + elif selector_key == "asset-groups": + output_json = "" + output_message = "Asset-groups - Currently not supported" + return output_json, output_message + + else: + self.helper.injector_logger.error( + f"{LOG_PREFIX} - Invalid selector key, expected keys 'manual', 'assets', or 'asset-groups", + {"selector_key": selector_key}, + ) + raise ValueError( + f"{LOG_PREFIX} - Invalid selector key, expected keys 'manual', 'assets', or 'asset-groups'." + ) + + def process_message(self, data: dict) -> None: + # Initialization to get the current start utc iso format. + start_utc_isoformat = datetime.now(timezone.utc).isoformat(timespec="seconds") + self.helper.injector_logger.info( + f"{LOG_PREFIX} - Triggering of the Shodan injector...", + { + "start_utc_isoformat": start_utc_isoformat, + }, + ) + start = time.time() + inject_id = data["injection"]["inject_id"] + + # Notify API of reception and expected number of operations + reception_data = {"tracking_total_count": 1} + self.helper.api.inject.execution_reception( + inject_id=inject_id, data=reception_data + ) + + # Execute inject + try: + output_json, output_message = self._shodan_execution(data) + execution_duration = int(time.time() - start) + callback_data = { + "execution_message": output_message, + # "execution_output_structured": json.dumps(result["outputs"]), + "execution_status": "SUCCESS", + "execution_duration": execution_duration, + "execution_action": "complete", + } + self.helper.api.inject.execution_callback( + inject_id=inject_id, data=callback_data + ) + self.helper.injector_logger.info( + f"{LOG_PREFIX} - The injector has completed its execution.", + {"execution_duration": f"{execution_duration} s"}, + ) + + except Exception as err: + self.helper.injector_logger.error( + f"{LOG_PREFIX} - An error has occurred", {"error": str(err)} + ) + callback_data = { + "execution_message": str(err), + "execution_status": "ERROR", + "execution_duration": int(time.time() - start), + "execution_action": "complete", + } + self.helper.api.inject.execution_callback( + inject_id=inject_id, data=callback_data + ) + + def start(self): + self.helper.listen(message_callback=self.process_message) diff --git a/shodan/shodan/models/__init__.py b/shodan/shodan/models/__init__.py new file mode 100644 index 00000000..f8751160 --- /dev/null +++ b/shodan/shodan/models/__init__.py @@ -0,0 +1,3 @@ +from shodan.models.configs.config_loader import ConfigLoader + +__all__ = ["ConfigLoader"] diff --git a/shodan/shodan/models/configs/__init__.py b/shodan/shodan/models/configs/__init__.py new file mode 100644 index 00000000..f37c3c0c --- /dev/null +++ b/shodan/shodan/models/configs/__init__.py @@ -0,0 +1,13 @@ +from shodan.models.configs.base_settings import _SettingsLoader +from shodan.models.configs.injector_configs import ( + _BaseInjectorConfig, + _BaseOpenAEVConfig, +) +from shodan.models.configs.shodan_configs import _ConfigLoaderShodan + +__all__ = [ + "_SettingsLoader", + "_BaseOpenAEVConfig", + "_BaseInjectorConfig", + "_ConfigLoaderShodan", +] diff --git a/shodan/shodan/models/configs/base_settings.py b/shodan/shodan/models/configs/base_settings.py new file mode 100644 index 00000000..993cd8f8 --- /dev/null +++ b/shodan/shodan/models/configs/base_settings.py @@ -0,0 +1,20 @@ +"""Base class for global config models.""" + +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class _SettingsLoader(BaseSettings): + """Base class for global config models. + + Provides common configuration settings and prevents attributes from being + modified after initialization by using frozen=True in the model config. + """ + + model_config = SettingsConfigDict( + frozen=True, + extra="allow", + env_nested_delimiter="_", + env_nested_max_split=1, + validate_default=True, + enable_decoding=False, + ) diff --git a/shodan/shodan/models/configs/config_loader.py b/shodan/shodan/models/configs/config_loader.py new file mode 100644 index 00000000..054a397b --- /dev/null +++ b/shodan/shodan/models/configs/config_loader.py @@ -0,0 +1,125 @@ +"""Base class for global config models.""" + +from pathlib import Path + +from pydantic import BaseModel, Field +from pydantic_settings import ( + BaseSettings, + DotEnvSettingsSource, + EnvSettingsSource, + PydanticBaseSettingsSource, + YamlConfigSettingsSource, +) + +from shodan.models.configs import ( + _BaseInjectorConfig, + _BaseOpenAEVConfig, + _ConfigLoaderShodan, + _SettingsLoader, +) + + +class _BaseInjectorConfigHelperAdapter: + def __init__(self, data: dict): + self.data = data + + def get_conf(self, key, default=None): + value = self.data.get(key, default) + if isinstance(value, dict) and "data" in value: + value = value["data"] + return value + + +class _BaseInjectorConfigUtils: + + def to_flatten(self, contracts=None) -> dict: + flatten_config = {} + for field_name in ["openaev", "injector"]: + value = getattr(self, field_name, None) + if isinstance(value, BaseModel): + for subfield, subvalue in value.__dict__.items(): + flatten_config[f"{field_name}_{subfield}"] = str(subvalue) + elif value is not None: + flatten_config[field_name] = str(value) + if contracts: + flatten_config["injector_contracts"] = contracts + return flatten_config + + def to_config_injector_helper_adapter( + self, contracts + ) -> _BaseInjectorConfigHelperAdapter: + """Returns an OpenAEVInjectorHelper-compatible object""" + flatten_dict = self.to_flatten(contracts) + return _BaseInjectorConfigHelperAdapter(flatten_dict) + + +class ConfigLoader(_BaseInjectorConfigUtils, _SettingsLoader): + """Configuration loader for the injector.""" + + openaev: _BaseOpenAEVConfig = Field( + default_factory=_BaseOpenAEVConfig, + description="Base OpenAEV configurations.", + ) + injector: _BaseInjectorConfig = Field( + default_factory=_BaseInjectorConfig, + description="Base Injector configurations.", + ) + shodan: _ConfigLoaderShodan = Field( + default_factory=_ConfigLoaderShodan, + description="Shodan configurations.", + ) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource]: + """Pydantic settings customisation sources. + + Defines the priority order for loading configuration settings: + 1. .env file (if exists) + 2. config.yml file (if exists) + 3. Environment variables (fallback) + + Args: + settings_cls: The settings class being configured. + init_settings: Initialization settings source. + env_settings: Environment variables settings source. + dotenv_settings: .env file settings source. + file_secret_settings: File secrets settings source. + + Returns: + Tuple containing the selected settings source. + + """ + env_path = Path(__file__).parents[2] / ".env" + yaml_path = Path(__file__).parents[2] / "config.yml" + + if env_path.exists(): + return ( + DotEnvSettingsSource( + settings_cls, + env_file=env_path, + env_ignore_empty=True, + env_file_encoding="utf-8", + ), + ) + elif yaml_path.exists(): + return ( + YamlConfigSettingsSource( + settings_cls, + yaml_file=yaml_path, + yaml_file_encoding="utf-8", + ), + ) + else: + return ( + EnvSettingsSource( + settings_cls, + env_ignore_empty=True, + ), + ) diff --git a/shodan/shodan/models/configs/injector_configs.py b/shodan/shodan/models/configs/injector_configs.py new file mode 100644 index 00000000..9e4c9c40 --- /dev/null +++ b/shodan/shodan/models/configs/injector_configs.py @@ -0,0 +1,51 @@ +"""Base class for global config models.""" + +from abc import ABC +from typing import Literal + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + HttpUrl, +) + + +class BaseConfigModel(BaseModel, ABC): + """Base class for global config models + To prevent attributes from being modified after initialization. + """ + + model_config = ConfigDict( + extra="allow", str_min_length=1, frozen=True, validate_default=True + ) + + +class _BaseOpenAEVConfig(BaseConfigModel, ABC): + url: HttpUrl = Field( + description="The base URL of the OpenAEV instance.", + ) + token: str = Field( + description="The API token to connect to OpenAEV.", + ) + + +class _BaseInjectorConfig(BaseConfigModel, ABC): + """Base class for connector configuration.""" + + id: str = Field( + default="shodan--a87488ad-2c72-4592-b429-69259d7bcef1", + description="A unique UUIDv4 identifier for this injector instance.", + ) + name: str = Field( + default="Shodan", + description="Name of the injector.", + ) + type: str = Field( + default="openaev_shodan", + description="Identifies the functional type of the injector in OpenAEV", + ) + log_level: Literal["debug", "info", "warning", "error"] = Field( + default="error", + description="The minimum level of logs to display.", + ) diff --git a/shodan/shodan/models/configs/shodan_configs.py b/shodan/shodan/models/configs/shodan_configs.py new file mode 100644 index 00000000..a28f2cdc --- /dev/null +++ b/shodan/shodan/models/configs/shodan_configs.py @@ -0,0 +1,41 @@ +"""Configuration for Shodan injector.""" + +from datetime import timedelta + +from pydantic import ( + Field, + PositiveInt, + SecretStr, +) + +from shodan.models.configs import _SettingsLoader + + +class _ConfigLoaderShodan(_SettingsLoader): + """Shodan API configuration settings.""" + + base_url: str = Field( + default="https://api.shodan.io", + description="This is the base URL for the Shodan API.", + ) + api_key: SecretStr = Field( + description="This is the API key for the Shodan API.", + ) + api_leaky_bucket_rate: PositiveInt = Field( + default=10, + description="Bucket refill rate (in tokens per second). Controls the rate at which API calls are allowed. " + "For example, a rate of 10 means that 10 calls can be made per second, if the bucket is not empty.", + ) + api_leaky_bucket_capacity: PositiveInt = Field( + default=10, + description="Maximum bucket capacity (in tokens). Defines the number of calls that can be made immediately in a " + "burst. Once the bucket is empty, it refills at the rate defined by 'api_leaky_bucket_rate'.", + ) + api_retry: PositiveInt = Field( + default=5, + description="Maximum number of attempts (including the initial request) in case of API failure.", + ) + api_backoff: timedelta = Field( + default="PT30S", + description="Maximum exponential backoff delay between retry attempts (ISO 8601 duration format).", + ) diff --git a/shodan/shodan/services/__init__.py b/shodan/shodan/services/__init__.py new file mode 100644 index 00000000..53803ca8 --- /dev/null +++ b/shodan/shodan/services/__init__.py @@ -0,0 +1,7 @@ +from shodan.services.client_api import ShodanClientAPI +from shodan.services.utils import Utils + +__all__ = [ + "ShodanClientAPI", + "Utils", +] diff --git a/shodan/shodan/services/client_api.py b/shodan/shodan/services/client_api.py new file mode 100644 index 00000000..6af23155 --- /dev/null +++ b/shodan/shodan/services/client_api.py @@ -0,0 +1,380 @@ +import re +from dataclasses import dataclass +from enum import Enum +from urllib.parse import quote_plus, urljoin + +import requests +from limiter import Limiter +from pyoaev.helpers import OpenAEVInjectorHelper +from tenacity import RetryError, retry, stop_after_attempt, wait_exponential_jitter + +from shodan.contracts import ShodanContractId +from shodan.models import ConfigLoader + +LOG_PREFIX = "[SHODAN_CLIENT_API]" + + +class MissingRequiredFieldError(ValueError): + pass + + +@dataclass +class ShodanRestAPIDefinition: + http_method: str + endpoint: str + + +class ShodanRestAPI(Enum): + SEARCH_SHODAN = ShodanRestAPIDefinition( + http_method="GET", + endpoint="shodan/host/search", + ) + API_PLAN_INFORMATION = ShodanRestAPIDefinition( + http_method="GET", endpoint="api-info" + ) + + @property + def http_method(self) -> str: + return self.value.http_method + + @property + def endpoint(self) -> str: + return self.value.endpoint + + +class ShodanClientAPI: + def __init__(self, config: ConfigLoader, helper: OpenAEVInjectorHelper): + """Initialize the Injector with necessary configurations""" + + # Load configuration file and connection helper + self.config = config + self.helper = helper + + self.base_url = self.config.shodan.base_url + self.api_key = self.config.shodan.api_key + + self.api_retry = self.config.shodan.api_retry + self.api_backoff = self.config.shodan.api_backoff.total_seconds() + + # Limiter config + self.rate_limiter = Limiter( + rate=self.config.shodan.api_leaky_bucket_rate, + capacity=self.config.shodan.api_leaky_bucket_capacity, + bucket="shodan", + ) + + @staticmethod + def _split_target(raw_input: str) -> list[str]: + return [target for target in re.split(r"[,\s]+", raw_input or "") if target] + + @staticmethod + def _secure_url(url: str) -> str: + if not url: + return url + if "?query" in url: + return url.split("&key=")[0] + return url.split("?key=")[0] + + @staticmethod + def _build_query(query_params: dict[str, tuple[str, str]]) -> str: + query = ["query="] + for key, (value, operator) in query_params.items(): + if operator == "and": + query.append(f"{key}:{value} ") + elif operator == "or": + query.append(f"{key}:{value},") + else: + query.append(f"{key}:{value}") + break + return "".join(query).strip() + + def _build_url( + self, + endpoint: str, + query_params: str | None = None, + is_custom_query: bool = False, + ) -> str: + url = urljoin(self.base_url, endpoint) + api_key = f"key={self.api_key.get_secret_value()}" + if not query_params and "?query=" not in url: + return f"{url}?{api_key}" + if is_custom_query and "?query=" in url: + return f"{url}&{api_key}" + return f"{url}?{query_params}&{api_key}" + + # SECTION INFO + def _get_user_info(self): + self.helper.injector_logger.info( + f"{LOG_PREFIX} - Preparation for user quota recovery...", + ) + + return self._process_request( + raw_input="user_info", + request_api=ShodanRestAPI.API_PLAN_INFORMATION, + ) + + # CONTRACT - CVE ENUMERATION + def _get_cve_enumeration(self, inject_content): + hostname = inject_content.get("hostname") + if not hostname: + raise MissingRequiredFieldError( + f"{LOG_PREFIX} - The 'Hostname' field is required and cannot be empty." + ) + + filters = { + "has_vuln": ("true", "and"), + "hostname": ("{target},*.{target}", "or"), + "org": (inject_content.get("organization") or "{target}", "end"), + } + return self._process_request( + raw_input=hostname, + request_api=ShodanRestAPI.SEARCH_SHODAN, + filters_template=filters, + ) + + # CONTRACT - CVE SPECIFIC WATCHLIST + def _get_cve_specific_watchlist(self, inject_content): + hostname = inject_content.get("hostname") + if not hostname: + raise MissingRequiredFieldError( + f"{LOG_PREFIX} - The 'Hostname' field is required and cannot be empty." + ) + + filters = { + "vuln": (inject_content.get("vulnerability"), "and"), + "hostname": ("{target},*.{target}", "and"), + "org": (inject_content.get("organization") or "{target}", "end"), + } + return self._process_request( + raw_input=hostname, + request_api=ShodanRestAPI.SEARCH_SHODAN, + filters_template=filters, + ) + + # CONTRACT - CLOUD PROVIDER ASSET DISCOVERY + def _get_cloud_provider_asset_discovery(self, inject_content): + hostname = inject_content.get("hostname") + if not hostname: + raise MissingRequiredFieldError( + f"{LOG_PREFIX} - The 'Hostname' field is required and cannot be empty." + ) + + cloud_provider = inject_content.get("cloud_provider") + # Please note that the default values are by nature a list. + if isinstance(cloud_provider, list): + cloud_provider = ",".join(cloud_provider) + + filters = { + "cloud.provider": (cloud_provider, "and"), + "hostname": ("{target},*.{target}", "or"), + "org": (inject_content.get("organization") or "{target}", "end"), + } + return self._process_request( + raw_input=hostname, + request_api=ShodanRestAPI.SEARCH_SHODAN, + filters_template=filters, + ) + + # CONTRACT - CRITICAL PORTS AND EXPOSED ADMIN INTERFACE + def _get_critical_ports_and_exposed_admin_interface(self, inject_content): + hostname = inject_content.get("hostname") + if not hostname: + raise MissingRequiredFieldError( + f"{LOG_PREFIX} - The 'Hostname' field is required and cannot be empty." + ) + + port = inject_content.get("port") + # Please note that the default values are by nature a list. + if isinstance(port, list): + port = ",".join(port) + + filters = { + "port": (port, "and"), + "hostname": ("{target},*.{target}", "or"), + "org": (inject_content.get("organization") or "{target}", "end"), + } + return self._process_request( + raw_input=hostname, + request_api=ShodanRestAPI.SEARCH_SHODAN, + filters_template=filters, + ) + + # CONTRACT - CUSTOM QUERY + def _get_custom_query(self, inject_content): + custom_query = inject_content.get("custom_query") + http_method = inject_content.get("http_method") + if not custom_query: + raise MissingRequiredFieldError( + f"{LOG_PREFIX} - The 'Custom Query' field is required and cannot be empty." + ) + + return self._process_request( + raw_input=custom_query, + request_api=None, + filters_template=None, + is_custom_query=True, + http_method_custom_query=http_method, + ) + + # CONTRACT - DOMAIN DISCOVERY + def _get_domain_discovery(self, inject_content): + hostname = inject_content.get("hostname") + if not hostname: + raise MissingRequiredFieldError( + f"{LOG_PREFIX} - The 'hostname' field is required and cannot be empty." + ) + + filters = { + "hostname": ("{target},*.{target}", "or"), + "org": (inject_content.get("organization") or "{target}", "end"), + } + return self._process_request( + raw_input=hostname, + request_api=ShodanRestAPI.SEARCH_SHODAN, + filters_template=filters, + ) + + # CONTRACT - IP ENUMERATION + def _get_ip_enumeration(self, inject_content): + ip = inject_content.get("ip") + if not ip: + raise MissingRequiredFieldError( + f"{LOG_PREFIX} - The 'ip' field is required and cannot be empty." + ) + + filters = { + "ip": ("{target}", "end"), + } + + return self._process_request( + raw_input=ip, + request_api=ShodanRestAPI.SEARCH_SHODAN, + filters_template=filters, + ) + + def _process_request( + self, + raw_input: str, + request_api: ShodanRestAPI | None, + filters_template: dict = None, + is_custom_query: bool = False, + http_method_custom_query: str | None = None, + ): + + if is_custom_query: + targets = [raw_input] + http_method = http_method_custom_query + endpoint_template = raw_input + else: + targets = self._split_target(raw_input) + http_method = request_api.value.http_method + endpoint_template = request_api.value.endpoint + + result = None + results = [] + for target in targets: + new_endpoint = endpoint_template + query_params = None + encoded_for_shodan = None + + if filters_template: + filters = {} + for key, (target_value, operator) in filters_template.items(): + filters[key] = (target_value.format(target=target), operator) + + query_params = self._build_query(filters) + encoded_for_shodan = quote_plus(query_params, safe="=:,*.") + target_url = self._build_url( + endpoint=new_endpoint, + query_params=encoded_for_shodan if filters_template else query_params, + is_custom_query=is_custom_query, + ) + + try: + result = self._request_data( + method=http_method, + url=target_url, + ) + results.append( + { + "target": target, + "url": f"{http_method} {self._secure_url(target_url)}", + "result": result, + } + ) + except RetryError as retry_exc: + inner_exception = retry_exc.last_attempt.exception() + + request = inner_exception.request + request_filtered = { + "method": request.method, + "url": f"{http_method} {self._secure_url(target_url)}", + } + + response = inner_exception.response + response_filtered = { + "status_code": response.status_code, + "reason": response.reason, + "error": response.text, + } + + results.append( + { + "target": target, + "is_error": True, + "request": request_filtered, + "response": response_filtered, + } + ) + return ( + result + if targets == ["user_info"] + else {"targets": targets, "data": results} + ) + + def _request_data(self, method: str, url: str): + @retry( + stop=stop_after_attempt(max_attempt_number=self.api_retry), + wait=wait_exponential_jitter( + initial=1, max=self.api_backoff, exp_base=2, jitter=1 + ), + ) + def _retry_wrapped(): + response = requests.request(method=method, url=url) + response.raise_for_status() + return response.json() + + with self.rate_limiter: + return _retry_wrapped() + + def process_shodan_search( + self, contract_id: ShodanContractId, inject_content: dict + ): + self.helper.injector_logger.info( + f"{LOG_PREFIX} - Starting the Shodan search process...", + ) + contract_handler = { + ShodanContractId.CVE_ENUMERATION: self._get_cve_enumeration, + ShodanContractId.CVE_SPECIFIC_WATCHLIST: self._get_cve_specific_watchlist, + ShodanContractId.CLOUD_PROVIDER_ASSET_DISCOVERY: self._get_cloud_provider_asset_discovery, + ShodanContractId.CRITICAL_PORTS_AND_EXPOSED_ADMIN_INTERFACE: self._get_critical_ports_and_exposed_admin_interface, + ShodanContractId.CUSTOM_QUERY: self._get_custom_query, + ShodanContractId.DOMAIN_DISCOVERY: self._get_domain_discovery, + ShodanContractId.IP_ENUMERATION: self._get_ip_enumeration, + } + + contract = contract_handler.get(contract_id) + if not contract: + self.helper.injector_logger.error( + f"{LOG_PREFIX} - The contract ID is invalid.", + {"contract_id": contract_id}, + ) + raise ValueError(f"{LOG_PREFIX} - The contract ID is invalid.") + + results = contract(inject_content) + shodan_credit_user = self._get_user_info() + + self.helper.injector_logger.info( + f"{LOG_PREFIX} - Finalization of the Shodan search process.", + ) + return results, shodan_credit_user diff --git a/shodan/shodan/services/utils.py b/shodan/shodan/services/utils.py new file mode 100644 index 00000000..8bcdbd49 --- /dev/null +++ b/shodan/shodan/services/utils.py @@ -0,0 +1,845 @@ +import json +from datetime import datetime +from enum import Enum +from typing import Any + +from rich import box +from rich.align import Align +from rich.console import Console, Group +from rich.json import JSON +from rich.panel import Panel +from rich.rule import Rule +from rich.table import Table +from rich.text import Text +from rich.tree import Tree + + +class OutputIcons(Enum): + SUCCESS = "✅" + FAILED = "❌" + SEARCH = "🔍" + CONFIG = "⚙️" + INFO = "ℹ️" + API = "🌐" + + +class Utils: + + @staticmethod + def _get_trace_config(output_trace_config: dict, path: str, default=None): + keys = path.split(".") + try: + for key in keys: + output_trace_config = output_trace_config[key] + return output_trace_config + except (KeyError, TypeError): + return default + + def _add_value_to_tree(self, tree: Tree, key: str, value: Any) -> None: + if isinstance(value, list): + node = tree.add(f"{key}:") + for item in value: + if isinstance(item, dict): + for sub_key, sub_value in item.items(): + if sub_value is None or sub_value == "": + continue + self._add_value_to_tree(node, sub_key, sub_value) + else: + node.add(str(item)) + elif isinstance(value, dict): + node = tree.add(f"{key}:") + for sub_key, sub_value in value.items(): + if sub_value is None or sub_value == "": + continue + self._add_value_to_tree(node, sub_key, sub_value) + else: + tree.add(f"{key}: {value}") + + def _build_tree( + self, + tree: Tree, + data: list[dict[str, Any]], + keys_to_exclude: list[str], + keys_list_to_string: list[str], + ): + keys_to_exclude = keys_to_exclude or [] + for section in data: + if not isinstance(section, dict): + continue + + for key, value in section.items(): + if key in keys_to_exclude or value is None or not value: + continue + + if key in keys_list_to_string and isinstance(value, list): + parts = [] + + for item in value: + if isinstance(item, dict): + parts.extend(f"{k}:{v}" for k, v in item.items()) + else: + parts.append(str(item)) + value = ",".join(parts) + + self._add_value_to_tree(tree, key, value) + + @staticmethod + def _get_output_icon(icon_name: str = "") -> str: + if not icon_name: + return "" + + icon = icon_name.upper() + if icon in OutputIcons.__members__: + return OutputIcons[icon].value + return "" + + # MAKE JSON + def _make_json(self, data, output_trace_config: dict): + options_show_json_is_active = self._get_trace_config( + output_trace_config, "options.show_json.is_active", False + ) + if not options_show_json_is_active: + return None + + options_show_json_indent = self._get_trace_config( + output_trace_config, "options.show_json.indent", 2 + ) + options_show_json_sort_keys = self._get_trace_config( + output_trace_config, "options.show_json.sort_keys", False + ) + + if isinstance(data, (dict, list)): + return JSON.from_data( + data, + indent=options_show_json_indent, + sort_keys=options_show_json_sort_keys, + ) + + if isinstance(data, str): + return JSON.from_data(data) + + return JSON.from_data({"data": str(data)}) + + @staticmethod + def _make_header(output_trace_config: dict): + options = output_trace_config.get("options", {}) + show_header = options.get("show_header", {}) + + if not show_header.get("is_active", True): + return None + + header = output_trace_config.get("header", {}) + title = header.get("title", "") + show_subtitle = show_header.get("show_subtitle", True) + now = datetime.now().isoformat(sep=" ", timespec="seconds") + subtitle = header.get("subtitle", now) or now + + return Panel( + renderable=Align.center(title), + padding=(1, 1), + subtitle=subtitle if show_subtitle else None, + subtitle_align="center", + ) + + # MAKE SEPARATOR + def _make_separator(self, output_trace_config: dict) -> Rule | None: + options_show_separator = self._get_trace_config( + output_trace_config, "options.show_separator.is_active", default=False + ) + if not options_show_separator: + return None + return Rule(characters="─") + + # MAKE SECTION CONFIG + def _make_section_config( + self, output_trace_config: dict, data_sections_config: list[dict] + ): + options_show_sections = self._get_trace_config( + output_trace_config, "options.show_sections.is_active", default=True + ) + options_show_sec_config = self._get_trace_config( + output_trace_config, "options.show_sections.sec_config", default=True + ) + + if not (options_show_sections and options_show_sec_config): + return None + + sec_config_icon = self._get_output_icon( + self._get_trace_config( + output_trace_config, "sections_config.header.icon", default="CONFIG" + ) + ) + sec_config_title = self._get_trace_config( + output_trace_config, + "sections_config.header.title", + default="[CONFIG] Summary of all configurations used for the contract.", + ) + keys_list_to_string = self._get_trace_config( + output_trace_config, "sections_config.keys_list_to_string", default=[] + ) + keys_to_exclude = self._get_trace_config( + output_trace_config, "sections_config.keys_to_exclude", default=[] + ) + + section_tree = Tree( + f"{sec_config_icon} {sec_config_title}".strip(), guide_style="bold" + ) + self._build_tree( + tree=section_tree, + data=data_sections_config, + keys_to_exclude=keys_to_exclude, + keys_list_to_string=keys_list_to_string, + ) + return section_tree + + # MAKE SECTION INFO + def _make_section_info( + self, output_trace_config: dict, data_sections_info: list[dict] + ): + options_show_sections = self._get_trace_config( + output_trace_config, "options.show_sections.is_active", default=True + ) + options_show_sec_info = self._get_trace_config( + output_trace_config, "options.show_sections.sec_info", default=True + ) + + if not (options_show_sections and options_show_sec_info): + return None + + sec_info_icon = self._get_output_icon( + self._get_trace_config( + output_trace_config, "sections_info.header.icon", default="INFO" + ) + ) + sec_info_title = self._get_trace_config( + output_trace_config, + "sections_info.header.title", + default="[INFO] The Injector information", + ) + keys_list_to_string = self._get_trace_config( + output_trace_config, "sections_info.keys_list_to_string", default=[] + ) + keys_to_exclude = self._get_trace_config( + output_trace_config, "sections_info.keys_to_exclude", default=[] + ) + + section_tree = Tree( + f"{sec_info_icon} {sec_info_title}".strip(), guide_style="bold" + ) + self._build_tree( + tree=section_tree, + data=data_sections_info, + keys_to_exclude=keys_to_exclude, + keys_list_to_string=keys_list_to_string, + ) + return section_tree + + @staticmethod + def _count_at_path(result: dict, path: str) -> int: + current = result + for key in path.split("."): + if not isinstance(current, dict): + return 0 + current = current.get(key, None) + if current is None: + return 0 + if isinstance(current, list): + return len(current) + else: + return 1 + + def _prepare_call_details( + self, output_trace_config: dict, data_sections_external_api: list[dict] + ): + results_success = [] + total_success_details_count = 0 + + results_failed = [] + total_call_failed_count = 0 + for result in data_sections_external_api: + + if result.get("is_error"): + request = result.get("request") + response = result.get("response") + + raw_error = response.get("error") + if isinstance(raw_error, str) and raw_error.startswith("{"): + error_message = json.loads(raw_error).get("error") + else: + error_message = raw_error + + call_failed_details = { + "data_target": result.get("target"), + "request": request.get("url"), + "error": f"{error_message} ({response.get('status_code')} - {response.get('reason')})", + } + total_call_failed_count += 1 + results_failed.append(call_failed_details) + else: + call_success_details = { + "data_target": result.get("target"), + "request": result.get("url"), + "result": result.get("result"), + } + results_success.append(call_success_details) + count_at_path = self._get_trace_config( + output_trace_config, + "sections_external_api.call_success.count_at_path", + default="", + ) + if count_at_path: + total_success_details_count += self._count_at_path( + call_success_details.get("result"), count_at_path + ) + + return (results_success, total_success_details_count), ( + results_failed, + total_call_failed_count, + ) + + # MAKE SECTION EXTERNAL API + def _make_section_external_api( + self, output_trace_config: dict, data_sections_external_api: list[dict] + ): + options_show_sections = self._get_trace_config( + output_trace_config, "options.show_sections.is_active", default=True + ) + options_show_sec_external_api = self._get_trace_config( + output_trace_config, "options.show_sections.sec_external_api", default=True + ) + + if not (options_show_sections and options_show_sec_external_api): + return None, None + + sec_external_api_icon = self._get_output_icon( + self._get_trace_config( + output_trace_config, "sections_external_api.header.icon", default="API" + ) + ) + sec_external_api_title = self._get_trace_config( + output_trace_config, + "sections_external_api.header.title", + default="[INJECTOR] Call API completed", + ) + section_tree = Tree( + f"{sec_external_api_icon} {sec_external_api_title}".strip(), + guide_style="bold", + ) + + # Add node call success + call_success_icon = self._get_output_icon( + self._get_trace_config( + output_trace_config, + "sections_external_api.call_success.icon", + default="SUCCESS", + ) + ) + call_success_title = self._get_trace_config( + output_trace_config, + "sections_external_api.call_success.title", + default="Call Success", + ) + success_node = section_tree.add( + f"{call_success_icon} {call_success_title}".strip() + ) + + # Add node call failed + call_failed_icon = self._get_output_icon( + self._get_trace_config( + output_trace_config, + "sections_external_api.call_failed.icon", + default="FAILED", + ) + ) + call_failed_title = self._get_trace_config( + output_trace_config, + "sections_external_api.call_failed.title", + default="Call Failed", + ) + failed_node = section_tree.add( + f"{call_failed_icon} {call_failed_title}".strip() + ) + + (results_success_details, total_success_details_count), ( + results_failed_details, + total_call_failed_count, + ) = self._prepare_call_details(output_trace_config, data_sections_external_api) + + # Add Total results (success and failed) + success_node.add(f"Total results: {total_success_details_count}") + failed_node.add(f"Total results: {total_call_failed_count}") + + # Add Details for call Success + success_details_node = success_node.add("Details:") + for result_detail in results_success_details: + target = result_detail.get("data_target") + result_details_count = 0 + request = result_detail.get("request") + + count_at_path = self._get_trace_config( + output_trace_config, + "sections_external_api.call_success.count_at_path", + default="", + ) + if count_at_path: + result_details_count += self._count_at_path( + result_detail.get("result"), count_at_path + ) + target_node = success_details_node.add( + f"• {target} → {result_details_count} results" + ) + target_node.add(f"Request: {request}") + + # Add Details for call Failed + failed_details_node = failed_node.add("Details:") + for result_detail in results_failed_details: + target = result_detail.get("data_target") + error = result_detail.get("error") + request = result_detail.get("request") + + target_node = failed_details_node.add(f"• {target}") + target_node.add(f"Error: {error}") + target_node.add(f"Request: {request}") + + return section_tree, results_success_details + + def _extract_level( + self, items: list[Any], parts_remaining: list[str], use_key: bool = False + ) -> list[Any]: + if not parts_remaining: + return items + + part = parts_remaining[0] + next_parts = parts_remaining[1:] + result = [] + + for item in items: + if isinstance(item, dict): + if part == "*": + # If there is a "next_parts", we retrieve the values + if next_parts: + sub_result = list(item.values()) + else: + # if we have a “*” and use_key is true, then we retrieve the keys rather than the values + sub_result = [k if use_key else v for k, v in item.items()] + + if next_parts: + sub_result = self._extract_level( + sub_result, next_parts, use_key + ) + + result.append(sub_result) + else: + value = item.get(part, []) + result.append(self._extract_level(value, next_parts, use_key)) + elif isinstance(item, list): + result.append(self._extract_level(item, parts_remaining, use_key)) + else: + if part == "*" and next_parts: + result.extend( + self._extract_level([items[item]], next_parts, use_key) + ) + else: + if part in items and not next_parts and isinstance(item, str): + if item == part: + result.append(items[part]) + else: + continue + else: + result.append(item) + return result + + def _extractor( + self, data: list[dict], path: str, use_key: bool = False + ) -> list[list[Any]]: + if not path: + return [] + + parts = path.split(".") + data_list = data if isinstance(data, list) else [data] + final_result = self._extract_level(data_list, parts, use_key) + return final_result + + @staticmethod + def _organizer_row( + tables_config_columns, + show_index_is_active, + index_start, + column_values, + column_extras, + ): + + single_columns = [ + values + for config, values in zip(tables_config_columns, column_values) + if config.get("mode") == "single" + ] + row_count = len(single_columns[0]) + + final_rows = [] + index = index_start + + for row_idx in range(row_count): + row_cells = [] + + if show_index_is_active: + row_cells.append(str(index)) + + for table_config_index, table_config in enumerate(tables_config_columns): + mode = table_config.get("mode", "inline") + + values = ( + column_values[table_config_index] + if table_config_index < len(column_values) + else [] + ) + extras = ( + column_extras[table_config_index] + if column_extras and table_config_index < len(column_extras) + else None + ) + + cell = "-" + if mode == "single": + if isinstance(values, list) and len(values) > 0: + for value in values: + if isinstance(value, list): + filtered = [ + str(v) for v in value if v not in (None, "") + ] + cell = ", ".join(filtered) if filtered else "-" + else: + val = values[row_idx] if row_idx < len(values) else None + cell = str(val) if val not in (None, "") else "-" + + elif values not in (None, ""): + cell = str(values) + else: + cell = "-" + + elif mode == "inline": + if row_idx == 0: + all_vals = [] + for val in values: + if isinstance(val, list): + all_vals.extend( + str(v) for v in val if v not in (None, "") + ) + elif val not in (None, ""): + all_vals.append(str(val)) + cell = ", ".join(all_vals) if all_vals else "-" + + else: + cell = "" + + elif mode == "align_to_single": + if row_idx < len(values): + val = values[row_idx] + extra = ( + extras[row_idx] + if extras and row_idx < len(extras) + else None + ) + if isinstance(val, list): + if val: + if extra: + cell = ", ".join( + f"{v} ({e if e is not None else '-'})" + for v, e in zip(val, extra or ["-"] * len(val)) + ) + else: + cell = ", ".join(str(v) for v in val) + else: + cell = "-" + elif val not in (None, ""): + if extra: + cell = f"{val} ({extra})" + else: + cell = str(val) + else: + cell = "-" + else: + cell = "-" + + elif mode == "repeat": + if values: + val = values + extra = extras if extras else None + + if isinstance(val, list): + if val: + val_join = ", ".join(str(v) for v in val) + cell = val_join + if extra and isinstance(extra, list): + extra_join = ", ".join(str(v) for v in extra) + cell = f"{val_join} ({extra_join})" + else: + cell = "-" + elif val not in (None, ""): + if extra: + cell = f"{val} ({extra})" + else: + cell = str(val) + else: + cell = "-" + else: + cell = "-" + + row_cells.append(cell) + final_rows.append(row_cells) + index += 1 + + return final_rows + + @staticmethod + def _rows_with_limit_cells(rows: list[list[str]], max_display_by_cell: int | None): + rows_with_limit = [] + + for row in rows: + main_row = [] + hidden_row = [] + has_hidden = False + + for cell in row: + if ( + isinstance(cell, str) + and "," in cell + and max_display_by_cell is not None + ): + items = [c.strip() for c in cell.split(",") if c.strip()] + if len(items) > max_display_by_cell: + shown = items[:max_display_by_cell] + hidden = len(items) - max_display_by_cell + + main_row.append(", ".join(shown)) + hidden_row.append(f"...(+{hidden} hidden)") + has_hidden = True + else: + main_row.append(", ".join(items)) + hidden_row.append("") + else: + main_row.append(cell) + hidden_row.append("") + + rows_with_limit.append(main_row) + if has_hidden: + rows_with_limit.append(hidden_row) + return rows_with_limit + + # MAKE TABLES + def _make_tables( + self, + output_trace_config: dict, + data_tables: list[dict], + auto_create_assets: bool | None, + ): + + options_show_tables_is_active = self._get_trace_config( + output_trace_config, "options.show_tables.is_active", default=True + ) + if not options_show_tables_is_active: + return [] + + show_index_is_active = self._get_trace_config( + output_trace_config, + "options.show_tables.show_index.is_active", + default=False, + ) + index_start = self._get_trace_config( + output_trace_config, "options.show_tables.show_index.index_start", default=0 + ) + show_lines = self._get_trace_config( + output_trace_config, "options.show_tables.show_lines", default=True + ) + max_display_by_cell = self._get_trace_config( + output_trace_config, "options.show_tables.max_display_by_cell", default=10 + ) + + tables = self._get_trace_config(output_trace_config, "tables", default=[]) + + tables_rendering = [] + for table_config in tables: + header_icon = self._get_output_icon( + self._get_trace_config(table_config, "header.icon", default="SEARCH") + ) + header_title = self._get_trace_config( + table_config, "header.title", default="" + ) + search_entity = self._get_trace_config( + table_config, "config.search_entity", default=None + ) + + table_rendering = [] + for data_table in data_tables: + result = data_table.get("result") + if header_title: + table_title = header_title.format(**result) + else: + table_title = ( + f"Asset(s) " + f"{'Created' if auto_create_assets else 'Not Created'} " + f"for {result.get(search_entity) or data_table.get("data_target")}" + ) + + table = Table( + title=f"{header_icon} {table_title}".strip(), + title_justify="left", + show_lines=show_lines, + box=box.HEAVY_HEAD if show_lines else None, + expand=True, + ) + + tables_config_columns = self._get_trace_config( + table_config, "config.columns" + ) + + if show_index_is_active: + table.add_column("#") + + column_values = [] + column_extras = [] + for config in tables_config_columns: + title = config.get("title", "-") + table.add_column(title, overflow="fold") + + path = config.get("path") + use_key = config.get("use_key", False) + values = self._extractor(result, path, use_key) + column_values.append(values[0]) + + extra_path = config.get("extra") + if extra_path: + extras = self._extractor(result, extra_path) + column_extras.append(extras[0]) + else: + column_extras.append(None) + + final_rows = self._organizer_row( + tables_config_columns, + show_index_is_active, + index_start, + column_values, + column_extras, + ) + + if not final_rows: + tables_rendering.append( + Panel( + renderable="No data found", + title=f"Result for {result.get(search_entity) or data_table.get("data_target")}", + title_align="left", + ) + ) + continue + + rows_with_limit_cell = self._rows_with_limit_cells( + final_rows, max_display_by_cell + ) + + for row_with_limit in rows_with_limit_cell: + table.add_row(*row_with_limit) + + table_rendering.append(table) + + tables_rendering.extend(table_rendering) + return tables_rendering + + def generate_output_message( + self, + output_trace_config: dict, + data_sections_config: list[dict], + data_sections_info: list[dict], + data_sections_external_api: list[dict], + auto_create_assets: bool | None, + ): + # Todo: Make split mode + + renderables = [] + + # Output Header + output_header = self._make_header(output_trace_config=output_trace_config) + if output_header: + renderables.append(Text("")) + separator = self._make_separator(output_trace_config=output_trace_config) + if separator: + renderables.append(separator) + renderables.append(output_header) + + # Output Sections Config + output_sections_config = self._make_section_config( + output_trace_config=output_trace_config, + data_sections_config=data_sections_config, + ) + if output_sections_config: + renderables.append(Text("")) + separator = self._make_separator(output_trace_config=output_trace_config) + if separator: + renderables.append(separator) + renderables.append(output_sections_config) + + # Output Sections Info + output_sections_info = self._make_section_info( + output_trace_config=output_trace_config, + data_sections_info=data_sections_info, + ) + if output_sections_info: + renderables.append(Text("")) + separator = self._make_separator(output_trace_config=output_trace_config) + if separator: + renderables.append(separator) + renderables.append(output_sections_info) + + # Output Sections Client API + output_sections_client_api, results_success_details = ( + self._make_section_external_api( + output_trace_config=output_trace_config, + data_sections_external_api=data_sections_external_api, + ) + ) + if output_sections_client_api: + renderables.append(Text("")) + separator = self._make_separator(output_trace_config=output_trace_config) + if separator: + renderables.append(separator) + renderables.append(output_sections_client_api) + + if results_success_details: + # Output Tables + output_tables = self._make_tables( + output_trace_config=output_trace_config, + auto_create_assets=auto_create_assets, + data_tables=results_success_details, + ) + for output_table in output_tables: + if output_table: + renderables.append(Text("")) + separator = self._make_separator( + output_trace_config=output_trace_config + ) + if separator: + renderables.append(separator) + renderables.append(output_table) + + # Output JSON + output_json = self._make_json( + data=data_sections_external_api, + output_trace_config=output_trace_config, + ) + + if output_json: + renderables.append(Text("")) + separator = self._make_separator(output_trace_config=output_trace_config) + if separator: + renderables.append(separator) + renderables.append(output_json) + + group = Group(*renderables) + console = Console( + color_system=None, + force_terminal=False, + width=150, + ) + with console.capture() as capture: + console.print(group) + output_str = capture.get() + return output_str diff --git a/shodan/tests/__init__.py b/shodan/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/tests/conftest.py b/shodan/tests/conftest.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/tests/shodan_contracts/cloud_provider_asset_discovery.py b/shodan/tests/shodan_contracts/cloud_provider_asset_discovery.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/tests/shodan_contracts/critical_ports_and_exposed_admin_interface.py b/shodan/tests/shodan_contracts/critical_ports_and_exposed_admin_interface.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/tests/shodan_contracts/custom_query.py b/shodan/tests/shodan_contracts/custom_query.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/tests/shodan_contracts/cve_enumeration.py b/shodan/tests/shodan_contracts/cve_enumeration.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/tests/shodan_contracts/cve_specific_watchlist.py b/shodan/tests/shodan_contracts/cve_specific_watchlist.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/tests/shodan_contracts/domain_discovery.py b/shodan/tests/shodan_contracts/domain_discovery.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/tests/shodan_contracts/host_enumeration.py b/shodan/tests/shodan_contracts/host_enumeration.py new file mode 100644 index 00000000..e69de29b diff --git a/shodan/tests/test-requirements.txt b/shodan/tests/test-requirements.txt new file mode 100644 index 00000000..e69de29b