From bddc5d86443f8d0f48cc73300cccabf838f43342 Mon Sep 17 00:00:00 2001 From: Stefan Tatschner Date: Mon, 9 Dec 2024 08:56:50 +0100 Subject: [PATCH 1/8] chore: Remove comment --- src/gallia/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/gallia/utils.py b/src/gallia/utils.py index f7a9fe4ab..4f29687e8 100644 --- a/src/gallia/utils.py +++ b/src/gallia/utils.py @@ -371,7 +371,6 @@ def net_if_broadcast_addrs() -> list[AddrInfo]: continue for addr in iface.addr_info: - # We only work with broadcastable IPv4. if not addr.is_v4() or addr.broadcast is None: continue out.append(addr) From 9782983820040e220270aba05ab77658bcfa4ea2 Mon Sep 17 00:00:00 2001 From: Stefan Tatschner Date: Mon, 9 Dec 2024 09:01:13 +0100 Subject: [PATCH 2/8] refactor: Move networking functions to gallia.net --- src/gallia/commands/discover/doip.py | 2 +- src/gallia/net.py | 81 ++++++++++++++++++++++++++++ src/gallia/utils.py | 71 ------------------------ tests/pytest/test_net_if.py | 2 +- 4 files changed, 83 insertions(+), 73 deletions(-) create mode 100644 src/gallia/net.py diff --git a/src/gallia/commands/discover/doip.py b/src/gallia/commands/discover/doip.py index 6409b2c2f..965ab4786 100644 --- a/src/gallia/commands/discover/doip.py +++ b/src/gallia/commands/discover/doip.py @@ -14,6 +14,7 @@ from gallia.command.base import AsyncScriptConfig from gallia.command.config import AutoInt, Field from gallia.log import get_logger +from gallia.net import net_if_broadcast_addrs from gallia.services.uds.core.service import TesterPresentRequest, TesterPresentResponse from gallia.transports.doip import ( DiagnosticMessageNegativeAckCodes, @@ -29,7 +30,6 @@ TimingAndCommunicationParameters, VehicleAnnouncementMessage, ) -from gallia.utils import net_if_broadcast_addrs logger = get_logger(__name__) diff --git a/src/gallia/net.py b/src/gallia/net.py new file mode 100644 index 000000000..d1ec81985 --- /dev/null +++ b/src/gallia/net.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: AISEC Pentesting Team +# +# SPDX-License-Identifier: Apache-2.0 + +import json +import subprocess +import sys + +import pydantic +from pydantic.networks import IPvAnyAddress + +from gallia.log import get_logger + +logger = get_logger(__name__) + + +class AddrInfo(pydantic.BaseModel): + family: str + local: IPvAnyAddress + prefixlen: int + broadcast: IPvAnyAddress | None = None + scope: str + label: str | None = None + valid_life_time: int + preferred_life_time: int + + def is_v4(self) -> bool: + return self.family == "inet" + + +class Interface(pydantic.BaseModel): + ifindex: int + ifname: str + flags: list[str] + mtu: int + qdisc: str + operstate: str + group: str + link_type: str + address: str | None = None + broadcast: str | None = None + addr_info: list[AddrInfo] + + def is_up(self) -> bool: + return self.operstate == "UP" + + def can_broadcast(self) -> bool: + return "BROADCAST" in self.flags + + +def net_if_addrs() -> list[Interface]: + if sys.platform != "linux": + raise NotImplementedError("net_if_addrs() is only supported on Linux platforms") + + try: + p = subprocess.run(["ip", "-j", "address", "show"], capture_output=True, check=True) + except FileNotFoundError as e: + logger.warning(f"Could not query information about interfaces: {e}") + return [] + + try: + return [Interface(**item) for item in json.loads(p.stdout.decode())] + except pydantic.ValidationError as e: + logger.error("BUG: A special case for `ip -j address show` is not handled!") + logger.error("Please report a bug including the following json string.") + logger.error("https://github.com/Fraunhofer-AISEC/gallia/issues") + logger.error(e.json()) + raise + + +def net_if_broadcast_addrs() -> list[AddrInfo]: + out = [] + for iface in net_if_addrs(): + if not (iface.is_up() and iface.can_broadcast()): + continue + + for addr in iface.addr_info: + if not addr.is_v4() or addr.broadcast is None: + continue + out.append(addr) + return out diff --git a/src/gallia/utils.py b/src/gallia/utils.py index 4f29687e8..404878eac 100644 --- a/src/gallia/utils.py +++ b/src/gallia/utils.py @@ -8,10 +8,8 @@ import contextvars import importlib.util import ipaddress -import json import logging import re -import subprocess import sys from collections.abc import Awaitable, Callable from pathlib import Path @@ -20,8 +18,6 @@ from urllib.parse import urlparse import aiofiles -import pydantic -from pydantic.networks import IPvAnyAddress from gallia.log import Loglevel, get_logger @@ -308,70 +304,3 @@ def handle_task_error(fut: asyncio.Future[Any]) -> None: # Info level is enough, since our aim is only to consume the stack trace logger.info(f"{task_name} ended with error: {e!r}") - - -class AddrInfo(pydantic.BaseModel): - family: str - local: IPvAnyAddress - prefixlen: int - broadcast: IPvAnyAddress | None = None - scope: str - label: str | None = None - valid_life_time: int - preferred_life_time: int - - def is_v4(self) -> bool: - return self.family == "inet" - - -class Interface(pydantic.BaseModel): - ifindex: int - ifname: str - flags: list[str] - mtu: int - qdisc: str - operstate: str - group: str - link_type: str - address: str | None = None - broadcast: str | None = None - addr_info: list[AddrInfo] - - def is_up(self) -> bool: - return self.operstate == "UP" - - def can_broadcast(self) -> bool: - return "BROADCAST" in self.flags - - -def net_if_addrs() -> list[Interface]: - if sys.platform != "linux": - raise NotImplementedError("net_if_addrs() is only supported on Linux platforms") - - try: - p = subprocess.run(["ip", "-j", "address", "show"], capture_output=True, check=True) - except FileNotFoundError as e: - logger.warning(f"Could not query information about interfaces: {e}") - return [] - - try: - return [Interface(**item) for item in json.loads(p.stdout.decode())] - except pydantic.ValidationError as e: - logger.error("BUG: A special case for `ip -j address show` is not handled!") - logger.error("Please report a bug including the following json string.") - logger.error("https://github.com/Fraunhofer-AISEC/gallia/issues") - logger.error(e.json()) - raise - - -def net_if_broadcast_addrs() -> list[AddrInfo]: - out = [] - for iface in net_if_addrs(): - if not (iface.is_up() and iface.can_broadcast()): - continue - - for addr in iface.addr_info: - if not addr.is_v4() or addr.broadcast is None: - continue - out.append(addr) - return out diff --git a/tests/pytest/test_net_if.py b/tests/pytest/test_net_if.py index 68113eefd..a1a259f91 100644 --- a/tests/pytest/test_net_if.py +++ b/tests/pytest/test_net_if.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 -from gallia.utils import net_if_addrs +from gallia.net import net_if_addrs def test_net_if_addrs() -> None: From 84b9d1ca5da1107d7c76e660c0bca69f1f984967 Mon Sep 17 00:00:00 2001 From: Stefan Tatschner Date: Mon, 9 Dec 2024 09:46:43 +0100 Subject: [PATCH 3/8] refactor: Use pydantic functions for parsing json --- src/gallia/net.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gallia/net.py b/src/gallia/net.py index d1ec81985..3c2127870 100644 --- a/src/gallia/net.py +++ b/src/gallia/net.py @@ -2,7 +2,6 @@ # # SPDX-License-Identifier: Apache-2.0 -import json import subprocess import sys @@ -59,7 +58,8 @@ def net_if_addrs() -> list[Interface]: return [] try: - return [Interface(**item) for item in json.loads(p.stdout.decode())] + iface_list_type = pydantic.TypeAdapter(list[Interface]) + return iface_list_type.validate_json(p.stdout.decode()) except pydantic.ValidationError as e: logger.error("BUG: A special case for `ip -j address show` is not handled!") logger.error("Please report a bug including the following json string.") From 35e9e18f06fb1c862dfa93d74bca43b17bf5f9a4 Mon Sep 17 00:00:00 2001 From: Stefan Tatschner Date: Mon, 9 Dec 2024 10:11:21 +0100 Subject: [PATCH 4/8] feat: Add a decorator for platform support --- src/gallia/net.py | 6 ++---- src/gallia/utils.py | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/gallia/net.py b/src/gallia/net.py index 3c2127870..71655f440 100644 --- a/src/gallia/net.py +++ b/src/gallia/net.py @@ -3,12 +3,12 @@ # SPDX-License-Identifier: Apache-2.0 import subprocess -import sys import pydantic from pydantic.networks import IPvAnyAddress from gallia.log import get_logger +from gallia.utils import supports_platform logger = get_logger(__name__) @@ -47,10 +47,8 @@ def can_broadcast(self) -> bool: return "BROADCAST" in self.flags +@supports_platform("linux") def net_if_addrs() -> list[Interface]: - if sys.platform != "linux": - raise NotImplementedError("net_if_addrs() is only supported on Linux platforms") - try: p = subprocess.run(["ip", "-j", "address", "show"], capture_output=True, check=True) except FileNotFoundError as e: diff --git a/src/gallia/utils.py b/src/gallia/utils.py index 404878eac..8878e9b6a 100644 --- a/src/gallia/utils.py +++ b/src/gallia/utils.py @@ -12,6 +12,7 @@ import re import sys from collections.abc import Awaitable, Callable +from functools import wraps from pathlib import Path from types import ModuleType from typing import TYPE_CHECKING, Any @@ -304,3 +305,24 @@ def handle_task_error(fut: asyncio.Future[Any]) -> None: # Info level is enough, since our aim is only to consume the stack trace logger.info(f"{task_name} ended with error: {e!r}") + + +def supports_platform[T, **P](*platform: str) -> Callable[[Callable[P, T]], Callable[P, T]]: + def decorator(function: Callable[P, T]) -> Callable[P, T]: + @wraps(function) + def wrapper(*args: P.args, **kwargs: P.kwargs) -> T: + supported = False + for p in platform: + if sys.platform == p: + supported = True + break + if supported is False: + raise NotImplementedError( + f'`{function.__name__}()` is not supported on: "{sys.platform}"' + ) + + return function(*args, **kwargs) + + return wrapper + + return decorator From 0ec576d17711bf5a27058e7db5237c802d6653ee Mon Sep 17 00:00:00 2001 From: Stefan Tatschner Date: Mon, 9 Dec 2024 10:17:10 +0100 Subject: [PATCH 5/8] fix: Add missing decorator to net_if_broadcast_addrs() --- src/gallia/net.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/gallia/net.py b/src/gallia/net.py index 71655f440..86f379dac 100644 --- a/src/gallia/net.py +++ b/src/gallia/net.py @@ -66,6 +66,7 @@ def net_if_addrs() -> list[Interface]: raise +@supports_platform("linux") def net_if_broadcast_addrs() -> list[AddrInfo]: out = [] for iface in net_if_addrs(): From bb871958a28fc2f7307d94c58edc70b2d767551f Mon Sep 17 00:00:00 2001 From: Stefan Tatschner Date: Tue, 17 Dec 2024 16:35:21 +0100 Subject: [PATCH 6/8] chore: Move more functions to gallia.net --- src/gallia/dumpcap.py | 3 ++- src/gallia/net.py | 35 +++++++++++++++++++++++++++++++++++ src/gallia/transports/base.py | 2 +- src/gallia/utils.py | 35 ----------------------------------- tests/pytest/test_helpers.py | 2 +- 5 files changed, 39 insertions(+), 38 deletions(-) diff --git a/src/gallia/dumpcap.py b/src/gallia/dumpcap.py index 758acf822..7d033f737 100644 --- a/src/gallia/dumpcap.py +++ b/src/gallia/dumpcap.py @@ -17,8 +17,9 @@ from urllib.parse import urlparse from gallia.log import get_logger +from gallia.net import split_host_port from gallia.transports import TargetURI, TransportScheme -from gallia.utils import auto_int, handle_task_error, set_task_handler_ctx_variable, split_host_port +from gallia.utils import auto_int, handle_task_error, set_task_handler_ctx_variable logger = get_logger(__name__) diff --git a/src/gallia/net.py b/src/gallia/net.py index 86f379dac..756ca68bd 100644 --- a/src/gallia/net.py +++ b/src/gallia/net.py @@ -2,7 +2,9 @@ # # SPDX-License-Identifier: Apache-2.0 +import ipaddress import subprocess +from urllib.parse import urlparse import pydantic from pydantic.networks import IPvAnyAddress @@ -13,6 +15,39 @@ logger = get_logger(__name__) +def split_host_port( + hostport: str, + default_port: int | None = None, +) -> tuple[str, int | None]: + """Splits a combination of ip address/hostname + port into hostname/ip address + and port. The default_port argument can be used to return a port if it is + absent in the hostport argument.""" + # Special case: If hostport is an ipv6 then the urlparser does some weird + # things with the colons and tries to parse ports. Catch this case early. + host = "" + port = default_port + try: + # If hostport is a valid ip address (v4 or v6) there + # is no port included + host = str(ipaddress.ip_address(hostport)) + except ValueError: + pass + + # Only parse if hostport is not a valid ip address. + if host == "": + # urlparse() and urlsplit() insists on absolute URLs starting with "//". + url = urlparse(f"//{hostport}") + host = url.hostname if url.hostname else url.netloc + port = url.port if url.port else default_port + return host, port + + +def join_host_port(host: str, port: int) -> str: + if ":" in host: + return f"[{host}]:port" + return f"{host}:{port}" + + class AddrInfo(pydantic.BaseModel): family: str local: IPvAnyAddress diff --git a/src/gallia/transports/base.py b/src/gallia/transports/base.py index bc9672c8f..c11bd8c18 100644 --- a/src/gallia/transports/base.py +++ b/src/gallia/transports/base.py @@ -10,8 +10,8 @@ from urllib.parse import parse_qs, urlencode, urlparse, urlunparse from gallia.log import get_logger +from gallia.net import join_host_port from gallia.transports.schemes import TransportScheme -from gallia.utils import join_host_port logger = get_logger(__name__) diff --git a/src/gallia/utils.py b/src/gallia/utils.py index 8878e9b6a..357c34dbf 100644 --- a/src/gallia/utils.py +++ b/src/gallia/utils.py @@ -7,7 +7,6 @@ import asyncio import contextvars import importlib.util -import ipaddress import logging import re import sys @@ -16,7 +15,6 @@ from pathlib import Path from types import ModuleType from typing import TYPE_CHECKING, Any -from urllib.parse import urlparse import aiofiles @@ -45,39 +43,6 @@ def strtobool(val: str) -> bool: raise ValueError(f"invalid truth value {val!r}") -def split_host_port( - hostport: str, - default_port: int | None = None, -) -> tuple[str, int | None]: - """Splits a combination of ip address/hostname + port into hostname/ip address - and port. The default_port argument can be used to return a port if it is - absent in the hostport argument.""" - # Special case: If hostport is an ipv6 then the urlparser does some weird - # things with the colons and tries to parse ports. Catch this case early. - host = "" - port = default_port - try: - # If hostport is a valid ip address (v4 or v6) there - # is no port included - host = str(ipaddress.ip_address(hostport)) - except ValueError: - pass - - # Only parse if hostport is not a valid ip address. - if host == "": - # urlparse() and urlsplit() insists on absolute URLs starting with "//". - url = urlparse(f"//{hostport}") - host = url.hostname if url.hostname else url.netloc - port = url.port if url.port else default_port - return host, port - - -def join_host_port(host: str, port: int) -> str: - if ":" in host: - return f"[{host}]:port" - return f"{host}:{port}" - - def camel_to_snake(s: str) -> str: """Convert a CamelCase string to a snake_case string.""" # https://stackoverflow.com/a/1176023 diff --git a/tests/pytest/test_helpers.py b/tests/pytest/test_helpers.py index 57b762120..b90a6c6c0 100644 --- a/tests/pytest/test_helpers.py +++ b/tests/pytest/test_helpers.py @@ -5,11 +5,11 @@ import pytest from gallia.log import setup_logging +from gallia.net import split_host_port from gallia.services.uds.core.utils import ( address_and_size_length, uds_memory_parameters, ) -from gallia.utils import split_host_port setup_logging() From 8e3005235587dd70486f78f42e7d19fa4d933be2 Mon Sep 17 00:00:00 2001 From: Stefan Tatschner Date: Thu, 19 Dec 2024 14:28:21 +0100 Subject: [PATCH 7/8] chore: Remove codeql stuff This throws errors since one year. Remove it. --- .github/workflows/codeql-analysis.yml | 53 --------------------------- 1 file changed, 53 deletions(-) delete mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index 4a902839f..000000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,53 +0,0 @@ -# SPDX-FileCopyrightText: AISEC Pentesting Team -# -# SPDX-License-Identifier: CC0-1.0 - -name: "CodeQL" - -on: - push: - branches: [ "master", "1.0-maint" ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ "master", "1.0-maint" ] - schedule: - - cron: '37 17 * * 5' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'python' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support - python-version: ['3.11', '3.12'] - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v3 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - - # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 From 79921a39f268d7f45e13528c648008c0e48c3745 Mon Sep 17 00:00:00 2001 From: Stefan Tatschner Date: Thu, 19 Dec 2024 14:38:08 +0100 Subject: [PATCH 8/8] chore: Run bats tests for all supported python versions --- .github/workflows/tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ad84631fa..e1c45c5f4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -49,6 +49,8 @@ jobs: bats: strategy: fail-fast: false + matrix: + python-version: ['3.12', '3.13'] runs-on: ubuntu-latest container: debian:trixie @@ -64,6 +66,7 @@ jobs: - name: Set up Python ${{ matrix.python-version }} run: | uv python install ${{ matrix.python-version }} + uv python pin ${{ matrix.python-version }} - name: Install Dependencies run: |