Switch re to re2 (#226)

* Switch re to re2

* Fixes for re2 commit

* Add missing stubs

* Clean out old view, fix attacker image

* Fix typing/security issues and update deps

.gitignore

@@ -5,6 +5,7 @@
 .pytest_cache
 .vscode
 __pycache__
+aica-manager/
 coverage.xml
 db.sqlite3
 venv

Makefile

@@ -6,7 +6,6 @@ CONDA := conda run --no-capture-output -n aica
 check-env:
 ifndef MODE
 		$(error MODE is undefined)
-
 endif
 
 init: environment.yml
@@ -36,10 +35,13 @@ test: lint security
 				/opt/venv/bin/coverage run --omit='*test*' manage.py test --noinput && \
 				/opt/venv/bin/coverage report --fail-under=30"
 
-start: build
+start: check-env 
 		@docker compose -f docker-compose.yml -f docker-compose-${MODE}.yml up --wait -d
 
 stop: check-env
+		@docker compose -f docker-compose.yml -f docker-compose-${MODE}.yml down
+
+stop_purge: check-env
 		@docker compose -f docker-compose.yml -f docker-compose-${MODE}.yml down -v
 
 rebuild: build stop start
@@ -48,7 +50,7 @@ restart: stop start
 
 web_attack: check-env
 		@docker compose -f docker-compose.yml -f docker-compose-emu.yml exec target /bin/bash -c "ipset add allowlist attacker"
-		@docker compose -f docker-compose.yml -f docker-compose-emu.yml exec attacker /bin/bash -c "source attacker/bin/activate && python -m unittest discover -s /root/tests -p 'test_*.py'"
+		@docker compose -f docker-compose.yml -f docker-compose-emu.yml exec attacker /bin/bash -c "source ./attacker/bin/activate && python -m unittest discover -s ./tests/ -p 'test_*.py'"
 		@docker compose -f docker-compose.yml -f docker-compose-emu.yml exec target /bin/bash -c "ipset del allowlist attacker"
 
 logs: check-env

attacker/Dockerfile

@@ -1,28 +1,34 @@
-FROM alpine:3.19
+FROM kalilinux/kali-rolling:latest 
 
-COPY requirements.txt .
-RUN apk update && apk upgrade && \
-    apk add \
-        nmap \
-        nmap-scripts \
-        bash \
-        openssh \
+USER root
+
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y --no-install-recommends \
+        build-essential \
         chromium \
-        chromium-chromedriver \
-        py3-pip \
-        build-base \
+        chromium-driver \
+        kali-linux-headless \
+        libffi-dev \
+        openssh-server \
         python3-dev \
-        libffi-dev
-
-RUN python3 -m venv attacker && \
-    source attacker/bin/activate && \
-    pip install -Ur requirements.txt
+        python3-pip \
+        python3-selenium \
+        python3-venv
 
 RUN ssh-keygen -A
 
-RUN adduser -D ssh_user
+COPY docker_entrypoint.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/docker_entrypoint.sh
 
-COPY tests/ /root/tests
+RUN useradd kali
 
-COPY docker_entrypoint.sh /usr/local/bin/
-RUN chmod +x /usr/local/bin/docker_entrypoint.sh
+USER kali
+
+WORKDIR /home/kali
+
+COPY requirements.txt .
+RUN python3 -m venv attacker && \
+    ./attacker/bin/python -m pip install -Ur requirements.txt
+
+COPY tests/ ./tests

attacker/docker_entrypoint.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-echo "ssh_user:attackersshpassword" | chpasswd
+echo "kali:attackersshpassword" | chpasswd
 sed -i '/AllowTcpForwarding/d' /etc/ssh/sshd_config
 
 /usr/sbin/sshd -Def /etc/ssh/sshd_config

attacker/requirements.txt

@@ -1,4 +1,4 @@
 PyJWT==2.8.0
-python-dotenv==1.0.0
+python-dotenv==1.0.1
 requests==2.31.0
-selenium==4.16.0
+selenium==4.18.1

docker-compose.yml

@@ -76,9 +76,8 @@ services:
     container_name: manager_graphdb
     restart: unless-stopped
     environment:
-      - NEO4J_PLUGINS=\[\"apoc\"\]
       # https://neo4j.com/docs/operations-manual/current/docker/ref-settings/
-      - NEO4J_dbms_security_procedures_unrestricted=apoc.*
+      - NEO4J_dbms_security_procedures_unrestricted=apoc.*,gds.*
       - apoc.export.file.enabled=true
       - apoc.import.file.enabled=true
       - apoc.import.file.use_neo4j_config=false

environment.yml

@@ -4,15 +4,21 @@ channels:
   - conda-forge
 dependencies:
   - python=3.11
-  - black=23.12.*
   - mypy=1.8.*
-  - safety=2.3.*
-  - yamllint=1.33.*
-  - pip=23.3.*
+  - yamllint=1.35.*
+  - pip=24.*
+  # Some things are down here for better cross-platform compatibility
   - pip:
-      - bandit==1.7.6
+      - bandit==1.7.7
       - bashlint==0.1.1
-      - celery-types==0.20.0
+      - black==24.2.0
+      - celery-types==0.22.0
+      - charset-normalizer==3.3.2
       - django-stubs==4.2.7
-      - types-dateparser==1.1.4.10
-      - types-paramiko==3.4.0.20240103
+      - pandas-stubs==2.2.0.240218
+      - safety==3.0.1
+      - types-requests==2.31.0.20240218
+      - types-dateparser==1.1.4.20240106
+      - types-openpyxl==3.1.0.20240220
+      - types-paramiko==3.4.0.20240103
+      - urllib3==2.2.1

manager/Dockerfile

@@ -37,6 +37,7 @@ RUN --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \
         g++ \
         musl-dev \
         tcpdump \
+        tshark \
         net-tools \
         vim
 

manager/aica_django/connectors/Antivirus.py

@@ -10,7 +10,7 @@
 import ipaddress
 import json
 import logging
-import re
+import re2 as re  # type: ignore
 import requests
 import time
 

manager/aica_django/connectors/CaddyServer.py

@@ -7,7 +7,7 @@
 
 import datetime
 import logging
-import re
+import re2 as re  # type: ignore
 import requests
 import time
 import json

manager/aica_django/connectors/HTTPServer.py

@@ -7,7 +7,7 @@
 
 import datetime
 import logging
-import re
+import re2 as re  # type: ignore
 import requests
 import time
 

manager/aica_django/connectors/IntrusionDetection.py

@@ -8,7 +8,7 @@
 import datetime
 import json
 import logging
-import re
+import re2 as re  # type: ignore
 import requests
 import time
 

manager/aica_django/connectors/NetworkScan.py

@@ -12,7 +12,7 @@
 import netifaces  # type: ignore
 import nmap3  # type: ignore
 import os
-import re
+import re2 as re  # type: ignore
 import time
 
 from hashlib import sha256

manager/aica_django/connectors/SSH.py

@@ -8,7 +8,7 @@
 import ipaddress
 import os
 import socket
-from typing import Tuple, Union
+from typing import Tuple
 
 from paramiko import SSHClient, AutoAddPolicy
 from paramiko.ssh_exception import NoValidConnectionsError
@@ -32,7 +32,7 @@ def send_ssh_command(target: str, command: str) -> Tuple[int, str, str]:
     """
     client = SSHClient()
     client.load_system_host_keys()
-    client.set_missing_host_key_policy(AutoAddPolicy())
+    client.set_missing_host_key_policy(AutoAddPolicy())  # nosec
 
     client.connect(target, username="root")
     logger.debug(f"Sending command: {command}")

manager/aica_django/connectors/WAF.py

@@ -7,7 +7,7 @@
 
 import datetime
 import logging
-import re
+import re2 as re  # type: ignore
 import requests
 import time
 import json

manager/aica_django/microagents/offline_loader.py

@@ -30,6 +30,7 @@
 from celery.signals import worker_ready
 from celery.utils.log import get_task_logger
 from io import StringIO
+from numpy import ndarray
 from py2neo import ConnectionUnavailable  # type: ignore
 from stix2 import AttackPattern, Note, Software  # type: ignore
 from typing import Any, Dict
@@ -118,10 +119,12 @@ def create_ports() -> bool:
         comment="#",
         header=None,
         names=["service", "port", "frequency", "comment"],
+        index_col=False,
     )
     nmap_df[["port_number", "protocol"]] = nmap_df["port"].str.split("/", expand=True)
     nmap_df.drop(columns=["comment", "port"], axis=1, inplace=True)
     nmap_df.sort_values(by="frequency", inplace=True, ascending=False)
+    nmap_df = nmap_df.reset_index(drop=True)
 
     port_objects = []
 
@@ -130,6 +133,15 @@ def create_ports() -> bool:
 
     for index, row in nmap_df.iterrows():
         rank = nmap_df.index.get_loc(key=index)
+        if isinstance(rank, int):
+            rank = int(rank)
+        elif isinstance(rank, slice):
+            rank = int(rank.start)
+        elif isinstance(rank, ndarray):
+            rank = int(rank[0])
+        else:
+            raise ValueError
+
         port_object = Note(
             abstract=f"{row['port_number']}/{row['protocol']}",
             content=json.dumps(

manager/aica_django/urls.py

@@ -13,6 +13,7 @@
     1. Import the include() function: from django.urls import include, path
     2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
+
 from django.urls import include, re_path
 from django.contrib import admin