Update README (#9)

* Add black tool config

* Add missing backend env files

* Updated with development docs

* Lint

.flake8

@@ -3,7 +3,8 @@
 max-line-length = 180
 #select = B,C,E,F,W,T4,B9
 #ignore = E203, E266, E501, W503, F403, F401
-ignore = W503, E231, W605
+ignore = E402, W503, E231, W605
+    # E402, # module level import not at top of file (using isort)
     # W503, # line break before binary operator
     # E231, # missing whitespace after ',' (caused by black style)
     # W605, # invalid escape sequence (caused by regex)

.pre-commit-config.yaml

@@ -22,18 +22,18 @@ repos:
             name: Sort python imports (fixes files)
 
     - repo: https://github.com/psf/black
-      rev: 23.3.0
+      rev: 23.7.0
       hooks:
           - id: black
             language_version: python3.11
 
     - repo: https://github.com/asottile/setup-cfg-fmt
-      rev: v2.3.0
+      rev: v2.4.0
       hooks:
           - id: setup-cfg-fmt
 
     - repo: https://github.com/asottile/add-trailing-comma
-      rev: v2.4.0
+      rev: v3.0.0
       hooks:
           - id: add-trailing-comma
 

README.md

@@ -1,3 +1,53 @@
 # CoPilot
 
 SOCFortress CoPilot
+
+# Development
+
+## Local development of backend
+
+Setup the env vars, adjust if required.
+
+```
+cd backend
+cp .env.example .env
+```
+
+Create and activate python, installing dependencies
+
+```
+python3.11 -m venv.venv --copies
+source .venv/bin/activate
+pip install -U pip setuptools wheel
+pip install -r requirements.in
+```
+
+Create a DB and apply any pending DB migrations
+
+```
+FLASK_APP=copilot.py flask db upgrade
+```
+
+Start local dev server
+
+```
+python3 app.py
+```
+
+If there any changes made to the model run the migrate command (example commment)
+and if any changes were detected, update your local DB instance.
+
+```
+FLASK_APP=copilot.py flask db migrate -m "Add User model."
+FLASK_APP=copilot.py flask db upgrade
+```
+
+See https://flask-migrate.readthedocs.io/en/latest/ for further information
+
+# Deployment
+
+## Production Deployment Notes
+
+```
+pip install -r requirements.txt
+```

backend/.env.example

@@ -0,0 +1,5 @@
+DEBUG=True
+SECRET_KEY="not so secret"
+ENV=development
+SQLALCHEMY_TRACK_MODIFICATIONS=True
+UPLOAD_FOLDER="/tmp/copilot_uploads"

backend/.env.prod.example

@@ -0,0 +1,6 @@
+DEBUG=False
+SECRET_KEY="not so secret"
+ENV=production
+SQLALCHEMY_DATABASE_URI="postgresql://postgres:root@localhost:5432/copilot"
+SQLALCHEMY_TRACK_MODIFICATIONS=False
+UPLOAD_FOLDER="/opt/socfortress/copilot/uploads"

backend/app/__init__.py

@@ -5,8 +5,6 @@
 from flask_sqlalchemy import SQLAlchemy
 from flask_swagger_ui import get_swaggerui_blueprint
 
-# from app.routes import bp  # Import the blueprint
-
 app = Flask(__name__)
 
 SWAGGER_URL = "/api/docs"  # URL for exposing Swagger UI (without trailing '/')
@@ -27,25 +25,23 @@
 )
 
 app.register_blueprint(swaggerui_blueprint)
-
 CORS(app)
-
-
 app.config.from_object("settings")
 
 db = SQLAlchemy(app)
 migrate = Migrate(app, db)
 ma = Marshmallow(app)
 
-from app.routes.agents import bp as agents_bp  # Import the blueprint
-from app.routes.alerts import bp as alerts_bp  # Import the blueprint
-from app.routes.connectors import bp as connectors_bp  # Import the blueprint
-from app.routes.dfir_iris import bp as dfir_iris_bp  # Import the blueprint
-from app.routes.graylog import bp as graylog_bp  # Import the blueprint
-from app.routes.rules import bp as rules_bp  # Import the blueprint
-from app.routes.shuffle import bp as shuffle_bp  # Import the blueprint
-from app.routes.velociraptor import bp as velociraptor_bp  # Import the blueprint
-from app.routes.wazuhindexer import bp as wazuhindexer_bp  # Import the blueprint
+
+from app.routes.agents import bp as agents_bp
+from app.routes.alerts import bp as alerts_bp
+from app.routes.connectors import bp as connectors_bp
+from app.routes.dfir_iris import bp as dfir_iris_bp
+from app.routes.graylog import bp as graylog_bp
+from app.routes.rules import bp as rules_bp
+from app.routes.shuffle import bp as shuffle_bp
+from app.routes.velociraptor import bp as velociraptor_bp
+from app.routes.wazuhindexer import bp as wazuhindexer_bp
 
 app.register_blueprint(connectors_bp)  # Register the connectors blueprint
 app.register_blueprint(agents_bp)  # Register the agents blueprint

backend/app/models/connectors.py

@@ -66,17 +66,9 @@ def get_connector_info_from_db(connector_name: str) -> Dict[str, Any]:
         Raises:
             NoResultFound: If the connector_name is not found in the database.
         """
-        connector = (
-            current_app.extensions["sqlalchemy"]
-            .db.session.query(Connectors)
-            .filter_by(connector_name=connector_name)
-            .first()
-        )
+        connector = current_app.extensions["sqlalchemy"].db.session.query(Connectors).filter_by(connector_name=connector_name).first()
         if connector:
-            attributes = {
-                col.name: getattr(connector, col.name)
-                for col in Connectors.__table__.columns
-            }
+            attributes = {col.name: getattr(connector, col.name) for col in Connectors.__table__.columns}
             return attributes
         else:
             raise NoResultFound

backend/app/models/graylog.py

@@ -88,9 +88,5 @@ class Meta:
         )
 
 
-graylog_metrics_allocation_schema: GraylogMetricsAllocationSchema = (
-    GraylogMetricsAllocationSchema()
-)
-graylog_metrics_allocations_schema: GraylogMetricsAllocationSchema = (
-    GraylogMetricsAllocationSchema(many=True)
-)
+graylog_metrics_allocation_schema: GraylogMetricsAllocationSchema = GraylogMetricsAllocationSchema()
+graylog_metrics_allocations_schema: GraylogMetricsAllocationSchema = GraylogMetricsAllocationSchema(many=True)

backend/app/models/models.py

@@ -128,11 +128,7 @@ def __init__(
         self.connector_username = connector_username
         self.connector_password = connector_password
 
-        if (
-            connector_name.lower() == "shuffle"
-            or connector_name.lower() == "dfir-irs"
-            or connector_name.lower() == "velociraptor"
-        ):
+        if connector_name.lower() == "shuffle" or connector_name.lower() == "dfir-irs" or connector_name.lower() == "velociraptor":
             logger.info(f"Setting the API key for {connector_name}")
             self.connector_api_key = connector_api_key
         else:

backend/app/models/wazuh_indexer.py

@@ -87,9 +87,5 @@ class Meta:
         )
 
 
-wazuh_indexer_allocation_schema: WazuhIndexerAllocationSchema = (
-    WazuhIndexerAllocationSchema()
-)
-wazuh_indexer_allocations_schema: WazuhIndexerAllocationSchema = (
-    WazuhIndexerAllocationSchema(many=True)
-)
+wazuh_indexer_allocation_schema: WazuhIndexerAllocationSchema = WazuhIndexerAllocationSchema()
+wazuh_indexer_allocations_schema: WazuhIndexerAllocationSchema = WazuhIndexerAllocationSchema(many=True)

backend/app/routes/connectors.py

@@ -20,6 +20,7 @@ def list_connectors_available():
     Returns:
         json: A JSON response containing the list of all available connectors along with their connection verification status.
     """
+    logger.info("Received request to get all available connectors")
     connectors_service = ConnectorService(db)
     connectors = ConnectorsAvailable.query.all()
     result = connectors_available_schema.dump(connectors)
@@ -44,6 +45,7 @@ def get_connector_details(id: str):
     Returns:
         json: A JSON response containing the details of the connector.
     """
+    logger.info("Received request to get a connector details")
     service = ConnectorService(db)
     connector = service.validate_connector_exists(int(id))
 
@@ -67,6 +69,7 @@ def update_connector_route(id: str):
         json: A JSON response containing the success status of the update operation and a message indicating the status.
         If the update operation was successful, it returns the connector name and the status of the connection verification.
     """
+    logger.info("Received request to update connector")
     api_key_connector = ["Shuffle", "DFIR-IRIS", "Velociraptor"]
 
     request_data = request.get_json()

backend/app/routes/graylog.py

@@ -1,14 +1,12 @@
 from flask import Blueprint
 from flask import jsonify
+from loguru import logger
 
 from app.services.Graylog.index import IndexService
 from app.services.Graylog.inputs import InputsService
 from app.services.Graylog.messages import MessagesService
 from app.services.Graylog.metrics import MetricsService
 
-# from loguru import logger
-
-
 bp = Blueprint("graylog", __name__)
 
 
@@ -20,6 +18,7 @@ def get_messages() -> dict:
     Returns:
         dict: A JSON object containing the list of all the messages.
     """
+    logger.info("Received request to get graylog messages")
     service = MessagesService()
     messages = service.collect_messages()
     return messages
@@ -34,6 +33,7 @@ def get_metrics() -> dict:
         dict: A JSON object containing the list of all metrics,
               including the uncommitted journal size.
     """
+    logger.info("Received request to get graylog metrics")
     service = MetricsService()
     uncommitted_journal_size = service.collect_uncommitted_journal_size()
     metrics = service.collect_throughput_metrics()
@@ -50,6 +50,7 @@ def get_indices() -> dict:
     Returns:
         dict: A JSON object containing the list of all indices.
     """
+    logger.info("Received request to get graylog indexes")
     service = IndexService()
     indices = service.collect_indices()
     return indices
@@ -66,6 +67,7 @@ def delete_index(index_name: str) -> dict:
     Returns:
         dict: A JSON object containing the result of the deletion operation.
     """
+    logger.info("Received request to delete index")
     service = IndexService()
     result = service.delete_index(index_name)
     return result
@@ -79,6 +81,7 @@ def get_inputs() -> dict:
     Returns:
         dict: A JSON object containing the list of all running and configured inputs.
     """
+    logger.info("Received request to get graylog inputs")
     service = InputsService()
     running_inputs = service.collect_running_inputs()
     configured_inputs = service.collect_configured_inputs()

backend/app/routes/shuffle.py

@@ -1,16 +1,9 @@
 from flask import Blueprint
 from flask import jsonify
+from loguru import logger
 
-# from app.models.connectors import Connector
-# from app.models.connectors import WazuhManagerConnector
-# from app.services.agents.agents import AgentService
-# from app.services.agents.agents import AgentSyncService
 from app.services.Shuffle.workflows import WorkflowsService
 
-# from flask import request
-# from loguru import logger
-
-
 bp = Blueprint("shuffle", __name__)
 
 
@@ -22,6 +15,7 @@ def get_workflows() -> jsonify:
     Returns:
         jsonify: A JSON response containing the list of all configured Workflows in Shuffle.
     """
+    logger.info("Received request to get all Shuffle workflows")
     service = WorkflowsService()
     workflows = service.collect_workflows()
     return workflows
@@ -37,6 +31,7 @@ def get_workflows_executions() -> jsonify:
     Returns:
         jsonify: A JSON response containing the list of all configured workflows and their last execution status.
     """
+    logger.info("Received request to get all Shuffle workflow execution status")
     service = WorkflowsService()
     workflow_details = service.collect_workflow_details()
     if "workflows" not in workflow_details:
@@ -62,6 +57,7 @@ def get_workflow_executions(workflow_id: str) -> jsonify:
     Returns:
         jsonify: A JSON response containing the last execution status of the specified workflow.
     """
+    logger.info("Received request to get a Shuffle workflow")
     service = WorkflowsService()
     workflow_details = service.collect_workflow_executions_status(workflow_id)
     return workflow_details

backend/app/routes/velociraptor.py

@@ -84,9 +84,7 @@ def collect_artifact():
     artifact_name = req_data["artifact_name"]
     client_name = req_data["client_name"]
     service = UniversalService()
-    client_id = service.get_client_id(client_name=client_name)["results"][0][
-        "client_id"
-    ]
+    client_id = service.get_client_id(client_name=client_name)["results"][0]["client_id"]
     if client_id is None:
         return (
             jsonify(

backend/app/services/DFIR_IRIS/universal.py

@@ -90,9 +90,7 @@ def create_session(self) -> Dict[str, Union[bool, Optional[ClientSession], str]]
                 "message": "Connection to DFIR-IRIS unsuccessful.",
             }
 
-    def fetch_and_parse_data(
-        self, session: ClientSession, action: Callable, *args
-    ) -> Dict[str, Union[bool, Optional[Dict]]]:
+    def fetch_and_parse_data(self, session: ClientSession, action: Callable, *args) -> Dict[str, Union[bool, Optional[Dict]]]:
         """
         Fetches and parses data from DFIR-IRIS using a specified action.
 

backend/app/services/Graylog/index.py

@@ -33,11 +33,7 @@ def collect_indices(self) -> Dict[str, Union[bool, str, Dict]]:
         Returns:
             dict: A dictionary containing the success status, a message, and potentially a dictionary with indices.
         """
-        if (
-            self.connector_url is None
-            or self.connector_username is None
-            or self.connector_password is None
-        ):
+        if self.connector_url is None or self.connector_username is None or self.connector_password is None:
             return {"message": "Failed to collect Graylog details", "success": False}
 
         managed_indices = self._collect_managed_indices()
@@ -95,11 +91,7 @@ def delete_index(self, index_name: str) -> Dict[str, Union[bool, str]]:
             dict: A dictionary containing the response.
         """
         logger.info(f"Deleting index {index_name} from Graylog")
-        if (
-            self.connector_url is None
-            or self.connector_username is None
-            or self.connector_password is None
-        ):
+        if self.connector_url is None or self.connector_username is None or self.connector_password is None:
             return {"message": "Failed to collect Graylog details", "success": False}
 
         # Check if the index exists in Graylog
@@ -143,7 +135,6 @@ def _delete_index(self, index_name: str) -> Dict[str, Union[bool, str]]:
         except Exception as e:
             logger.error(f"Failed to delete index {index_name} from Graylog: {e}")
             return {
-                "message": f"Failed to delete index {index_name} from Graylog. If this is the current index, "
-                "it cannot be deleted.",
+                "message": f"Failed to delete index {index_name} from Graylog. If this is the current index, " "it cannot be deleted.",
                 "success": False,
             }