Scheduler page (#201)

* add job description to scheduler

* Update example for dashboards in ProvisionNewCustomer class

* updated dependencies

* updated logs page link location

* Add logging statement to execute_integration function in integrations.py

* put office365 route

* added scheduler api/types

* updated login components

* added scheduler page

* updated scheduler api

* tmp

* tmp

* updated job type

* updated job card

* Update monitoring_alert.py and monitoring_alert.schema with response model changes

* Refactor monitoring_alert.py and monitoring_alert.schema with response model changes

* Refactor monitoring_alert.py and monitoring_alert.schema with response model changes

* updated scheduler page icon

* refactor props

* Add delete_monitoring_alert endpoint to monitoring_alert.py

* Update branch name in Docker workflow from 'scheduler-page' to 'main'

* added job actions component

* Refactor provision_content_pack function to accept ProvisionContentPackRequest in graylog/routes/provision.py

Added fortinet content pack templates

* Update docker-compose.yml to version v0.0.8

* add network connectors db things

* added network connectors population to db

* network connectors routes

* Update fortinet.md

* Update fortinet.md

* Update fortinet.md

* Update fortinet.md

* Update fortinet.md

* Create opnsense.md

* Update opnsense.md

* Update opnsense.md

* Fix error handling in provision.py

* Fix error handling in provision.py

* Add route to remove a user from a customer in dfir_iris/routes/users.py

* Fix error handling in provision.py

* Fix error handling in provision.py

* Refactor customer network connector processing in routes.py

* fortinet provisioning initial setup

* just about all fortinet provision stuff...but need to still assign newly created stream to the pipeline

* updated scheduler api

* added next run time component

* updated dependencies

* updated job actions component

* added job form

* precommit fixes

---------

Co-authored-by: Davide Di Modica <webmaster.ddm@gmail.com>
Co-authored-by: juan-socfortress <111928961+juan-socfortress@users.noreply.github.com>

README.md

@@ -84,15 +84,14 @@ systemctl restart docker
 
 ```bash
 #  Clone the CoPilot repository
-wget https://raw.githubusercontent.com/socfortress/CoPilot/v0.0.7/docker-compose.yml
+wget https://raw.githubusercontent.com/socfortress/CoPilot/v0.0.8/docker-compose.yml
 
 # Edit the docker-compose.yml file to set the server name and/or the services you want to use
 
 # Create the path for storing your data
 mkdir data
 
-# Copy .env.example to .env
-cp .env.example .env
+# Create the .env file based on the .env.example
 
 # Run Copilot
 docker compose up -d

backend/alembic/alembic.ini

@@ -60,7 +60,7 @@ version_path_separator = os  # Use os.pathsep. Default configuration used for ne
 # are written from script.py.mako
 # output_encoding = utf-8
 
-sqlalchemy.url = mysql+pymysql://copilot:REPLACE_WITH_PASS@copilot-mysql/copilot
+sqlalchemy.url = mysql+pymysql://copilot:REPLACE_ME@copilot-mysql/copilot
 
 
 [post_write_hooks]

backend/alembic/env.py

@@ -26,6 +26,19 @@
 )
 from app.integrations.models.customer_integration_settings import CustomerIntegrations
 from app.integrations.monitoring_alert.models.monitoring_alert import MonitoringAlerts
+from app.network_connectors.models.network_connectors import AvailableNetworkConnectors
+from app.network_connectors.models.network_connectors import (
+    AvailableNetworkConnectorsKeys,
+)
+from app.network_connectors.models.network_connectors import CustomerNetworkConnectors
+from app.network_connectors.models.network_connectors import (
+    CustomerNetworkConnectorsMeta,
+)
+from app.network_connectors.models.network_connectors import NetworkConnectorsConfig
+from app.network_connectors.models.network_connectors import NetworkConnectorsService
+from app.network_connectors.models.network_connectors import (
+    NetworkConnectorsSubscription,
+)
 from app.schedulers.models.scheduler import JobMetadata
 
 # this is the Alembic Config object, which provides

backend/alembic/versions/74a095d63af4_add_network_connectors_tables.py

@@ -0,0 +1,123 @@
+"""Add Network Connectors Tables
+
+Revision ID: 74a095d63af4
+Revises: c3ad5012f4db
+Create Date: 2024-04-25 13:03:22.718120
+
+"""
+from typing import Sequence
+from typing import Union
+
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "74a095d63af4"
+down_revision: Union[str, None] = "c3ad5012f4db"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "available_network_connectors",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("network_connector_name", sa.String(length=255), nullable=False),
+        sa.Column("description", sa.String(length=1024), nullable=False),
+        sa.Column("network_connector_details", mysql.TEXT(length=1000000), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "customer_network_connectors",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("customer_code", sa.String(length=50), nullable=False),
+        sa.Column("customer_name", sa.String(length=255), nullable=False),
+        sa.Column("network_connector_service_id", sa.Integer(), nullable=False),
+        sa.Column("network_connector_service_name", sa.String(length=255), nullable=False),
+        sa.Column("deployed", sa.Boolean(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "customer_network_connectors_meta",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("customer_code", sa.String(length=50), nullable=False),
+        sa.Column("network_connector_name", sa.String(length=255), nullable=False),
+        sa.Column("graylog_input_id", sa.String(length=1024), nullable=True),
+        sa.Column("graylog_index_id", sa.String(length=1024), nullable=False),
+        sa.Column("graylog_stream_id", sa.String(length=1024), nullable=False),
+        sa.Column("grafana_org_id", sa.String(length=1024), nullable=False),
+        sa.Column("grafana_dashboard_folder_id", sa.String(length=1024), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "network_connectors_services",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("service_name", sa.String(length=255), nullable=False),
+        sa.Column("auth_type", sa.String(length=50), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "available_network_connectors_keys",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("network_connector_id", sa.Integer(), nullable=True),
+        sa.Column("network_connector_name", sa.String(length=255), nullable=False),
+        sa.Column("auth_key_name", sa.String(length=255), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["network_connector_id"],
+            ["available_network_connectors.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "network_connectors_configs",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("network_connector_service_id", sa.Integer(), nullable=True),
+        sa.Column("config_key", sa.String(length=255), nullable=False),
+        sa.Column("config_value", sa.String(length=1024), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["network_connector_service_id"],
+            ["network_connectors_services.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "network_connectors_subscriptions",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("customer_id", sa.Integer(), nullable=True),
+        sa.Column("network_connectors_service_id", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["customer_id"],
+            ["customer_network_connectors.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["network_connectors_service_id"],
+            ["network_connectors_services.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "network_connectors_keys",
+        sa.Column("id", sa.Integer(), nullable=True),
+        sa.Column("subscription_id", sa.Integer(), nullable=True),
+        sa.Column("auth_key_name", sa.String(length=255), nullable=False),
+        sa.Column("auth_value", sa.String(length=1024), nullable=False),
+        sa.ForeignKeyConstraint(["subscription_id"], ["network_connectors_subscriptions.id"]),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("network_connectors_keys")
+    op.drop_table("network_connectors_subscriptions")
+    op.drop_table("network_connectors_configs")
+    op.drop_table("available_network_connectors_keys")
+    op.drop_table("network_connectors_services")
+    op.drop_table("customer_network_connectors_meta")
+    op.drop_table("customer_network_connectors")
+    op.drop_table("available_network_connectors")
+    # ### end Alembic commands ###

backend/alembic/versions/c3ad5012f4db_add_job_description_to_job_metadata_.py

@@ -0,0 +1,31 @@
+"""Add Job Description to Job Metadata Table
+
+Revision ID: c3ad5012f4db
+Revises: bdf40d064ed1
+Create Date: 2024-04-23 14:11:01.313978
+
+"""
+from typing import Sequence
+from typing import Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "c3ad5012f4db"
+down_revision: Union[str, None] = "bdf40d064ed1"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("scheduled_job_metadata", sa.Column("job_description", sa.String(length=1024), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

backend/app/connectors/dfir_iris/routes/users.py

@@ -8,6 +8,7 @@
 from app.connectors.dfir_iris.schema.alerts import AlertResponse
 from app.connectors.dfir_iris.schema.users import User
 from app.connectors.dfir_iris.schema.users import UserAddedToCustomerResponse
+from app.connectors.dfir_iris.schema.users import UserRemovedFromCustomerResponse
 from app.connectors.dfir_iris.schema.users import UsersResponse
 from app.connectors.dfir_iris.services.users import assign_user_to_alert
 from app.connectors.dfir_iris.services.users import delete_user_from_alert
@@ -134,6 +135,44 @@ async def add_user_to_customers_route(
         raise HTTPException(status_code=400, detail=f"Failed to add user {user_id} to customers {customers}")
 
 
+@dfir_iris_users_router.delete(
+    "/remove/{user_id}/{customer_id}",
+    response_model=AlertResponse,
+    description="Remove a user from a customer",
+    dependencies=[Security(AuthHandler().require_any_scope("admin", "analyst"))],
+)
+async def remove_user_from_customer_route(
+    user_id: int,
+    customer_id: str,
+) -> UserRemovedFromCustomerResponse:
+    """
+    Remove a user from a customer.
+
+    Parameters:
+    - customer_id (str): The ID of the customer.
+    - user_id (int): The ID of the user.
+
+    Returns:
+    - AlertResponse: The response containing the removed user.
+
+    Raises:
+    - HTTPException: If the customer or user does not exist.
+    """
+    customers = await collect_all_customers()
+    customer_ids = [str(customer["customer_id"]) for customer in customers]
+    if customer_id in customer_ids:
+        customer_ids.remove(customer_id)
+    else:
+        raise HTTPException(status_code=404, detail="Customer ID not found")
+    logger.info(f"Customer IDs: {customer_ids}")
+    logger.info(f"Removing user {user_id} from customers {customer_ids}")
+    success = await add_user_to_customers(customer_ids, user_id)
+    if success:
+        return UserRemovedFromCustomerResponse(message=f"User {user_id} removed from customer {customer_id}", success=True)
+    else:
+        raise HTTPException(status_code=400, detail=f"Failed to remove user {user_id} from customer {customer_id}")
+
+
 @dfir_iris_users_router.delete(
     "/assign/{alert_id}/{user_id}",
     response_model=AlertResponse,

backend/app/connectors/dfir_iris/schema/users.py

@@ -20,3 +20,8 @@ class UsersResponse(BaseModel):
 class UserAddedToCustomerResponse(BaseModel):
     success: bool
     message: str
+
+
+class UserRemovedFromCustomerResponse(BaseModel):
+    success: bool
+    message: str

backend/app/connectors/shuffle/services/integrations.py

@@ -16,4 +16,5 @@ async def execute_integration(request: IntegrationRequest) -> dict:
     """
     logger.info(f"Executing integration: {request}")
     response = await send_post_request("/api/v1/apps/categories/run", request.dict())
+    logger.info(f"Response: {response}")
     return response

backend/app/connectors/shuffle/utils/universal.py

@@ -150,6 +150,7 @@ async def send_post_request(
             json=data,
             verify=False,
         )
+        logger.info(f"Response from Shuffle API: {response.json()}")
 
         if response.status_code == 204:
             return {

backend/app/customer_provisioning/routes/provision.py

@@ -18,6 +18,8 @@
 from app.customer_provisioning.schema.provision import ProvisionDashboardRequest
 from app.customer_provisioning.schema.provision import ProvisionDashboardResponse
 from app.customer_provisioning.schema.provision import ProvisionNewCustomer
+from app.customer_provisioning.schema.provision import UpdateOffice365OrgIdRequest
+from app.customer_provisioning.schema.provision import UpdateOffice365OrgIdResponse
 from app.customer_provisioning.schema.wazuh_worker import ProvisionWorkerRequest
 from app.customer_provisioning.schema.wazuh_worker import ProvisionWorkerResponse
 from app.customer_provisioning.services.provision import provision_dashboards
@@ -364,3 +366,43 @@ async def provision_dashboards_route(
             grafana_url=request.grafana_url,
         ),
     )
+
+
+@customer_provisioning_router.put(
+    "/update/office365_org_id/{customer_code}",
+    response_model=UpdateOffice365OrgIdResponse,
+    description="Update Office 365 organization ID",
+    dependencies=[Security(AuthHandler().require_any_scope("admin", "analyst"))],
+)
+async def update_office_365_org_id(
+    customer_code: str,
+    request: UpdateOffice365OrgIdRequest = Body(...),
+    session: AsyncSession = Depends(get_db),
+):
+    """
+    Update Office 365 organization ID for a customer.
+
+    Args:
+        customer_code (str): The code of the customer to update.
+        request (UpdateOffice365OrgIdRequest): The request data for updating Office 365 organization ID.
+        session (AsyncSession): The database session.
+
+    Returns:
+        UpdateOffice365OrgIdResponse: The response data for the updated Office 365 organization ID.
+    """
+    logger.info("Updating Office 365 organization ID")
+    # Update within the `CustomersMeta` table based on the customer code
+    stmt = select(CustomersMeta).where(CustomersMeta.customer_code == customer_code)
+    result = await session.execute(stmt)
+    customer_meta = result.scalars().first()
+    if not customer_meta:
+        raise HTTPException(
+            status_code=404,
+            detail=f"Customer meta not found for customer: {customer_code}. Please provision the customer first.",
+        )
+    customer_meta.customer_meta_office365_organization_id = request.office365_org_id
+    await session.commit()
+    return UpdateOffice365OrgIdResponse(
+        message="Office 365 organization ID updated successfully",
+        success=True,
+    )

backend/app/customer_provisioning/schema/provision.py

@@ -54,7 +54,7 @@ class ProvisionNewCustomer(BaseModel):
     )
     customer_subscription: List[CustomerSubsctipion] = Field(
         ...,
-        example=["Wazuh", "Office365"],
+        example=["Wazuh"],
         description="List of subscriptions for the customer",
     )
     dashboards_to_include: DashboardProvisionRequest = Field(
@@ -269,3 +269,18 @@ class ProvisionDashboardResponse(BaseModel):
         description="Message indicating the status of the request",
     )
     success: bool = Field(..., description="Whether the request was successful or not")
+
+
+class UpdateOffice365OrgIdRequest(BaseModel):
+    office365_org_id: str = Field(
+        ...,
+        description="Office 365 organization ID",
+    )
+
+
+class UpdateOffice365OrgIdResponse(BaseModel):
+    message: str = Field(
+        ...,
+        description="Message indicating the status of the request",
+    )
+    success: bool = Field(..., description="Whether the request was successful or not")

backend/app/customer_provisioning/services/decommission.py

@@ -40,6 +40,9 @@ async def decomission_wazuh_customer(
     """
     logger.info(f"Decomissioning customer {customer_meta.customer_name}")
 
+    # Delete DFIR-IRIS Customer
+    await delete_customer(customer_id=customer_meta.customer_meta_iris_customer_id)
+
     # Delete the Wazuh Agents
     agents = await gather_wazuh_agents(customer_meta.customer_code)
     agents_deleted = await delete_wazuh_agents(agents)
@@ -61,9 +64,6 @@ async def decomission_wazuh_customer(
         organization_id=int(customer_meta.customer_meta_grafana_org_id),
     )
 
-    # Delete DFIR-IRIS Customer
-    await delete_customer(customer_id=customer_meta.customer_meta_iris_customer_id)
-
     # Decommission Wazuh Worker
     await decommission_wazuh_worker(
         request=DecommissionWorkerRequest(customer_name=customer_meta.customer_name),

backend/app/customer_provisioning/services/dfir_iris.py

@@ -82,6 +82,13 @@ async def delete_customer(customer_id: int):
         None
     """
     client, admin = await initialize_client_and_admin("DFIR-IRIS")
-    result = await fetch_and_validate_data(client, admin.delete_customer, customer_id)
+    try:
+        result = await fetch_and_validate_data(client, admin.delete_customer, customer_id)
+    except Exception as e:
+        logger.error(f"Failed to delete customer: please remove the user from the iris customer within DFIR-IRIS {e}")
+        raise HTTPException(
+            status_code=400,
+            detail="Failed to delete IRIS customer: please remove the user from the iris customer within DFIR-IRIS",
+        )
     logger.info(f"Result: {result}")
     return None