Capture upload via web UI

.pre-commit-config.yaml

@@ -31,21 +31,21 @@ repos:
     #   biome-format    Format the committed files
     #   biome-lint      Lint and apply safe fixes to the committed files
     - repo: https://github.com/biomejs/pre-commit
-      rev: v2.2.2
+      rev: v2.2.4
       hooks:
           - id: biome-check
             additional_dependencies: ["@biomejs/biome@^1.0.0"]
 
     # automatically upgrades Django code to migrates patterns and avoid deprecation warnings
     - repo: https://github.com/adamchainz/django-upgrade
-      rev: "1.27.0"
+      rev: "1.28.0"
       hooks:
           - id: django-upgrade
             args: ["--target-version", "4.2"]
 
     # runs the ruff linter and formatter
     - repo: https://github.com/astral-sh/ruff-pre-commit
-      rev: v0.12.11
+      rev: v0.13.0
       hooks:
           # linter
           - id: ruff # runs ruff check --force-exclude

gateway/config/settings/base.py

@@ -507,3 +507,16 @@ def __get_random_token(length: int) -> str:
     "SDS_NEW_USERS_APPROVED_ON_CREATION",
     default=False,
 )
+
+# File upload limits
+# ------------------------------------------------------------------------------
+# Maximum number of files that can be uploaded at once
+DATA_UPLOAD_MAX_NUMBER_FILES: int = env.int(
+    "DATA_UPLOAD_MAX_NUMBER_FILES", default=1000
+)
+
+# Maximum memory size for file uploads (default: 2.5MB, increased to 100MB)
+DATA_UPLOAD_MAX_MEMORY_SIZE: int = env.int(
+    "DATA_UPLOAD_MAX_MEMORY_SIZE",
+    default=104857600,  # 100MB
+)

gateway/sds_gateway/api_methods/helpers/file_helpers.py

@@ -0,0 +1,146 @@
+from http import HTTPStatus
+
+from rest_framework import status
+from rest_framework.parsers import MultiPartParser
+from rest_framework.request import Request
+from rest_framework.test import APIRequestFactory
+
+from sds_gateway.api_methods.views.capture_endpoints import CaptureViewSet
+from sds_gateway.api_methods.views.file_endpoints import CheckFileContentsExistView
+from sds_gateway.api_methods.views.file_endpoints import FileViewSet
+
+
+def upload_file_helper_simple(request, file_data):
+    """Upload a single file using FileViewSet.create.
+
+    file_data should contain all required fields: name, directory, file,
+    media_type, etc. Returns ([response], []) for success, ([], [error]) for
+    error, and handles 409 as a warning.
+    """
+    factory = APIRequestFactory()
+    django_request = factory.post(
+        request.path,
+        file_data,
+        format="multipart",
+    )
+    django_request.user = request.user
+    drf_request = Request(django_request, parsers=[MultiPartParser()])
+    drf_request.user = request.user
+    view = FileViewSet()
+    view.request = drf_request
+    view.action = "create"
+    view.format_kwarg = None
+    view.args = ()
+    view.kwargs = {}
+    try:
+        response = view.create(drf_request)
+    except (ValueError, TypeError, AttributeError, KeyError) as e:
+        return [], [f"Data validation error: {e}"]
+    else:
+        responses = []
+        errors = []
+
+        if not hasattr(response, "status_code"):
+            errors.append(getattr(response, "data", str(response)))
+        else:
+            http_status = HTTPStatus(response.status_code)
+            response_data = getattr(response, "data", str(response))
+
+            if http_status.is_success:
+                responses.append(response)
+            elif response.status_code == status.HTTP_409_CONFLICT:
+                # Already exists, treat as warning
+                errors.append(response_data)
+            elif http_status.is_server_error:
+                # Handle 500 and other server errors
+                errors.append("Internal server error")
+            elif http_status.is_client_error:
+                # Handle 4xx client errors
+                errors.append(f"Client error ({response.status_code}): {response_data}")
+            else:
+                # Handle any other status codes
+                errors.append(response_data)
+
+        return responses, errors
+
+
+# TODO: Use this helper method when implementing the file upload mode multiplexer.
+def check_file_contents_exist_helper(request, check_data):
+    """Call the post method of CheckFileContentsExistView with the given data.
+
+    check_data should contain the required fields: directory, name, sum_blake3,
+    etc.
+    """
+    factory = APIRequestFactory()
+    django_request = factory.post(
+        request.path,  # or a specific path for the check endpoint
+        check_data,
+        format="multipart",
+    )
+    django_request.user = request.user
+    drf_request = Request(django_request, parsers=[MultiPartParser()])
+    drf_request.user = request.user
+    view = CheckFileContentsExistView()
+    view.request = drf_request
+    view.action = None
+    view.format_kwarg = None
+    view.args = ()
+    view.kwargs = {}
+    return view.post(drf_request)
+
+
+def create_capture_helper_simple(request, capture_data):
+    """Create a capture using CaptureViewSet.create.
+
+    capture_data should contain all required fields for capture creation:
+    owner, top_level_dir, capture_type, channel, index_name, etc.
+    Returns ([response], []) for success, ([], [error]) for error, and handles
+    409 as a warning.
+    """
+    factory = APIRequestFactory()
+    django_request = factory.post(
+        request.path,
+        capture_data,
+        format="multipart",
+    )
+    django_request.user = request.user
+    drf_request = Request(django_request, parsers=[MultiPartParser()])
+    drf_request.user = request.user
+    view = CaptureViewSet()
+    view.request = drf_request
+    view.action = "create"
+    view.format_kwarg = None
+    view.args = ()
+    view.kwargs = {}
+    # Set the context for the serializer
+    view.get_serializer_context = lambda: {"request_user": request.user}
+    try:
+        response = view.create(drf_request)
+    except (ValueError, TypeError, AttributeError, KeyError) as e:
+        return [], [f"Data validation error: {e}"]
+    else:
+        responses = []
+        errors = []
+
+        if not hasattr(response, "status_code"):
+            errors.append(getattr(response, "data", str(response)))
+        else:
+            http_status = HTTPStatus(response.status_code)
+            response_data = getattr(response, "data", str(response))
+
+            if http_status.is_success:
+                responses.append(response)
+            elif response.status_code == status.HTTP_409_CONFLICT:
+                # Already exists, treat as warning
+                errors.append(response_data)
+            elif http_status.is_server_error:
+                # Handle 500 and other server errors
+                errors.append(f"Server error ({response.status_code}): {response_data}")
+            elif http_status.is_client_error:
+                # Handle 4xx client errors
+                errors.append(f"Client error ({response.status_code}): {response_data}")
+            else:
+                # Handle any other status codes
+                errors.append(response_data)
+
+        return responses, errors

gateway/sds_gateway/api_methods/serializers/file_serializers.py

@@ -219,7 +219,6 @@ def check_file_contents_exist(
             user=user,
         )
 
-        log.debug(f"Checking file contents for user in directory: {safe_dir}")
         identical_file: File | None = identical_user_owned_file.filter(
             directory=safe_dir,
             name=name,
@@ -242,14 +241,12 @@ def check_file_contents_exist(
                     user_mutable_attributes_differ = True
                     break
 
-        payload = {
+        return {
             "file_exists_in_tree": identical_file is not None,
             "file_contents_exist_for_user": file_contents_exist_for_user,
             "user_mutable_attributes_differ": user_mutable_attributes_differ,
             "asset_id": asset.uuid if asset else None,
         }
-        log.debug(payload)
-        return payload
 
 
 class FileCheckResponseSerializer(serializers.Serializer[File]):

gateway/sds_gateway/api_methods/tasks.py

@@ -758,7 +758,7 @@ def _process_item_files(
         # Get available space for logging
         media_root = Path(settings.MEDIA_ROOT)
         try:
-            total, used, free = shutil.disk_usage(media_root)
+            _total, _used, free = shutil.disk_usage(media_root)
             available_space = free - DISK_SPACE_BUFFER
         except (OSError, ValueError):
             available_space = 0

gateway/sds_gateway/api_methods/tests/test_capture_endpoints.py

@@ -241,13 +241,18 @@ def test_create_drf_capture_v0_201(self) -> None:
                 "sds_gateway.api_methods.views.capture_endpoints.infer_index_name",
                 return_value=self.drf_capture_v0.index_name,
             ),
+            patch(
+                "sds_gateway.api_methods.views.capture_endpoints.reconstruct_tree",
+                return_value=(Path("mock_path"), []),
+            ),
         ):
             response_raw = self.client.post(
                 self.list_url,
                 data={
                     "capture_type": CaptureType.DigitalRF,
                     "channel": unique_channel,
                     "top_level_dir": unique_top_level_dir,
+                    "index_name": self.drf_capture_v0.index_name,
                 },
             )
             assert response_raw.status_code == status.HTTP_201_CREATED, (
@@ -291,13 +296,18 @@ def test_create_drf_capture_v1_201(self) -> None:
                 "sds_gateway.api_methods.views.capture_endpoints.infer_index_name",
                 return_value=self.drf_capture_v1.index_name,
             ),
+            patch(
+                "sds_gateway.api_methods.views.capture_endpoints.reconstruct_tree",
+                return_value=(Path("mock_path"), []),
+            ),
         ):
             response_raw = self.client.post(
                 self.list_url,
                 data={
                     "capture_type": CaptureType.DigitalRF,
                     "channel": unique_channel,
                     "top_level_dir": unique_top_level_dir,
+                    "index_name": self.drf_capture_v1.index_name,
                 },
             )
             assert response_raw.status_code == status.HTTP_201_CREATED, (
@@ -353,13 +363,18 @@ def test_create_rh_capture_201(self) -> None:
                 "sds_gateway.api_methods.views.capture_endpoints.infer_index_name",
                 return_value=self.rh_capture.index_name,
             ),
+            patch(
+                "sds_gateway.api_methods.views.capture_endpoints.reconstruct_tree",
+                return_value=(Path("mock_path"), []),
+            ),
         ):
             response_raw = self.client.post(
                 self.list_url,
                 data={
                     "capture_type": CaptureType.RadioHound,
                     "scan_group": str(unique_scan_group),
                     "top_level_dir": self.top_level_dir_rh,
+                    "index_name": self.rh_capture.index_name,
                 },
             )
             assert response_raw.status_code == status.HTTP_201_CREATED, (
@@ -390,13 +405,18 @@ def test_create_drf_capture_already_exists(self) -> None:
                 "sds_gateway.api_methods.views.capture_endpoints.infer_index_name",
                 return_value=self.drf_capture_v0.index_name,
             ),
+            patch(
+                "sds_gateway.api_methods.views.capture_endpoints.reconstruct_tree",
+                return_value=(Path("mock_path"), []),
+            ),
         ):
             response_raw = self.client.post(
                 self.list_url,
                 data={
                     "capture_type": CaptureType.DigitalRF,
                     "channel": self.channel_v0,
                     "top_level_dir": self.top_level_dir_v0,
+                    "index_name": self.drf_capture_v0.index_name,
                 },
             )
             assert response_raw.status_code == status.HTTP_400_BAD_REQUEST
@@ -427,6 +447,10 @@ def test_create_rh_capture_scan_group_conflict(self) -> None:
                 "sds_gateway.api_methods.views.capture_endpoints.infer_index_name",
                 return_value=self.rh_capture.index_name,
             ),
+            patch(
+                "sds_gateway.api_methods.views.capture_endpoints.reconstruct_tree",
+                return_value=(Path("mock_path"), []),
+            ),
         ):
             # ACT
             response_raw = self.client.post(
@@ -435,6 +459,7 @@ def test_create_rh_capture_scan_group_conflict(self) -> None:
                     "capture_type": CaptureType.RadioHound,
                     "scan_group": str(self.scan_group),
                     "top_level_dir": self.top_level_dir_rh,
+                    "index_name": self.rh_capture.index_name,
                 },
             )
 
@@ -1215,6 +1240,7 @@ def test_duplicate_check_path_normalization_regression(self) -> None:
                     "capture_type": CaptureType.DigitalRF,
                     "channel": channel_without_slash,
                     "top_level_dir": path_without_slash,
+                    "index_name": self.drf_capture_v0.index_name,
                 },
             )
             assert response1.status_code == status.HTTP_201_CREATED, (
@@ -1228,6 +1254,7 @@ def test_duplicate_check_path_normalization_regression(self) -> None:
                     "capture_type": CaptureType.DigitalRF,
                     "channel": channel_without_slash,
                     "top_level_dir": f"/{path_without_slash}",  # Add leading slash
+                    "index_name": self.drf_capture_v0.index_name,
                 },
             )
             assert response2.status_code == status.HTTP_400_BAD_REQUEST, (
@@ -1267,6 +1294,7 @@ def test_duplicate_check_path_normalization_reverse_regression(self) -> None:
                     "capture_type": CaptureType.DigitalRF,
                     "channel": channel_with_slash,
                     "top_level_dir": path_with_slash,
+                    "index_name": self.drf_capture_v0.index_name,
                 },
             )
             assert response1.status_code == status.HTTP_201_CREATED, (
@@ -1280,6 +1308,7 @@ def test_duplicate_check_path_normalization_reverse_regression(self) -> None:
                     "capture_type": CaptureType.DigitalRF,
                     "channel": channel_with_slash,
                     "top_level_dir": path_with_slash.lstrip("/"),  # Remove slash
+                    "index_name": self.drf_capture_v0.index_name,
                 },
             )
             assert response2.status_code == status.HTTP_400_BAD_REQUEST, (
@@ -1318,6 +1347,7 @@ def test_duplicate_check_whitespace_path_regression(self) -> None:
                     "capture_type": CaptureType.DigitalRF,
                     "channel": channel_whitespace_path,
                     "top_level_dir": f" {base_path}",  # Leading space
+                    "index_name": self.drf_capture_v0.index_name,
                 },
             )
             assert response1.status_code == status.HTTP_201_CREATED, (
@@ -1331,6 +1361,7 @@ def test_duplicate_check_whitespace_path_regression(self) -> None:
                     "capture_type": CaptureType.DigitalRF,
                     "channel": channel_whitespace_path,
                     "top_level_dir": base_path,  # No leading space
+                    "index_name": self.drf_capture_v0.index_name,
                 },
             )
             # With whitespace stripping, these should be detected as duplicates
@@ -1425,6 +1456,7 @@ def test_duplicate_check_multiple_slashes_regression(self) -> None:
                     "capture_type": CaptureType.DigitalRF,
                     "channel": channel_multi_slash,
                     "top_level_dir": f"/{base_path}",
+                    "index_name": self.drf_capture_v0.index_name,
                 },
             )
             assert response1.status_code == status.HTTP_201_CREATED, (
@@ -1438,6 +1470,7 @@ def test_duplicate_check_multiple_slashes_regression(self) -> None:
                     "capture_type": CaptureType.DigitalRF,
                     "channel": channel_multi_slash,
                     "top_level_dir": f"//{base_path}",  # Double slash
+                    "index_name": self.drf_capture_v0.index_name,
                 },
             )
             # Multiple slashes are now normalized by _normalize_top_level_dir(),
@@ -1594,7 +1627,7 @@ def setUp(self) -> None:
             password="testpassword",  # noqa: S106
             is_approved=True,
         )
-        api_key, key = UserAPIKey.objects.create_key(
+        _api_key, key = UserAPIKey.objects.create_key(
             name="test-key",
             user=self.user,
         )

gateway/sds_gateway/api_methods/utils/disk_utils.py

@@ -29,7 +29,7 @@ def check_disk_space_available(
         directory = Path(settings.MEDIA_ROOT)
 
     try:
-        total, used, free = shutil.disk_usage(directory)
+        _total, _used, free = shutil.disk_usage(directory)
         available_space = free - DISK_SPACE_BUFFER
     except (OSError, ValueError) as e:
         logger.error(f"Error checking disk space for {directory}: {e}")