Merge pull request #28 from kartoza/fix-s3-staging-env

Fix s3 staging env
kartoza · May 1, 2024 · 090fbeb · 090fbeb
2 parents 9b4e32b + 3309875
commit 090fbeb
Show file tree

Hide file tree

Showing 7 changed files with 72 additions and 21 deletions.
diff --git a/django_project/core/settings/dev.py b/django_project/core/settings/dev.py
@@ -54,3 +54,33 @@
         'level': 'WARNING'
     }
 }
+
+# for dev, we use minio for both default and input_layer
+STORAGES = {
+    "default": {
+        "BACKEND": "storages.backends.s3.S3Storage",
+        "OPTIONS": {
+          "access_key": os.environ.get("S3_AWS_ACCESS_KEY_ID"),
+          "secret_key": os.environ.get("S3_AWS_SECRET_ACCESS_KEY"),
+          "bucket_name": os.environ.get("AWS_S3_BUCKET_NAME"),
+          "file_overwrite": False,
+          "max_memory_size": 300 * 1024 * 1024,  # 300MB
+          "endpoint_url": os.environ.get("AWS_S3_ENDPOINT"),
+          "session_profile": None
+        },
+    },
+    "staticfiles": {
+        "BACKEND": "django.contrib.staticfiles.storage.ManifestStaticFilesStorage",
+    },
+    "input_layer_storage": {
+        "BACKEND": "storages.backends.s3.S3Storage",
+        "OPTIONS": {
+          "access_key": os.environ.get("MINIO_ACCESS_KEY_ID"),
+          "secret_key": os.environ.get("MINIO_SECRET_ACCESS_KEY"),
+          "bucket_name": os.environ.get("MINIO_BUCKET_NAME"),
+          "file_overwrite": False,
+          "max_memory_size": 300 * 1024 * 1024,  # 300MB
+          "endpoint_url": os.environ.get("MINIO_ENDPOINT"),
+        },
+    },
+}
diff --git a/django_project/core/settings/project.py b/django_project/core/settings/project.py
@@ -50,27 +50,20 @@
     "default": {
         "BACKEND": "storages.backends.s3.S3Storage",
         "OPTIONS": {
-          "access_key": os.environ.get("S3_AWS_ACCESS_KEY_ID"),
-          "secret_key": os.environ.get("S3_AWS_SECRET_ACCESS_KEY"),
           "bucket_name": os.environ.get("AWS_S3_BUCKET_NAME"),
           "file_overwrite": False,
           "max_memory_size": 300 * 1024 * 1024,  # 300MB
-          "endpoint_url": os.environ.get("AWS_S3_ENDPOINT"),
-          "session_profile": None
         },
     },
     "staticfiles": {
         "BACKEND": "django.contrib.staticfiles.storage.ManifestStaticFilesStorage",
     },
-    "minio": {
+    "input_layer_storage": {
         "BACKEND": "storages.backends.s3.S3Storage",
         "OPTIONS": {
-          "access_key": os.environ.get("MINIO_ACCESS_KEY_ID"),
-          "secret_key": os.environ.get("MINIO_SECRET_ACCESS_KEY"),
           "bucket_name": os.environ.get("MINIO_BUCKET_NAME"),
           "file_overwrite": False,
           "max_memory_size": 300 * 1024 * 1024,  # 300MB
-          "endpoint_url": os.environ.get("MINIO_ENDPOINT"),
         },
     },
 }
diff --git a/django_project/core/settings/test.py b/django_project/core/settings/test.py
@@ -16,7 +16,7 @@
     "staticfiles": {
         "BACKEND": "django.contrib.staticfiles.storage.ManifestStaticFilesStorage",
     },
-    "minio": {
+    "input_layer_storage": {
         "BACKEND": "django.core.files.storage.FileSystemStorage",
         "OPTIONS": {
           "location": "/home/web/media/minio_test",

diff --git a/django_project/cplus_api/models/layer.py b/django_project/cplus_api/models/layer.py
@@ -28,8 +28,8 @@ def output_layer_dir_path(instance, filename):
 
 
 def select_input_layer_storage():
-    """Return minio storage for input layer."""
-    return storages['minio']
+    """Return storage for input layer."""
+    return storages['input_layer_storage']
 
 
 class BaseLayer(models.Model):

diff --git a/django_project/cplus_api/tests/common.py b/django_project/cplus_api/tests/common.py
@@ -62,7 +62,7 @@ def cleanup(self):
         """Delete storage used in default and minio."""
         default_storage = storages['default']
         clear_test_dir(default_storage.location)
-        minio_storage = storages['minio']
+        minio_storage = storages['input_layer_storage']
         clear_test_dir(minio_storage.location)
 
     def store_layer_file(self, layer: InputLayer | OutputLayer,

diff --git a/django_project/cplus_api/tests/test_layer_api_view.py b/django_project/cplus_api/tests/test_layer_api_view.py
@@ -1,5 +1,6 @@
 import os
 import uuid
+from django.test import override_settings
 from django.urls import reverse
 from rest_framework.exceptions import PermissionDenied
 from core.settings.utils import absolute_path
@@ -175,6 +176,30 @@ def test_layer_detail(self):
         response = view(request, **kwargs)
         self.assertEqual(response.status_code, 403)
 
+    @override_settings(DEBUG=True)
+    def test_layer_detail_from_dev(self):
+        view = LayerDetail.as_view()
+        input_layer = InputLayerF.create(
+            privacy_type=InputLayer.PrivacyTypes.PRIVATE
+        )
+        file_path = absolute_path(
+            'cplus_api', 'tests', 'data',
+            'models', 'test_model_1.tif'
+        )
+        self.store_layer_file(input_layer, file_path)
+        kwargs = {
+            'layer_uuid': str(input_layer.uuid)
+        }
+        request = self.factory.get(
+            reverse('v1:layer-detail', kwargs=kwargs)
+        )
+        request.resolver_match = FakeResolverMatchV1
+        request.user = self.superuser
+        response = view(request, **kwargs)
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(response.data['url'])
+        self.assertEqual(response.data['uuid'], str(input_layer.uuid))
+
     def test_layer_delete(self):
         view = LayerDetail.as_view()
         input_layer = InputLayerF.create(

diff --git a/django_project/cplus_api/utils/api_helper.py b/django_project/cplus_api/utils/api_helper.py
@@ -76,13 +76,13 @@ def get_page_size(request):
 
 
 def build_minio_absolute_url(url):
+    if not settings.DEBUG:
+        return url
     minio_site = Site.objects.filter(
         name__icontains='minio api'
     ).first()
     current_site = minio_site if minio_site else Site.objects.get_current()
     scheme = 'https://'
-    if settings.DEBUG:
-        scheme = 'http://'
     domain = current_site.domain
     if not domain.endswith('/'):
         domain = domain + '/'
@@ -92,13 +92,16 @@ def build_minio_absolute_url(url):
 
 def get_minio_client():
     # Initialize MinIO client
-    minio_client = Minio(
-        os.environ.get("MINIO_ENDPOINT", "").replace(
-            "https://", "").replace("http://", ""),
-        access_key=os.environ.get("MINIO_ACCESS_KEY_ID"),
-        secret_key=os.environ.get("MINIO_SECRET_ACCESS_KEY"),
-        secure=False  # Set to True if using HTTPS
-    )
+    if settings.DEBUG:
+        minio_client = Minio(
+            os.environ.get("MINIO_ENDPOINT", "").replace(
+                "https://", "").replace("http://", ""),
+            access_key=os.environ.get("MINIO_ACCESS_KEY_ID"),
+            secret_key=os.environ.get("MINIO_SECRET_ACCESS_KEY"),
+            secure=False  # Set to True if using HTTPS
+        )
+    else:
+        minio_client = Minio("s3.amazonaws.com")
     return minio_client