Allow accessing admin interface under a subpath (#98)

* Switch to settings based limitations

* If zoom-level == -1 use most efficient zoom level to pull out snippet

* Update

* Adjust to Pete's comments

* Retrieve project name in TilesetSerializer

* Bumped django version

* Bumped clodius version number

* Added the APP_BASEPATH option to allow accessing the admin interface under a subpath

CHANGELOG.md

@@ -1,3 +1,7 @@
+v1.7.? (????-??-??)
+
+- Snippets API now allows limiting the size of snippets via `config.json`
+
 v1.7.3 (2018-07-12)
 
 - Return datatype along with tileset info

fragments/utils.py

@@ -23,6 +23,8 @@
 
 from clodius.tiles.geo import get_tile_pos_from_lng_lat
 
+import higlass_server.settings as hss
+
 from higlass_server.utils import getRdb
 from fragments.exceptions import SnippetTooLarge
 
@@ -608,10 +610,6 @@ def get_frag_from_image_tiles(
     start2_rel = from_y - tile_start2_id * tile_size
     end2_rel = to_y - tile_start2_id * tile_size
 
-    # Make sure that image snippets are smaller or equal to 1024px
-    if end1_rel - start1_rel > 1024: raise SnippetTooLarge()
-    if end2_rel - start2_rel > 1024: raise SnippetTooLarge()
-
     # Notice the shape: height x width x channel
     return np.array(im.crop((start1_rel, start2_rel, end1_rel, end2_rel)))
 
@@ -679,8 +677,10 @@ def get_frag_by_loc_from_imtiles(
         tiles_y_range = range(tile_start2_id, tile_end2_id + 1)
 
         # Make sure that no more than 6 standard tiles (256px) are loaded.
-        if tile_size * len(tiles_x_range) > 1536: raise SnippetTooLarge()
-        if tile_size * len(tiles_y_range) > 1536: raise SnippetTooLarge()
+        if tile_size * len(tiles_x_range) > hss.SNIPPET_IMT_MAX_DATA_DIM:
+            raise SnippetTooLarge()
+        if tile_size * len(tiles_y_range) > hss.SNIPPET_IMT_MAX_DATA_DIM:
+            raise SnippetTooLarge()
 
         # Extract image tiles
         tiles = []
@@ -796,8 +796,10 @@ def get_frag_by_loc_from_osm(
         tiles_y_range = range(tile_start2_id, tile_end2_id + 1)
 
         # Make sure that no more than 6 standard tiles (256px) are loaded.
-        if tile_size * len(tiles_x_range) > 1536: raise SnippetTooLarge()
-        if tile_size * len(tiles_y_range) > 1536: raise SnippetTooLarge()
+        if tile_size * len(tiles_x_range) > hss.SNIPPET_OSM_MAX_DATA_DIM:
+            raise SnippetTooLarge()
+        if tile_size * len(tiles_y_range) > hss.SNIPPET_OSM_MAX_DATA_DIM:
+            raise SnippetTooLarge()
 
         # Extract image tiles
         tiles = []
@@ -1152,8 +1154,8 @@ def get_frag(
         abs_dim2 = height
 
     # Maximum width / height is 512
-    if abs_dim1 > 512: raise SnippetTooLarge()
-    if abs_dim2 > 512: raise SnippetTooLarge()
+    if abs_dim1 > hss.SNIPPET_MAT_MAX_DATA_DIM: raise SnippetTooLarge()
+    if abs_dim2 > hss.SNIPPET_MAT_MAX_DATA_DIM: raise SnippetTooLarge()
 
     # Finally, adjust to negative values.
     # Since relative bin IDs are adjusted by the start this will lead to a
@@ -1218,7 +1220,7 @@ def get_frag(
     # Assign 0 for now to avoid influencing the max values
     frag[low_quality_bins] = 0
 
-    # Scale array if needed
+    # Scale fragment down if needed
     scaled = False
     scale_x = width / frag.shape[0]
     if frag.shape[0] > width or frag.shape[1] > height:

fragments/views.py

@@ -11,6 +11,8 @@
 except:
     import pickle
 
+import higlass_server.settings as hss
+
 from rest_framework.authentication import BasicAuthentication
 from .drf_disable_csrf import CsrfExemptSessionAuthentication
 from io import BytesIO
@@ -39,6 +41,10 @@
 from higlass_server.utils import getRdb
 from fragments.exceptions import SnippetTooLarge
 
+import h5py
+
+from math import floor, log
+
 rdb = getRdb()
 
 logger = logging.getLogger(__name__)
@@ -232,7 +238,9 @@ def get_fragments_by_loci(request):
     encoding = params['encoding']
     representatives = params['representatives']
 
-    tileset_idx = 6 if len(loci) and len(loci[0]) > 7 else 4
+    # Check if requesting a snippet from a `.cool` cooler file
+    is_cool = len(loci) and len(loci[0]) > 7
+    tileset_idx = 6 if is_cool else 4
     zoom_level_idx = tileset_idx + 1
 
     filetype = None
@@ -286,12 +294,7 @@ def get_fragments_by_loci(request):
                     'error': 'Tileset not specified',
                 }, status=400)
 
-            if tileset_file not in loci_lists:
-                loci_lists[tileset_file] = {}
-
-            if locus[zoom_level_idx] not in loci_lists[tileset_file]:
-                loci_lists[tileset_file][locus[zoom_level_idx]] = []
-
+            # Get the dimensions of the snippets (i.e., width and height in px)
             inset_dim = (
                 locus[zoom_level_idx + 1]
                 if (
@@ -300,10 +303,41 @@ def get_fragments_by_loci(request):
                 )
                 else 0
             )
+            out_dim = inset_dim | dims
+
+            # Make sure out dim (in pixel) is not too large
+            if is_cool and out_dim > hss.SNIPPET_MAT_MAX_OUT_DIM:
+                raise SnippetTooLarge()
+            if not is_cool and out_dim > hss.SNIPPET_IMG_MAX_OUT_DIM:
+                raise SnippetTooLarge()
+
+            if tileset_file not in loci_lists:
+                loci_lists[tileset_file] = {}
+
+            if is_cool:
+                with h5py.File(tileset_file, 'r') as f:
+                    # get base resolution of cooler file
+                    max_zoom = f.attrs['max-zoom']
+                    bin_size = int(f[str(max_zoom)].attrs['bin-size'])
+            else:
+                bin_size = 1
+
+            # Get max abs dim in base pairs
+            max_abs_dim = max(locus[2] - locus[1], locus[5] - locus[4])
+
+            # Find closest zoom level if `zoomout_level < 0`
+            zoomout_level = (
+                locus[zoom_level_idx]
+                if locus[zoom_level_idx] >= 0
+                else floor(log((max_abs_dim / bin_size) / out_dim, 2))
+            )
+
+            if zoomout_level not in loci_lists[tileset_file]:
+                loci_lists[tileset_file][zoomout_level] = []
 
             locus_id = '.'.join(map(str, locus))
 
-            loci_lists[tileset_file][locus[zoom_level_idx]].append(
+            loci_lists[tileset_file][zoomout_level].append(
                 locus[0:tileset_idx] + [i, inset_dim, locus_id]
             )
             loci_ids.append(locus_id)
@@ -409,12 +443,6 @@ def get_fragments_by_loci(request):
 
                         data_types[idx] = 'matrix'
 
-    except SnippetTooLarge as ex:
-        raise
-        return JsonResponse({
-            'error': 'Requested fragment too large. Max is 1024x1024! Behave!',
-            'error_message': str(ex)
-        }, status=400)
     except Exception as ex:
         raise
         return JsonResponse({

higlass_server/settings.py

@@ -14,6 +14,7 @@
 import os
 import os.path as op
 import slugid
+import math
 
 from django.core.exceptions import ImproperlyConfigured
 
@@ -292,13 +293,31 @@ def get_setting(name, default=None, settings=local_settings):
 UPLOAD_ENABLED = get_setting('UPLOAD_ENABLED', True)
 PUBLIC_UPLOAD_ENABLED = get_setting('PUBLIC_UPLOAD_ENABLED', True)
 
+SNIPPET_MAT_MAX_OUT_DIM = get_setting('SNIPPET_MAT_MAX_OUT_DIM', math.inf)
+SNIPPET_MAT_MAX_DATA_DIM = get_setting('SNIPPET_MAT_MAX_DATA_DIM', math.inf)
+SNIPPET_IMG_MAX_OUT_DIM = get_setting('SNIPPET_IMG_MAX_OUT_DIM', math.inf)
+SNIPPET_OSM_MAX_DATA_DIM = get_setting('SNIPPET_OSM_MAX_DATA_DIM', math.inf)
+SNIPPET_IMT_MAX_DATA_DIM = get_setting('SNIPPET_IMT_MAX_DATA_DIM', math.inf)
+
 
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/1.10/howto/static-files/
 
 STATIC_URL = '/hgs-static/'
 STATIC_ROOT = 'hgs-static/'
 
+if 'APP_BASEPATH' in os.environ:
+    # https://stackoverflow.com/questions/44987110/django-in-subdirectory-admin-site-is-not-working
+    USE_X_FORWARDED_HOST = True
+    FORCE_SCRIPT_NAME = os.environ['APP_BASEPATH']
+    SESSION_COOKIE_PATH = os.environ['APP_BASEPATH']
+    LOGIN_REDIRECT_URL = os.environ['APP_BASEPATH']
+    LOGOUT_REDIRECT_URL = os.environ['APP_BASEPATH']
+
+    STATIC_URL = op.join(os.environ['APP_BASEPATH'], 'hgs-static')
+
+ADMIN_URL = r'^admin/'
+
 # STATICFILES_DIRS = (
 #    os.path.join(BASE_DIR, 'static'),
 # )

higlass_server/urls.py

@@ -15,10 +15,10 @@
 """
 from django.conf.urls import url, include
 from django.contrib import admin
-# from django.conf import settings
+from django.conf import settings
 
 urlpatterns = [
-    url(r'^admin/', admin.site.urls),
+    url(settings.ADMIN_URL, admin.site.urls),
     url(r'^api/v1/', include('tilesets.urls')),
     url(r'^api/v1/', include('fragments.urls')),
 ]

requirements.txt

@@ -5,7 +5,7 @@ cooler==0.7.11
 django-cors-headers
 django-guardian
 django-rest-swagger
-django==2.0.9
+django==2.1.5
 djangorestframework==3.7.3
 h5py==2.6.0
 numba==0.37.0

tilesets/serializers.py

@@ -42,6 +42,13 @@ class TilesetSerializer(serializers.ModelSerializer):
             slug_field='uuid',
             allow_null=True,
             required=False)
+    project_name = serializers.SerializerMethodField('retrieve_project_name')
+
+    def retrieve_project_name(self, obj):
+        if obj.project is None:
+            return ''
+
+        return obj.project.name
 
     class Meta:
         owner = serializers.ReadOnlyField(source='owner.username')
@@ -56,6 +63,7 @@ class Meta:
             'coordSystem2',
             'created',
             'project',
+            'project_name',
             'description',
             'private',
         )
@@ -66,7 +74,6 @@ class UserFacingTilesetSerializer(TilesetSerializer):
     project_name = serializers.SerializerMethodField('retrieve_project_name')
     project_owner = serializers.SerializerMethodField('retrieve_project_owner')
 
-
     def retrieve_project_name(self, obj):
         if obj.project is None:
             return ''