From 50e30f7d1cd0784941597bdfca19373388c1c68d Mon Sep 17 00:00:00 2001 From: Peter Kerpedjiev Date: Mon, 4 Feb 2019 21:16:00 -0500 Subject: [PATCH] Allow accessing admin interface under a subpath (#98) * Switch to settings based limitations * If zoom-level == -1 use most efficient zoom level to pull out snippet * Update * Adjust to Pete's comments * Retrieve project name in TilesetSerializer * Bumped django version * Bumped clodius version number * Added the APP_BASEPATH option to allow accessing the admin interface under a subpath --- CHANGELOG.md | 4 +++ fragments/utils.py | 24 ++++++++-------- fragments/views.py | 56 ++++++++++++++++++++++++++++---------- higlass_server/settings.py | 19 +++++++++++++ higlass_server/urls.py | 4 +-- requirements.txt | 2 +- tilesets/serializers.py | 9 +++++- 7 files changed, 89 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f2a662d..965a0dad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v1.7.? (????-??-??) + +- Snippets API now allows limiting the size of snippets via `config.json` + v1.7.3 (2018-07-12) - Return datatype along with tileset info diff --git a/fragments/utils.py b/fragments/utils.py index dd051eeb..de4e56f7 100644 --- a/fragments/utils.py +++ b/fragments/utils.py @@ -23,6 +23,8 @@ from clodius.tiles.geo import get_tile_pos_from_lng_lat +import higlass_server.settings as hss + from higlass_server.utils import getRdb from fragments.exceptions import SnippetTooLarge @@ -608,10 +610,6 @@ def get_frag_from_image_tiles( start2_rel = from_y - tile_start2_id * tile_size end2_rel = to_y - tile_start2_id * tile_size - # Make sure that image snippets are smaller or equal to 1024px - if end1_rel - start1_rel > 1024: raise SnippetTooLarge() - if end2_rel - start2_rel > 1024: raise SnippetTooLarge() - # Notice the shape: height x width x channel return np.array(im.crop((start1_rel, start2_rel, end1_rel, end2_rel))) @@ -679,8 +677,10 @@ def get_frag_by_loc_from_imtiles( tiles_y_range = range(tile_start2_id, tile_end2_id + 1) # Make sure that no more than 6 standard tiles (256px) are loaded. - if tile_size * len(tiles_x_range) > 1536: raise SnippetTooLarge() - if tile_size * len(tiles_y_range) > 1536: raise SnippetTooLarge() + if tile_size * len(tiles_x_range) > hss.SNIPPET_IMT_MAX_DATA_DIM: + raise SnippetTooLarge() + if tile_size * len(tiles_y_range) > hss.SNIPPET_IMT_MAX_DATA_DIM: + raise SnippetTooLarge() # Extract image tiles tiles = [] @@ -796,8 +796,10 @@ def get_frag_by_loc_from_osm( tiles_y_range = range(tile_start2_id, tile_end2_id + 1) # Make sure that no more than 6 standard tiles (256px) are loaded. - if tile_size * len(tiles_x_range) > 1536: raise SnippetTooLarge() - if tile_size * len(tiles_y_range) > 1536: raise SnippetTooLarge() + if tile_size * len(tiles_x_range) > hss.SNIPPET_OSM_MAX_DATA_DIM: + raise SnippetTooLarge() + if tile_size * len(tiles_y_range) > hss.SNIPPET_OSM_MAX_DATA_DIM: + raise SnippetTooLarge() # Extract image tiles tiles = [] @@ -1152,8 +1154,8 @@ def get_frag( abs_dim2 = height # Maximum width / height is 512 - if abs_dim1 > 512: raise SnippetTooLarge() - if abs_dim2 > 512: raise SnippetTooLarge() + if abs_dim1 > hss.SNIPPET_MAT_MAX_DATA_DIM: raise SnippetTooLarge() + if abs_dim2 > hss.SNIPPET_MAT_MAX_DATA_DIM: raise SnippetTooLarge() # Finally, adjust to negative values. # Since relative bin IDs are adjusted by the start this will lead to a @@ -1218,7 +1220,7 @@ def get_frag( # Assign 0 for now to avoid influencing the max values frag[low_quality_bins] = 0 - # Scale array if needed + # Scale fragment down if needed scaled = False scale_x = width / frag.shape[0] if frag.shape[0] > width or frag.shape[1] > height: diff --git a/fragments/views.py b/fragments/views.py index 5ff83d8d..8dc3f77e 100755 --- a/fragments/views.py +++ b/fragments/views.py @@ -11,6 +11,8 @@ except: import pickle +import higlass_server.settings as hss + from rest_framework.authentication import BasicAuthentication from .drf_disable_csrf import CsrfExemptSessionAuthentication from io import BytesIO @@ -39,6 +41,10 @@ from higlass_server.utils import getRdb from fragments.exceptions import SnippetTooLarge +import h5py + +from math import floor, log + rdb = getRdb() logger = logging.getLogger(__name__) @@ -232,7 +238,9 @@ def get_fragments_by_loci(request): encoding = params['encoding'] representatives = params['representatives'] - tileset_idx = 6 if len(loci) and len(loci[0]) > 7 else 4 + # Check if requesting a snippet from a `.cool` cooler file + is_cool = len(loci) and len(loci[0]) > 7 + tileset_idx = 6 if is_cool else 4 zoom_level_idx = tileset_idx + 1 filetype = None @@ -286,12 +294,7 @@ def get_fragments_by_loci(request): 'error': 'Tileset not specified', }, status=400) - if tileset_file not in loci_lists: - loci_lists[tileset_file] = {} - - if locus[zoom_level_idx] not in loci_lists[tileset_file]: - loci_lists[tileset_file][locus[zoom_level_idx]] = [] - + # Get the dimensions of the snippets (i.e., width and height in px) inset_dim = ( locus[zoom_level_idx + 1] if ( @@ -300,10 +303,41 @@ def get_fragments_by_loci(request): ) else 0 ) + out_dim = inset_dim | dims + + # Make sure out dim (in pixel) is not too large + if is_cool and out_dim > hss.SNIPPET_MAT_MAX_OUT_DIM: + raise SnippetTooLarge() + if not is_cool and out_dim > hss.SNIPPET_IMG_MAX_OUT_DIM: + raise SnippetTooLarge() + + if tileset_file not in loci_lists: + loci_lists[tileset_file] = {} + + if is_cool: + with h5py.File(tileset_file, 'r') as f: + # get base resolution of cooler file + max_zoom = f.attrs['max-zoom'] + bin_size = int(f[str(max_zoom)].attrs['bin-size']) + else: + bin_size = 1 + + # Get max abs dim in base pairs + max_abs_dim = max(locus[2] - locus[1], locus[5] - locus[4]) + + # Find closest zoom level if `zoomout_level < 0` + zoomout_level = ( + locus[zoom_level_idx] + if locus[zoom_level_idx] >= 0 + else floor(log((max_abs_dim / bin_size) / out_dim, 2)) + ) + + if zoomout_level not in loci_lists[tileset_file]: + loci_lists[tileset_file][zoomout_level] = [] locus_id = '.'.join(map(str, locus)) - loci_lists[tileset_file][locus[zoom_level_idx]].append( + loci_lists[tileset_file][zoomout_level].append( locus[0:tileset_idx] + [i, inset_dim, locus_id] ) loci_ids.append(locus_id) @@ -409,12 +443,6 @@ def get_fragments_by_loci(request): data_types[idx] = 'matrix' - except SnippetTooLarge as ex: - raise - return JsonResponse({ - 'error': 'Requested fragment too large. Max is 1024x1024! Behave!', - 'error_message': str(ex) - }, status=400) except Exception as ex: raise return JsonResponse({ diff --git a/higlass_server/settings.py b/higlass_server/settings.py index faddfe27..e2dd8ef0 100644 --- a/higlass_server/settings.py +++ b/higlass_server/settings.py @@ -14,6 +14,7 @@ import os import os.path as op import slugid +import math from django.core.exceptions import ImproperlyConfigured @@ -292,6 +293,12 @@ def get_setting(name, default=None, settings=local_settings): UPLOAD_ENABLED = get_setting('UPLOAD_ENABLED', True) PUBLIC_UPLOAD_ENABLED = get_setting('PUBLIC_UPLOAD_ENABLED', True) +SNIPPET_MAT_MAX_OUT_DIM = get_setting('SNIPPET_MAT_MAX_OUT_DIM', math.inf) +SNIPPET_MAT_MAX_DATA_DIM = get_setting('SNIPPET_MAT_MAX_DATA_DIM', math.inf) +SNIPPET_IMG_MAX_OUT_DIM = get_setting('SNIPPET_IMG_MAX_OUT_DIM', math.inf) +SNIPPET_OSM_MAX_DATA_DIM = get_setting('SNIPPET_OSM_MAX_DATA_DIM', math.inf) +SNIPPET_IMT_MAX_DATA_DIM = get_setting('SNIPPET_IMT_MAX_DATA_DIM', math.inf) + # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.10/howto/static-files/ @@ -299,6 +306,18 @@ def get_setting(name, default=None, settings=local_settings): STATIC_URL = '/hgs-static/' STATIC_ROOT = 'hgs-static/' +if 'APP_BASEPATH' in os.environ: + # https://stackoverflow.com/questions/44987110/django-in-subdirectory-admin-site-is-not-working + USE_X_FORWARDED_HOST = True + FORCE_SCRIPT_NAME = os.environ['APP_BASEPATH'] + SESSION_COOKIE_PATH = os.environ['APP_BASEPATH'] + LOGIN_REDIRECT_URL = os.environ['APP_BASEPATH'] + LOGOUT_REDIRECT_URL = os.environ['APP_BASEPATH'] + + STATIC_URL = op.join(os.environ['APP_BASEPATH'], 'hgs-static') + +ADMIN_URL = r'^admin/' + # STATICFILES_DIRS = ( # os.path.join(BASE_DIR, 'static'), # ) diff --git a/higlass_server/urls.py b/higlass_server/urls.py index 0db90f1a..be08bad0 100644 --- a/higlass_server/urls.py +++ b/higlass_server/urls.py @@ -15,10 +15,10 @@ """ from django.conf.urls import url, include from django.contrib import admin -# from django.conf import settings +from django.conf import settings urlpatterns = [ - url(r'^admin/', admin.site.urls), + url(settings.ADMIN_URL, admin.site.urls), url(r'^api/v1/', include('tilesets.urls')), url(r'^api/v1/', include('fragments.urls')), ] diff --git a/requirements.txt b/requirements.txt index 629b2b50..9900ea12 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ cooler==0.7.11 django-cors-headers django-guardian django-rest-swagger -django==2.0.9 +django==2.1.5 djangorestframework==3.7.3 h5py==2.6.0 numba==0.37.0 diff --git a/tilesets/serializers.py b/tilesets/serializers.py index c3083329..a6367e6c 100644 --- a/tilesets/serializers.py +++ b/tilesets/serializers.py @@ -42,6 +42,13 @@ class TilesetSerializer(serializers.ModelSerializer): slug_field='uuid', allow_null=True, required=False) + project_name = serializers.SerializerMethodField('retrieve_project_name') + + def retrieve_project_name(self, obj): + if obj.project is None: + return '' + + return obj.project.name class Meta: owner = serializers.ReadOnlyField(source='owner.username') @@ -56,6 +63,7 @@ class Meta: 'coordSystem2', 'created', 'project', + 'project_name', 'description', 'private', ) @@ -66,7 +74,6 @@ class UserFacingTilesetSerializer(TilesetSerializer): project_name = serializers.SerializerMethodField('retrieve_project_name') project_owner = serializers.SerializerMethodField('retrieve_project_owner') - def retrieve_project_name(self, obj): if obj.project is None: return ''