merge v0.13.0 release

.coveragerc

@@ -1,6 +1,6 @@
 [run]
-omit = *migrations*, *tests*, *config*, *docs*, manage.py, *env/lib/*,
-       *templates/bootstrap4/*, *__init__.py, *env/src/*, *staticfiles/*,
+omit = *migrations*, *tests*, *config*, docs_manual/*, manage.py, *env/lib/*,
+       *templates/bootstrap4/*, *__init__.py, *env/src/*, src/*, *staticfiles/*,
        versioneer.py, setup.py, *node_modules*, htmlcov/*
 plugins =
     django_coverage_plugin

.github/workflows/build.yml

@@ -56,11 +56,11 @@ jobs:
           sudo apt install -qq build-essential zlib1g-dev libtiff5-dev libjpeg8-dev libfreetype6-dev
           sudo apt install -qq liblcms2-dev libwebp-dev libpq-dev graphviz-dev
       - name: Install Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Install project Python dependencies
         run: |
           pip install wheel==0.37.1

CHANGELOG.rst

@@ -5,6 +5,92 @@ Changelog for the SODAR project. Loosely follows the
 `Keep a Changelog <http://keepachangelog.com/en/1.0.0/>`_ guidelines.
 
 
+v0.13.0 (2023-02-08)
+====================
+
+Added
+-----
+
+- **Irodsbackend**
+    - Create iRODS user accounts at login for users with LDAP/SODAR auth (#1315, #1587)
+- **Landingzones**
+    - Optional zone write access restriction to created collections (#1050, #1540)
+    - Project archiving support (#1573)
+    - UI warning for user without access for zone updating (#1581)
+- **Samplesheets**
+    - Mac keyboard shortcut support for multi-cell copying (#1531)
+    - Study render table caching (#1509)
+    - ``syncstudytables`` management command (#1509)
+    - ``get_last_material_index()`` helper (#1554)
+    - ``get_latest_file_path()`` helper (#1554)
+    - "Not found" element for iRODS modal filter (#1562)
+    - Existing iRODS file check in material name editing (#1494)
+    - Omit IGV session files by file name suffix (#1575, #1577)
+    - ``SHEETS_IGV_OMIT_BAM`` and ``SHEETS_IGV_OMIT_VCF`` Django settings (#1575, #1595)
+    - ``get_igv_omit_override()`` and ``check_igv_file_name()`` in study app utils (#1575, #1577)
+    - Project archiving support (#1572)
+    - ``igv_omit_bam``, ``igv_omit_vcf`` and ``igv_genome`` project settings (#1478, #1577)
+    - Project-wide genome selecting for IGV session generation (#1478)
+
+Changed
+-------
+
+- **General**
+    - Upgrade to django-sodar-core v0.12.0 (#1567, #1576)
+    - Use default Gunicorn worker class in production (#1536)
+    - Upgrade to fastobo v0.12.2 (#1561)
+    - Update ``.coveragerc`` (#1582)
+    - Upgrade ``checkout`` and ``setup-python`` GitHub actions (#1591)
+- **Irodsbackend**
+    - Update backend iRODS connection handling (#909, #1542, #1545)
+    - Rename ``IrodsAPI.get_child_colls()``
+- **Landingzones**
+    - Refactor permissions (#1573)
+- **Samplesheets**
+    - Upgrade critical Vue app dependencies (#1527, #1571)
+    - Remove redundant node UUIDs from render tables (#708)
+    - Improve IGV session file XML generating (#1585)
+    - Do not create ``sheet_edit_start`` timeline events (#1570)
+    - Use role ranking in ``EditConfigMixin`` (#1589)
+- **Taskflowbackend**
+    - Remove legacy ``landing_zone_create`` build error handling (#1530)
+
+Fixed
+-----
+
+- **General**
+    - Missing ``LDAP_ALT_DOMAINS`` Django setting (#1594)
+- **Irodsbackend**
+    - Unhandled backend init exception in ``IrodsStatisticsAjaxView`` (#1539)
+    - iRODS session disconnection issues (#909, #1542)
+    - Ajax view access for inherited owners (#1566)
+- **Landingzones**
+    - Typo in ``LANDINGZONES_TRIGGER_MOVE_INTERVAL`` (#1541)
+- **Samplesheets**
+    - Crash from incompatibility with ``packaging==0.22`` (#1550)
+    - Cancer shortcuts expecting specific naming convention (#1554, #1563)
+    - Cancer shortcut caching with identical library names in study (#1560, #1564)
+    - iRODS modal filter input not cleared on modal re-open (#1555)
+    - Column config editing access for inherited owners (#1568)
+    - iRODS delete request accept view crash with collection request (#1584)
+    - Germline study shortcuts enabled if sample not found in assay (#1579)
+
+Removed
+-------
+
+- **Irodsbackend**
+    - Backend API ``conn`` argument (#909)
+    - ``IrodsAPI.collection_exists()`` helper (#1546)
+    - ``IrodsAPI.get_coll_by_path()`` helper
+- **Landingzones**
+    - Legacy ``LandingZoneOldListAPIView`` (#1580)
+- **Samplesheets**
+    - Unused ``config_set`` and ``num_col`` header parameters (#1551)
+    - ``get_sample_libraries()`` helper (#1554)
+    - ``get_study_libraries()`` helper (#1554)
+    - ``GenericMaterial.get_samples()`` (#1557)
+
+
 v0.12.1 (2022-11-09)
 ====================
 
@@ -69,7 +155,7 @@ Added
     - ``get_zone_path()`` helper (#1399)
     - ``get_user_group_name()`` helper (#1397)
     - ``get_ticket()`` method
-- **Landingzones***
+- **Landingzones**
     - ``LandingZone.can_display_files()`` helper (#1401)
 - **Samplesheets**
     - Statistics badge in iRODS dir modal (#1434)

config/settings/base.py

@@ -345,6 +345,9 @@
 ENABLE_LDAP = env.bool('ENABLE_LDAP', False)
 ENABLE_LDAP_SECONDARY = env.bool('ENABLE_LDAP_SECONDARY', False)
 
+# Alternative domains for detecting LDAP access by email address
+LDAP_ALT_DOMAINS = env.list('LDAP_ALT_DOMAINS', None, default=[])
+
 if ENABLE_LDAP:
     import itertools
     import ldap
@@ -583,7 +586,7 @@ def set_logging(level=None):
 
 
 # General API settings
-SODAR_API_DEFAULT_VERSION = '0.12.1'
+SODAR_API_DEFAULT_VERSION = '0.13.0'
 SODAR_API_ALLOWED_VERSIONS = [
     '0.7.0',
     '0.7.1',
@@ -597,6 +600,7 @@ def set_logging(level=None):
     '0.11.3',
     '0.12.0',
     '0.12.1',
+    '0.13.0',
 ]
 SODAR_API_MEDIA_TYPE = 'application/vnd.bihealth.sodar+json'
 SODAR_API_DEFAULT_HOST = env.url(
@@ -780,14 +784,22 @@ def set_logging(level=None):
 # Remote sample sheet sync interval in minutes
 SHEETS_SYNC_INTERVAL = env.int('SHEETS_SYNC_INTERVAL', 5)
 
+# BAM file name suffixes to omit from study shortcuts and IGV session generation
+SHEETS_IGV_OMIT_BAM = env.list(
+    'SHEETS_IGV_OMIT_BAM', default=['dragen_evidence.bam']
+)
+# VCF file name suffixes to omit from study shortcuts and IGV session generation
+SHEETS_IGV_OMIT_VCF = env.list(
+    'SHEETS_IGV_OMIT_VCF', default=['cnv.vcf.gz', 'ploidy.vcf.gz', 'sv.vcf.gz']
+)
 
 # Landingzones app settings
 # Status query interval in seconds
 LANDINGZONES_STATUS_INTERVAL = env.int('LANDINGZONES_STATUS_INTERVAL', 3)
 # Enable automated move triggering based on touched file
 LANDINGZONES_TRIGGER_ENABLE = env.bool('LANDINGZONES_TRIGGER_ENABLE', True)
 # Automatic move triggering check interval in seconds
-LANDINGZONES_TRIGGER_MOVE_INVERVAL = env.int(
+LANDINGZONES_TRIGGER_MOVE_INTERVAL = env.int(
     'LANDINGZONES_TRIGGER_MOVE_INTERVAL', 30
 )
 # File name for automated move triggering

config/settings/test.py

@@ -101,11 +101,12 @@
 SHEETS_EXTERNAL_LINK_PATH = os.path.join(
     ROOT_DIR, 'samplesheets/tests/config/ext_links.json'
 )
+SHEETS_IGV_OMIT_BAM = ['dragen_evidence.bam']
+SHEETS_IGV_OMIT_VCF = ['cnv.vcf.gz', 'ploidy.vcf.gz', 'sv.vcf.gz']
 
 # Landingzones app settings
 LANDINGZONES_TRIGGER_ENABLE = True
 
-
 # iRODS settings shared by iRODS using apps
 ENABLE_IRODS = True
 IRODS_HOST = '127.0.0.1'

docker/docker-entrypoint.sh

@@ -67,7 +67,6 @@ if [[ "$1" == wsgi ]]; then
     --bind "$HTTP_HOST:$HTTP_PORT" \
     --timeout "$GUNICORN_TIMEOUT" \
     --workers 4 \
-    --worker-class gevent \
     config.wsgi
 elif [[ "$1" == celeryd ]]; then
   cd $APP_DIR

docs_manual/source/admin_commands.rst

@@ -56,3 +56,8 @@ operations regarding sample sheets, landing zones, iRODS data and ontologies.
     Find orphans in iRODS project collections.
 ``syncnames``
     Synchronize alternative names for sample sheet material search.
+``syncstudytables``
+    Build study render tables in cache for all study tables. These will be
+    automatically built when accessing sample sheets if existing  cache is not
+    up-to-date, but this can be used to e.g. regenerate the cache if something
+    has been changed in study table rendering.

docs_manual/source/admin_settings.rst

@@ -181,6 +181,12 @@ Sample Sheets Settings
     URL pattern in form of ``https://example.com/{id}``.
 ``SHEETS_SYNC_INTERVAL``
     Interval for remote sheet synchronization in minutes (integer).
+``SHEETS_IGV_OMIT_BAM``
+    BAM file name suffixes to omit from study shortcuts and IGV session
+    generation.
+``SHEETS_IGV_OMIT_VCF``
+    VCF file name suffixes to omit from study shortcuts and IGV session
+    generation.
 
 Landing Zones Settings
 ----------------------
@@ -189,7 +195,7 @@ Landing Zones Settings
     Zone status query interval in seconds (integer).
 ``LANDINGZONES_TRIGGER_ENABLE``
     Enable automated move triggering (boolean).
-``LANDINGZONES_TRIGGER_MOVE_INVERVAL``
+``LANDINGZONES_TRIGGER_MOVE_INTERVAL``
     Automated move file check interval in seconds (integer).
 ``LANDINGZONES_TRIGGER_FILE``
     File name for automated move triggering (string,

docs_manual/source/api_documentation.rst

@@ -45,7 +45,7 @@ expected version.
 
 .. code-block:: console
 
-    Accept: application/vnd.bihealth.sodar+json; version=0.12.1
+    Accept: application/vnd.bihealth.sodar+json; version=0.13.0
 
 Specific sections of the SODAR API may require their own accept header. See the
 exact header requirement in the respective documentation on each section of the

docs_manual/source/api_landingzones.rst

@@ -11,15 +11,15 @@ API Views
 
 .. currentmodule:: landingzones.views_api
 
-.. autoclass:: LandingZoneListAPIView
+.. autoclass:: ZoneListAPIView
 
-.. autoclass:: LandingZoneRetrieveAPIView
+.. autoclass:: ZoneRetrieveAPIView
 
-.. autoclass:: LandingZoneCreateAPIView
+.. autoclass:: ZoneCreateAPIView
 
-.. autoclass:: LandingZoneSubmitDeleteAPIView
+.. autoclass:: ZoneSubmitDeleteAPIView
 
-.. autoclass:: LandingZoneSubmitMoveAPIView
+.. autoclass:: ZoneSubmitMoveAPIView
 
 
 Versioning
@@ -30,4 +30,4 @@ SODAR version:
 
 .. code-block:: console
 
-    Accept: application/vnd.bihealth.sodar+json; version=0.12.1
+    Accept: application/vnd.bihealth.sodar+json; version=0.13.0

docs_manual/source/api_projectroles.rst

@@ -23,4 +23,4 @@ in the current SODAR version:
 
 .. code-block:: console
 
-    Accept: application/vnd.bihealth.sodar-core+json; version=0.11.0
+    Accept: application/vnd.bihealth.sodar-core+json; version=0.12.0

docs_manual/source/api_samplesheets.rst

@@ -30,4 +30,4 @@ SODAR version:
 
 .. code-block:: console
 
-    Accept: application/vnd.bihealth.sodar+json; version=0.12.1
+    Accept: application/vnd.bihealth.sodar+json; version=0.13.0

docs_manual/source/app_landingzones_create.rst

@@ -52,6 +52,15 @@ Create Collections
     collection names for libraries and avoid errors such as typos. Creation is
     enabled by default. When moving files from the landing zone, empty
     collections will not be created in the sample repository.
+Restrict Collections
+    If set true and "create collections" is enabled, user write access is
+    restricted to the automatically created collections. Subcollections can be
+    freely created under existing collections, but the user has no permission
+    to create new collections in the landing zone root. This will prevent
+    accidental uploads to collections with names not corresponding to paths
+    expected in the related sample sheets. If there is need to e.g. upload data
+    corresponding to libraries not yet in the sample sheets, this option can be
+    set false.
 Configuration
     Selection for special configurations of landing zones with extra features.
     In most use cases this should be left blank.

docs_manual/source/app_samplesheets_browse.rst

@@ -95,8 +95,9 @@ Row Filtering
     For filtering rows, a string can be entered into the *Filter* text input in
     the table header.
 Drag Selection
-    Multiple cells can be selected and copied into the clipboard by holding down
-    the left mouse button and "painting" cells.
+    Multiple cells can be selected by holding down the left mouse button and
+    "painting" cells. They can be copied into the clipboard with ``Ctrl+C``,
+    ``Meta+C`` or ``Cmd+C``.
 
 In SODAR certain cells offer special notation of data:
 

docs_manual/source/app_samplesheets_edit.rst

@@ -97,7 +97,10 @@ Node Names
     hyperlinks supported. Note that editing the name cell of an existing node
     will rename the existing node. Changing the row to contain a different
     existing node in case of splitting/pooling is not currently supported by the
-    editor.
+    editor. If iRODS collections have been created, SODAR will check for
+    existing files under the name of the material. If fles exist, the UI will
+    alert the user and prompt for confirmation, in order to avoid generating
+    orphaned files.
 Contacts
     Contact cells act as string cells with the following expected syntax:
     ``Contact Name <contact@email.org>``. The email can be omitted.

docs_manual/source/conf.py

@@ -22,11 +22,11 @@
 # -- Project information -----------------------------------------------------
 
 project = 'SODAR'
-copyright = '2018-2022, BIH Core Unit Bioinformatics'
+copyright = '2018-2023, BIH Core Unit Bioinformatics'
 author = 'BIH Core Unit Bioinformatics'
 
 # The full version, including alpha/beta/rc tags
-release = '0.12.1'
+release = '0.13.0'
 
 
 # -- General configuration ---------------------------------------------------

docs_manual/source/dev_apps.rst

@@ -335,17 +335,23 @@ uses an admin account to perform actions on the iRODS server. The
 ``IRODS_ENV_BACKEND`` setting can be used to override environment values for
 these backend connections.
 
-The API object handles opening and closing iRODS connections when alive, thus a
-manual cleanup of the iRODS connection is not needed.
+To create and access the iRODS session, it is recommended to do it using the
+backend API using ``get_session()`` via a context manager:
 
-By default, creating the API also creates an iRODS connection. If you only want
-to use the API for offline activity such as building iRODS paths from an object
-and want to avoid opening unnecessary connections, you can use the
-``conn=False`` kwarg:
+.. code-block:: python
+
+    with irods_backend.get_session() as irods:
+        pass  # Your session code here
+
+Alternatively, you can get the object directly with ``get_session_obj()``. This
+requires you to call ``cleanup()`` manually and it is mostly recommended for
+testing.
 
 .. code-block:: python
 
-    irods_backend = get_backend_api('omics_irods', conn=False)
+    irods = irods_backend.get_session_obj()
+    pass  # Your session code here
+    irods.cleanup()
 
 
 Irodsinfo

docs_manual/source/metadata_advanced.rst

@@ -35,6 +35,21 @@ SODAR currently supports the following study configurations:
 If the configuration is not specified or is not known to SODAR, the shortcut
 column will not be visible.
 
+Study plugins will search for the "latest" BAM and VCF files for shortcuts and
+IGV session generation. This is determined by file name: in case of multiple
+files, the last file sorted by file name is returned. Hence to ensure the most
+recent file is returned, it is recommended to use dates in the file names.
+
+Placement of files in subcollections under the library collection does not
+affect study shortcuts or IGV session inclusion. This also means files can be
+freely organized within desired subcollections.
+
+If there is need to omit BAM or VCF files with certain file name patterns from
+shortcuts and IGV sessions, file name suffixes can be defined in the
+:ref:`project update view <ui_project_update>`. Administrators can also define
+site-wide file suffixes to omit via :ref:`Django settings <admin_settings>`
+under ``SHEETS_IGV_OMIT_BAM`` and ``SHEETS_IGV_OMIT_VCF``.
+
 
 Assay iRODS Data Linking
 ========================