Feature/ro crates runs support (#322)

* created model, manager, view, url, renderer, serializer, admin panel for RunExtraAnnotations

* Established support for RO creates keyed against runs

* Added extra_annotations relationship for runs in test_api_surface

emgapi/admin.py

@@ -87,7 +87,7 @@ class StudyAdmin(admin.ModelAdmin, NoRemoveMixin):
         'project_id',
         'study_name',
     )
-    list_filter = ('is_private', )
+    list_filter = ('is_private',)
     raw_id_fields = ('biome',)
 
     def save_model(self, request, obj, form, change):
@@ -127,7 +127,6 @@ class Meta:
 
 @admin.register(emg_models.SuperStudy)
 class SuperStudyAdmin(admin.ModelAdmin):
-
     inlines = [SuperStudyStudiesInline, SuperStudyBiomesInline, SuperStudyGenomeCataloguesInline]
     form = SuperStudyAdminForm
 
@@ -182,6 +181,18 @@ def get_search_results(self, request, queryset, search_term):
             return super().get_search_results(request, queryset, search_term)
 
 
+class RunExtraAnnotationDownloads(admin.TabularInline):
+    model = emg_models.RunExtraAnnotation
+    raw_id_fields = [
+        'run',
+        'parent_id',
+        'group_type',
+        'subdir',
+        'description',
+        'file_format'
+    ]
+    extra = 0
+
 @admin.register(emg_models.Run)
 class RunAdmin(admin.ModelAdmin, AccessionSearch):
     change_list_template = "admin/change_list_filter_sidebar.html"
@@ -209,6 +220,9 @@ class RunAdmin(admin.ModelAdmin, AccessionSearch):
         'sample',
         'study',
     ]
+    inlines = [
+        RunExtraAnnotationDownloads,
+    ]
 
     filter_property = 'study'
     prefix = 'MGYS'
@@ -361,13 +375,13 @@ class AnalysisJobAdmin(admin.ModelAdmin, AccessionSearch, NoRemoveMixin):
     def get_queryset(self, request):
         return emg_models.AnalysisJob.objects_admin.all() \
             .select_related(
-                'pipeline',
-                'analysis_status',
-                'experiment_type',
-                'run',
-                'study',
-                'assembly',
-                'sample')
+            'pipeline',
+            'analysis_status',
+            'experiment_type',
+            'run',
+            'study',
+            'assembly',
+            'sample')
 
 
 @admin.register(emg_models.StudyErrorType)

emgapi/fields.py

@@ -49,6 +49,11 @@ def get_url(self, obj, view_name, request, format):
             kwargs = {
                 'accession': obj.assembly.accession
             }
+
+        elif hasattr(obj, 'run'):
+            kwargs = {
+                'accession': obj.run.accession
+            }
         kwargs['alias'] = obj.alias
 
         return reverse(

emgapi/migrations/0010_runextraannotation.py

@@ -0,0 +1,35 @@
+# Generated by Django 3.2.18 on 2023-07-17 13:35
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('emgapi', '0009_genome_annotations_v2_downloads'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='RunExtraAnnotation',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('realname', models.CharField(db_column='REAL_NAME', max_length=255)),
+                ('alias', models.CharField(db_column='ALIAS', max_length=255)),
+                ('file_checksum', models.CharField(blank=True, db_column='CHECKSUM', max_length=255)),
+                ('checksum_algorithm', models.ForeignKey(blank=True, db_column='CHECKSUM_ALGORITHM', null=True, on_delete=django.db.models.deletion.CASCADE, to='emgapi.checksumalgorithm')),
+                ('description', models.ForeignKey(blank=True, db_column='DESCRIPTION_ID', null=True, on_delete=django.db.models.deletion.CASCADE, to='emgapi.downloaddescriptionlabel')),
+                ('file_format', models.ForeignKey(blank=True, db_column='FORMAT_ID', null=True, on_delete=django.db.models.deletion.CASCADE, to='emgapi.fileformat')),
+                ('group_type', models.ForeignKey(blank=True, db_column='GROUP_ID', null=True, on_delete=django.db.models.deletion.CASCADE, to='emgapi.downloadgrouptype')),
+                ('parent_id', models.ForeignKey(blank=True, db_column='PARENT_DOWNLOAD_ID', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='parent', to='emgapi.runextraannotation')),
+                ('run', models.ForeignKey(db_column='RUN_ID', on_delete=django.db.models.deletion.CASCADE, related_name='extra_annotations', to='emgapi.run')),
+                ('subdir', models.ForeignKey(blank=True, db_column='SUBDIR_ID', null=True, on_delete=django.db.models.deletion.CASCADE, to='emgapi.downloadsubdir')),
+            ],
+            options={
+                'db_table': 'RUN_DOWNLOAD',
+                'ordering': ('group_type', 'alias'),
+                'unique_together': {('realname', 'alias', 'run')},
+            },
+        ),
+    ]

emgapi/models.py

@@ -213,6 +213,11 @@ def available(self, request=None):
                     Q(assembly__is_private=False),
                 ],
             },
+            'RunExtraAnnotationQuerySet': {
+                'all': [
+                    Q(run__is_private=False),
+                ],
+            },
         }
 
         if request is not None and request.user.is_authenticated:
@@ -241,6 +246,10 @@ def available(self, request=None):
                 [Q(assembly__samples__studies__submission_account_id__iexact=_username,
                    is_private=True) |
                  Q(assembly__is_private=False)]
+            _query_filters['RunExtraAnnotationQuerySet']['authenticated'] = \
+                [Q(sun__samples__studies__submission_account_id__iexact=_username,
+                   is_private=True) |
+                 Q(run__is_private=False)]
 
         filters = _query_filters.get(self.__class__.__name__)
 
@@ -700,6 +709,7 @@ class AssemblyExtraAnnotationManager(BaseDownloadManager):
     pass
 
 
+
 class AssemblyExtraAnnotation(BaseDownload):
     assembly = models.ForeignKey(
         'Assembly', db_column='ASSEMBLY_ID', related_name='extra_annotations',
@@ -719,6 +729,33 @@ class Meta:
     def __str__(self):
         return f'AssemblyExtraAnnotation: {self.id} {self.alias}'
 
+class RunExtraAnnotationQuerySet(BaseQuerySet):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+class RunExtraAnnotationManager(BaseDownloadManager):
+    pass
+
+class RunExtraAnnotation(BaseDownload):
+    run = models.ForeignKey(
+        'Run', db_column='RUN_ID', related_name='extra_annotations',
+        on_delete=models.CASCADE)
+
+    @property
+    def accession(self):
+        return self.run.accession
+
+    objects = RunExtraAnnotationManager(select_related=[])
+
+    class Meta:
+        db_table = 'RUN_DOWNLOAD'
+        unique_together = (('realname', 'alias', 'run'),)
+        ordering = ('group_type', 'alias',)
+
+    def __str__(self):
+        return f'RunExtraAnnotation: {self.id} {self.alias}'
+
 
 class StudyDownloadQuerySet(BaseQuerySet):
     pass

emgapi/serializers.py

@@ -504,6 +504,19 @@ def get_pipelines(self, obj):
     def get_analyses(self, obj):
         return None
 
+    extra_annotations = relations.SerializerMethodHyperlinkedRelatedField(
+        many=True,
+        read_only=True,
+        source='get_extra_annotations',
+        model=emg_models.RunExtraAnnotation,
+        related_link_view_name='emgapi_v1:run-extra-annotations-list',
+        related_link_url_kwarg='accession',
+        related_link_lookup_field='accession',
+    )
+
+    def get_extra_annotations(self, obj):
+        return None
+
     class Meta:
         model = emg_models.Run
         exclude = (
@@ -672,6 +685,24 @@ class Meta:
         )
 
 
+class RunExtraAnnotationSerializer(BaseDownloadSerializer):
+    url = emg_fields.DownloadHyperlinkedIdentityField(
+        view_name='emgapi_v1:run-extra-annotations-detail',
+        lookup_field='alias',
+    )
+
+    class Meta:
+        model = emg_models.RunExtraAnnotation
+        fields = (
+            'id',
+            'url',
+            'alias',
+            'file_format',
+            'description',
+            'group_type',
+            'file_checksum'
+        )
+
 class RetrieveAssemblySerializer(AssemblySerializer):
 
     pipelines = emg_relations.HyperlinkedSerializerMethodResourceRelatedField(

emgapi/urls.py

@@ -65,6 +65,13 @@
     basename='assembly-extra-annotations'
 )
 
+router.register(
+    r'runs/(?P<accession>[^/]+)/extra-annotations',
+    views.RunExtraAnnotationViewSet,
+    basename='run-extra-annotations'
+)
+
+
 router.register(
     r'analyses',
     views.AnalysisJobViewSet,

emgapi/views.py

@@ -827,6 +827,60 @@ def retrieve(self, request, accession, alias,
             file_path = obj.realname
         return emg_utils.prepare_results_file_download_response(file_path, alias)
 
+class RunExtraAnnotationViewSet(
+        emg_mixins.ListModelMixin,
+        viewsets.GenericViewSet
+    ):
+        serializer_class = emg_serializers.RunExtraAnnotationSerializer
+
+        filter_backends = (
+            filters.OrderingFilter,
+        )
+
+        ordering_fields = (
+            'alias',
+        )
+
+        ordering = ('alias',)
+
+        lookup_field = 'alias'
+        lookup_value_regex = '[^/]+'
+
+        def get_queryset(self):
+            try:
+                accession = self.kwargs['accession']
+            except ValueError:
+                raise Http404()
+            return emg_models.RunExtraAnnotation.objects.available(self.request) \
+                .filter(run__accession=accession)
+
+        def get_object(self):
+            return get_object_or_404(
+                self.get_queryset(), Q(alias=self.kwargs['alias'])
+            )
+
+        def get_serializer_class(self):
+            return super(RunExtraAnnotationViewSet, self) \
+                .get_serializer_class()
+
+        def list(self, request, *args, **kwargs):
+            """
+            Retrieves list of Run Extra Annotation downloads
+            Example:
+            ---
+            `/run/<accession>/extra-annotations`
+            """
+            return super(RunExtraAnnotationViewSet, self).list(request, *args, **kwargs)
+
+        def retrieve(self, request, accession, alias,
+                     *args, **kwargs):
+            obj = self.get_object()
+            if obj.subdir is not None:
+                file_path = f'{obj.subdir}/{obj.realname}'
+            else:
+                file_path = obj.realname
+            return emg_utils.prepare_results_file_download_response(file_path, alias)
+
 
 class AnalysisJobViewSet(mixins.RetrieveModelMixin,
                          emg_mixins.ListModelMixin,

emgapianns/management/commands/import_extra_assembly_annotations.py

@@ -74,6 +74,7 @@ def handle(self, *args, **options):
             logger.info('Looking for RO Crates (.zips')
             for file in Path(self.gffs_dir).glob('**/*.zip'):
                 logger.info(f'Handling RO Crate Zip file {file}')
+                logger.info('this is the FILE NAME ' + file.name)
                 erz = 'ERZ' + file.name.split('ERZ')[1].strip('.zip')
                 try:
                     assembly = emg_models.Assembly.objects.get(accession=erz)