Fix species harvester (#4287)

* Fix species harvester

* Fix test

bims/api_views/taxon.py

@@ -5,6 +5,7 @@
 
 from django.contrib.auth import get_user_model
 from django.db import transaction, IntegrityError
+from django.forms import model_to_dict
 from django.http import Http404
 from django.db.models import Count, Case, Value, When, F, CharField, Prefetch, Q
 from django.contrib.auth.mixins import LoginRequiredMixin
@@ -358,20 +359,34 @@ def post(self, request, *args):
                 taxonomy.save()
 
         with transaction.atomic():
-            TaxonomyUpdateProposal.objects.get_or_create(
-                original_taxonomy=taxonomy,
-                taxon_group=taxon_group,
-                status='pending',
-                new_data=True,
-                scientific_name=taxonomy.scientific_name,
-                canonical_name=taxonomy.canonical_name,
-                rank=taxonomy.rank,
-                parent=taxonomy.parent,
-                taxonomic_status=taxonomy.taxonomic_status,
-                legacy_canonical_name=taxonomy.legacy_canonical_name,
-                taxon_group_under_review=taxon_group,
-                author=author_name,
+            taxonomy_data = model_to_dict(
+                taxonomy,
+                exclude=[
+                    'id',
+                    'iucn_status',
+                    'vernacular_names',
+                    'author',
+                    'tags',
+                    'biographic_distributions',
+                    'owner',
+                    'parent'])
+            taxonomy_update_proposal, created = (
+                TaxonomyUpdateProposal.objects.get_or_create(
+                    original_taxonomy=taxonomy,
+                    taxon_group=taxon_group,
+                    status='pending',
+                    new_data=True,
+                    owner=taxonomy.owner,
+                    parent=taxonomy.parent,
+                    taxon_group_under_review=taxon_group,
+                    author=author_name,
+                    iucn_status=taxonomy.iucn_status,
+                    **taxonomy_data
+                )
             )
+            if created:
+                vernacular_names_instances = list(taxonomy.vernacular_names.all())
+                taxonomy_update_proposal.vernacular_names.set(vernacular_names_instances)
 
         return Response(response)
 

bims/tasks/harvest_gbif_species.py

@@ -8,7 +8,6 @@ def harvest_gbif_species(session_id):
     from bims.utils.logger import log
     from bims.models import HarvestSession
     from bims.utils.gbif import find_species_by_area
-    from bims.enums import TaxonomicRank
 
     try:
         harvest_session = (
@@ -30,7 +29,7 @@ def harvest_gbif_species(session_id):
 
     parent_species = harvest_session.module_group.gbif_parent_species
 
-    taxa = find_species_by_area(
+    find_species_by_area(
         harvest_session.boundary_id,
         harvest_session=harvest_session,
         parent_species=parent_species,

bims/templates/harvest_gbif_species.html

@@ -38,6 +38,15 @@
 
             <div class="help-block-space"></div>
             <h3>Harvest GBIF species</h3>
+
+            {% if messages %}
+                <div class="alert alert-warning" role="alert">
+                    {% for message in messages %}
+                        <div>{{ message }}</div>
+                    {% endfor %}
+                </div>
+            {% endif %}
+
             <div id="csv-alert" class="alert alert-warning" role="alert" style="display: none;">
             </div>
             <div class="modal-body">

bims/utils/gbif.py

@@ -15,7 +15,6 @@
 from bims.models.vernacular_name import VernacularName
 from bims.enums import TaxonomicRank, TaxonomicStatus
 from bims.models.harvest_session import HarvestSession
-from bims.models.boundary import Boundary
 
 logger = logging.getLogger(__name__)
 
@@ -510,6 +509,7 @@ def add_parent_to_group(taxon: Taxonomy, group: TaxonGroup):
             return
 
     def fetch_occurrences_by_area(geometry_string):
+        from bims.scripts.import_gbif_occurrences import API_BASE_URL
         facet_offset = 0
 
         while True:
@@ -531,6 +531,22 @@ def fetch_occurrences_by_area(geometry_string):
                     ]
                 }
 
+                gbif_url = (
+                    f"{API_BASE_URL}?"
+                    f"{parent_species.rank.lower()}Key={parent_species.gbif_key}&"
+                    f"facet=acceptedTaxonKey&"
+                    f"facetLimit=100&"
+                    f"facetMinCount=1&"
+                    f"hasCoordinate=true&"
+                    f"hasGeospatialIssue=false&"
+                    f"basisOfRecord=HUMAN_OBSERVATION&"
+                    f"limit={0}&"
+                    f"geometry={geometry_string}&"
+                    f"facetOffset={facet_offset}&"
+                )
+
+                log_info(gbif_url)
+
                 occurrences_data = search(**params)
                 log_info(occurrences_data)
             except Exception as e:

bims/views/harvest_gbif_species.py

@@ -4,9 +4,10 @@
 from datetime import datetime
 
 from django.conf import settings
+from django.contrib import messages
 from django.contrib.auth.mixins import UserPassesTestMixin, LoginRequiredMixin
 from django.db.models import Q
-from django.http import HttpResponseRedirect
+from django.http import HttpResponseRedirect, Http404
 from django.views.generic import TemplateView
 from django.core.files import File
 from bims.models.taxon_group import TaxonGroup
@@ -72,8 +73,6 @@ def get_context_data(self, **kwargs):
 
     def post(self, request, *args, **kwargs):
         taxon_group_id = request.POST.get('taxon_group', None)
-        taxon_group_logo = request.FILES.get('taxon_group_logo')
-        taxon_group_name = request.POST.get('taxon_group_name', '')
         boundary_id = request.POST.get('boundary', None)
         cancel = ast.literal_eval(request.POST.get(
             'cancel', 'False'
@@ -89,21 +88,25 @@ def post(self, request, *args, **kwargs):
                 return HttpResponseRedirect(request.path_info)
             except (HarvestSession.DoesNotExist, ValueError):
                 pass
-        if taxon_group_logo and taxon_group_logo:
-            taxon_groups = TaxonGroup.objects.filter(
-                category='SPECIES_MODULE'
-            ).order_by('-display_order')
-            display_order = 1
-            if taxon_groups:
-                display_order = taxon_groups[0].display_order + 1
-            TaxonGroup.objects.create(
-                name=taxon_group_name,
-                logo=taxon_group_logo,
-                category='SPECIES_MODULE',
-                display_order=display_order
-            )
+
+        taxon_group = TaxonGroup.objects.get(id=taxon_group_id)
+        if not taxon_group.gbif_parent_species:
+            messages.error(
+                self.request,
+                'GBIF Species is missing. Please add it on the taxon management page.')
             return HttpResponseRedirect(request.path_info)
 
+        gbif_taxonomy = taxon_group.gbif_parent_species
+        if not gbif_taxonomy.gbif_key:
+            if gbif_taxonomy.gbif_data and 'key' in gbif_taxonomy.gbif_data:
+                gbif_taxonomy.gbif_key = gbif_taxonomy.gbif_data['key']
+                gbif_taxonomy.save()
+            else:
+                messages.error(
+                    self.request,
+                    f'GBIF key is missing for {taxon_group.gbif_parent_species}.')
+                return HttpResponseRedirect(request.path_info)
+
         harvest_session = HarvestSession.objects.create(
             harvester=request.user,
             start_time=datetime.now(),
@@ -127,7 +130,8 @@ def post(self, request, *args, **kwargs):
             os.mkdir(log_file_folder)
 
         with open(log_file_path, 'a+') as fi:
-            harvest_session.log_file = File(fi, name=os.path.basename(fi.name))
+            harvest_session.log_file = File(
+                fi, name=os.path.basename(fi.name))
             harvest_session.save()
 
         harvest_gbif_species.delay(harvest_session.id)