Merge pull request #252 from eaudeweb/debug

Small fixes + improved logging/debugging

app/management/commands/base/notify.py

@@ -36,6 +36,12 @@ def get_tenders(self):
         )
 
     def handle(self, *args, **options):
+        self.stdout.write(
+            self.style.SUCCESS(
+                f'Sending notifications about {self.notification_type()} '
+                f'tender updates...'
+            )
+        )
         digest = options['digest']
         changed_tenders = self.scrape_tenders()
 
@@ -44,7 +50,8 @@ def handle(self, *args, **options):
                 changed_tenders, digest, self.notification_type())
         self.stdout.write(
             self.style.SUCCESS(
-                f'Sent notifications about {changed_tenders.count()} tender(s).'
+                f'Sent notifications about {changed_tenders.count()} '
+                f'tender(s).'
             )
         )
 

app/management/commands/notify_awards.py

@@ -25,7 +25,15 @@ def add_arguments(self, parser):
         )
 
     def handle(self, *args, **options):
+        self.stdout.write(
+            self.style.SUCCESS('Sending notifications about awards...')
+        )
         digest = options['digest']
         awards = Award.objects.filter(notified=False).order_by('-award_date')
         if awards:
             send_awards_email(awards, digest)
+        self.stdout.write(
+            self.style.SUCCESS(
+                f'Sent notifications about {awards.count()} awards...'
+            )
+        )

app/management/commands/notify_tenders.py

@@ -25,6 +25,9 @@ def add_arguments(self, parser):
         )
 
     def handle(self, *args, **options):
+        self.stdout.write(
+            self.style.SUCCESS('Sending notifications about new tenders...')
+        )
         digest = options['digest']
         tenders = Tender.objects.filter(notified=False).order_by('-published')
         tender_count = tenders.count()

app/management/commands/update_ted.py

@@ -33,6 +33,9 @@ def add_arguments(self, parser):
         )
 
     def handle(self, *args, **options):
+        self.stdout.write(
+            self.style.SUCCESS('Importing new TED tenders...')
+        )
         try:
             last_ted_update = Command.days_ago(int(options['days_ago']))
         except TypeError:

app/management/commands/update_ungm.py

@@ -1,3 +1,4 @@
+
 from app.models import Tender
 from datetime import datetime, timedelta
 from app.parsers.ungm import UNGMWorker
@@ -22,7 +23,9 @@ def get_parameters():
         ]
 
     def handle(self, *args, **kwargs):
-
+        self.stdout.write(
+            self.style.SUCCESS('Importing new UNGM tenders...')
+        )
         old_tender_count = Tender.objects.count()
         if kwargs['days_ago']:
             days_ago = kwargs['days_ago']

app/parsers/ted.py

@@ -92,25 +92,9 @@ def download_archive(self, ftp, archive_date, archives):
             if not os.path.exists(self.path):
                 os.makedirs(self.path)
             file_path = os.path.join(self.path, archive_name)
-            count = 0
-            while True:
-                if not os.path.exists(file_path) or count >= 5:
-                    if count >= 5:
-                        try:
-                            ftp = self.ftp_login()
-                            os.remove(file_path)
-                        except OSError as e:
-                            logging.warning(e)
-                    with open(file_path, "wb") as f:
-                        ftp.retrbinary("RETR %s" % archive_name, f.write)
-                    self.archives.append(file_path)
-                    break
-                logging.warning(
-                    f"File already downloaded, waiting 30 seconds: {file_path}")
-                count += 1
-                time.sleep(30 * count)
-
-                logging.warning("Retrying")
+            with open(file_path, "wb") as f:
+                ftp.retrbinary("RETR %s" % archive_name, f.write)
+            self.archives.append(file_path)
 
     def parse_notices(
         self, tenders=None, set_notified=False

app/parsers/ungm.py

@@ -1,7 +1,8 @@
-from datetime import date, datetime, timedelta
 import logging
 import re
 import requests
+
+from datetime import date, datetime, timedelta
 from tempfile import TemporaryFile
 
 from bs4 import BeautifulSoup
@@ -21,17 +22,9 @@ def parse_tenders(self, tenders):
         Args:
             tenders: QuerySet
         """
-        codes = UNSPSCCode.objects.all()
-        parsed_tenders = []
-        for tender in tenders:
-            html = self.requester.get_request(tender.url)
-            parsed_tender = self.parse_ungm_notice(html, tender.url, codes)
-            if parsed_tender:
-                parsed_tenders.append(parsed_tender)
-        return UNGMWorker.update_ungm_tenders(parsed_tenders)
+        return UNGMWorker.update_ungm_tenders(self.parsed_tenders(tenders))
 
     def parse_latest_notices(self, last_date):
-        codes = UNSPSCCode.objects.all()
         last_date = last_date.strftime('%d-%b-%Y')
         page_index = 0
         tenders_count = 0
@@ -43,15 +36,8 @@ def parse_latest_notices(self, last_date):
                 requested_html_tenders)
             if not len(extracted_tenders):
                 break
-            parsed_tenders = []
-            for tender in extracted_tenders:
-                html = self.requester.get_request(tender['url'])
-                parsed_tender = self.parse_ungm_notice(
-                    html, tender['url'], codes)
-                if parsed_tender:
-                    parsed_tenders.append(parsed_tender)
             ungm_tenders, added_tenders = UNGMWorker.update_ungm_tenders(
-                parsed_tenders)
+                self.parsed_tenders(extracted_tenders))
             tenders_count += added_tenders
 
         WorkerLog.objects.create(
@@ -175,6 +161,18 @@ def parse_ungm_notice(html, url, codes):
         }
 
         return tender_item
+
+    def parsed_tenders(self, tenders):
+        codes = UNSPSCCode.objects.all()
+        for tender in tenders:
+            try:
+                url = tender.url
+            except AttributeError:
+                url = tender['url']
+            html = self.requester.get_request(url)
+            parsed_tender = self.parse_ungm_notice(html, url, codes)
+            if parsed_tender:
+                yield parsed_tender
 
     @staticmethod
     def find_by_span(soup, span):

app/server_requests.py

@@ -93,9 +93,11 @@ def request_document(self, url):
             return None
 
     def request_tenders_list(self, last_date, index):
+        """ Returns HTML page with a list of tenders """
         return self.request(TENDERS_ENDPOINT_URI, last_date, index)
 
     def request_awards_list(self):
+        """ Returns HTML page with a list of awards """
         return self.request(WINNERS_ENDPOINT_URI)
 
 
@@ -115,19 +117,20 @@ def get_data(self, url, last_date, index):
         return json.dumps(payload)
 
     def request(self, url, last_date, index):
-        # Original
         for i in range(0, 3):
-            resp = self.post_request(
+            html = self.post_request(
                 url, url + '/Search', self.get_data(url, last_date, index))
-            if resp:
-                return resp
+            if html:
+                return html
             sleep(randint(10, 15))
 
     def post_request(
             self, get_url, post_url, data, headers=HEADERS, content_type=None):
         """
         AJAX-like POST request. Does a GET initially to receive cookies that
         are used to the subsequent POST request.
+
+        Returns HTML, NOT Response object.
         """
         try:
             resp = requests.get(get_url)

app/views/tenders.py

@@ -42,7 +42,10 @@ def get_context_data(self, **kwargs):
             keyword = self.request.GET.get("keyword")
             notice_type = self.request.GET.get("type")
             seen = self.request.GET.get("seen")
-            reset = any([source, organization, status, favourite, keyword, notice_type, seen])
+            reset = any([
+                source, organization, status, favourite, keyword, notice_type,
+                seen
+            ])
             form = TendersFilter(
                 initial={
                     "organization": organization,
@@ -57,10 +60,14 @@ def get_context_data(self, **kwargs):
         else:
             form = TendersFilter()
 
-        ungm_published_today |= (self.request.GET.get("ungm_published_today") == 'True')
-        ungm_deadline_today |= (self.request.GET.get("ungm_deadline_today") == 'True')
-        ted_published_today |= (self.request.GET.get("ted_published_today") == 'True')
-        ted_deadline_today |= (self.request.GET.get("ted_deadline_today") == 'True')
+        ungm_published_today |= (
+            self.request.GET.get("ungm_published_today") == 'True')
+        ungm_deadline_today |= (
+            self.request.GET.get("ungm_deadline_today") == 'True')
+        ted_published_today |= (
+            self.request.GET.get("ted_published_today") == 'True')
+        ted_deadline_today |= (
+            self.request.GET.get("ted_deadline_today") == 'True')
 
         context["form"] = form
         context["reset"] = reset
@@ -81,7 +88,10 @@ class TenderListAjaxView(BaseAjaxListingView):
         ('has_keywords', 'has_keywords'),
         ('notice_type', 'notice_type'),
     ]
-    order_fields = ['title', 'source', 'organization', 'deadline', 'published', 'notice_type']
+    order_fields = [
+        'title', 'source', 'organization', 'deadline', 'published',
+        'notice_type'
+    ]
     case_sensitive_fields = ['title', 'source', 'organization', 'notice_type']
     model = Tender
 
@@ -92,9 +102,12 @@ def format_data(self, object_list):
                 'url': reverse('tender_detail_view', kwargs={'pk': tender.id}),
                 'source': tender.source,
                 'organization': tender.organization,
-                'deadline': 'Not specified' if not tender.deadline else tender.deadline.strftime("%m/%d/%Y, %H:%M"),
-                'published': 'Not specified' if not tender.published else tender.published.strftime("%m/%d/%Y"),
-                'notice_type': render_to_string('tenders_buttons.html', {'tender': tender})
+                'deadline': 'Not specified' if not tender.deadline else (
+                    tender.deadline.strftime("%m/%d/%Y, %H:%M")),
+                'published': 'Not specified' if not tender.published else (
+                    tender.published.strftime("%m/%d/%Y")),
+                'notice_type': render_to_string(
+                    'tenders_buttons.html', {'tender': tender})
             } for tender in object_list
         ]
         return data
@@ -121,9 +134,11 @@ def filter_data(self, request):
             awards = Award.objects.all()
             award_refs = [award.tender.reference for award in awards]
             if status == "open":
-                tenders = tenders.exclude(Q(reference__in=award_refs) | Q(deadline__lt=date.today()))
+                tenders = tenders.exclude(
+                    Q(reference__in=award_refs) | Q(deadline__lt=date.today()))
             else:
-                tenders = tenders.filter(Q(reference__in=award_refs) | Q(deadline__lt=date.today()))
+                tenders = tenders.filter(
+                    Q(reference__in=award_refs) | Q(deadline__lt=date.today()))
 
         seen = self.request.GET.get("seen")
         if seen:
@@ -180,7 +195,8 @@ def get_context_data(self, **kwargs):
         if deadline and deadline >= datetime.now(timezone.utc):
             deadline -= datetime.now(timezone.utc)
             context["deadline_in"] = self.deadline_in_string(deadline)
-        context["documents_set"] = TenderDocument.objects.filter(tender=self.object)
+        context["documents_set"] = TenderDocument.objects.filter(
+            tender=self.object)
         return context
 
 
@@ -234,7 +250,8 @@ class TenderArchiveAjaxView(TenderListAjaxView):
 
     def get_objects(self):
         current_time = datetime.now(timezone.utc)
-        tenders = Tender.objects.filter(deadline__lt=current_time, deadline__isnull=False)
+        tenders = Tender.objects.filter(
+            deadline__lt=current_time, deadline__isnull=False)
         return tenders
 
 
@@ -265,9 +282,13 @@ def get_context_data(self, **kwargs):
                 ]
             )
         )
-        tender_references = [o.reference for o in result_tender_documents if o.reference is not None]
+        tender_references = [
+            o.reference
+            for o in result_tender_documents if o.reference is not None
+        ]
 
-        # Find the tenders that match the input string and the tender ids of the tender documents
+        # Find the tenders that match the input string and the tender ids of
+        # the tender documents
         result_tenders = TenderDoc.search().query(
             elasticQ(
                 "multi_match",

scratch/settings.py

@@ -228,5 +228,9 @@
     "version": 1,
     "disable_existing_loggers": False,
     "handlers": {"console": {"class": "logging.StreamHandler"}},
-    "loggers": {"django_auth_ldap": {"level": "DEBUG", "handlers": ["console"]}},
+    "loggers": {
+        "django_auth_ldap": {
+            "level": "DEBUG", "handlers": ["console"]
+        }
+    }
 }