add tests for warn parameter

Makefile

@@ -12,7 +12,7 @@ test:
 	pytest --record-mode=once --cov-report term --cov=habanero test/
 
 test_no_vcr:
-	pytest --disable-vcr --cov-report term --cov=habanero test/
+	pytest --disable-recording --cov-report term --cov=habanero test/
 
 docs:
 	cd docs;\

habanero/crossref/crossref.py

@@ -206,6 +206,7 @@ def works(
         cursor=None,
         cursor_max=5000,
         progress_bar=False,
+        warn=False,
         **kwargs
     ):
         """
@@ -247,6 +248,9 @@ def works(
             found than this value, you will get only those found.
         :param progress_bar: [Boolean] print progress bar. only used when doing deep paging (
             when using cursor parameter). default: False
+        :param warn: [Boolean] warn instead of raise error upon HTTP request error. default: False
+            Especially helpful when passing in many DOIs where some may lead to request failures.
+            Returns `None` when `warn=True` for each DOI that errors.
         :param kwargs: additional named arguments passed on to `requests.get`, e.g., field
             queries (see examples and FieldQueries_)
 
@@ -357,6 +361,7 @@ def works(
                 None,
                 None,
                 progress_bar,
+                warn,
                 **kwargs
             )
         else:
@@ -379,7 +384,7 @@ def works(
                 None,
                 progress_bar,
                 **kwargs
-            ).do_request()
+            ).do_request(should_warn=warn)
 
     def members(
         self,
@@ -397,6 +402,7 @@ def members(
         cursor=None,
         cursor_max=5000,
         progress_bar=False,
+        warn=False,
         **kwargs
     ):
         """
@@ -440,6 +446,9 @@ def members(
             found than this value, you will get only those found. Only used if `works=True`
         :param progress_bar: [Boolean] print progress bar. only used when doing deep paging (
             when using cursor parameter). Only used if `works=True`. default: False
+        :param warn: [Boolean] warn instead of raise error upon HTTP request error. default: False
+            Especially helpful when passing in many DOIs where some may lead to request failures.
+            Returns `None` when `warn=True` for each DOI that errors.
         :param kwargs: additional named arguments passed on to `requests.get`, e.g., field
             queries (see examples and FieldQueries_)
 
@@ -492,6 +501,7 @@ def members(
             cursor_max,
             None,
             progress_bar,
+            warn,
             **kwargs
         )
 
@@ -510,6 +520,7 @@ def prefixes(
         cursor=None,
         cursor_max=5000,
         progress_bar=False,
+        warn=False,
         **kwargs
     ):
         """
@@ -550,6 +561,9 @@ def prefixes(
             found than this value, you will get only those found. Only used if `works=True`
         :param progress_bar: [Boolean] print progress bar. only used when doing deep paging (
             when using cursor parameter). Only used if `works=True`. default: False
+        :param warn: [Boolean] warn instead of raise error upon HTTP request error. default: False
+            Especially helpful when passing in many DOIs where some may lead to request failures.
+            Returns `None` when `warn=True` for each DOI that errors.
         :param kwargs: additional named arguments passed on to `requests.get`, e.g., field
             queries (see examples and FieldQueries_)
 
@@ -605,6 +619,7 @@ def prefixes(
             cursor=cursor,
             cursor_max=cursor_max,
             progress_bar=progress_bar,
+            should_warn=warn,
             **kwargs
         )
 
@@ -624,6 +639,7 @@ def funders(
         cursor=None,
         cursor_max=5000,
         progress_bar=False,
+        warn=False,
         **kwargs
     ):
         """
@@ -670,6 +686,9 @@ def funders(
             found than this value, you will get only those found. Only used if `works=True`
         :param progress_bar: [Boolean] print progress bar. only used when doing deep paging (
             when using cursor parameter). Only used if `works=True`. default: False
+        :param warn: [Boolean] warn instead of raise error upon HTTP request error. default: False
+            Especially helpful when passing in many DOIs where some may lead to request failures.
+            Returns `None` when `warn=True` for each DOI that errors.
         :param kwargs: additional named arguments passed on to `requests.get`, e.g., field
             queries (see examples and FieldQueries_)
 
@@ -701,6 +720,15 @@ def funders(
 
             # filters (as of this writing, only 1 filter is avail., "location")
             cr.funders(filter = {'location': "Sweden"})
+
+            # warn
+            cr.funders(ids = '10.13039/notarealdoi')
+            cr.funders(ids = '10.13039/notarealdoi', warn=True)
+            cr.funders(ids = '10.13039/notarealdoi', works=True, warn=True)
+            cr.funders(ids = ['10.13039/100000001','10.13039/notarealdoi'], works=True, warn=True)
+            x = cr.funders(ids = ['10.13039/100000001','10.13039/notarealdoi'], warn=True)
+            len(x) # 2
+            [type(w) for w in x] # [dict, NoneType]
         """
         return request(
             self.mailto,
@@ -722,6 +750,7 @@ def funders(
             cursor_max,
             None,
             progress_bar,
+            warn,
             **kwargs
         )
 
@@ -741,6 +770,7 @@ def journals(
         cursor=None,
         cursor_max=5000,
         progress_bar=False,
+        warn=False,
         **kwargs
     ):
         """
@@ -782,6 +812,9 @@ def journals(
             found than this value, you will get only those found. Only used if `works=True`
         :param progress_bar: [Boolean] print progress bar. only used when doing deep paging (
             when using cursor parameter). Only used if `works=True`. default: False
+        :param warn: [Boolean] warn instead of raise error upon HTTP request error. default: False
+            Especially helpful when passing in many DOIs where some may lead to request failures.
+            Returns `None` when `warn=True` for each DOI that errors.
         :param kwargs: additional named arguments passed on to `requests.get`, e.g., field
             queries (see examples and FieldQueries_)
 
@@ -843,6 +876,7 @@ def journals(
             cursor_max,
             None,
             progress_bar,
+            warn,
             **kwargs
         )
 
@@ -862,6 +896,7 @@ def types(
         cursor=None,
         cursor_max=5000,
         progress_bar=False,
+        warn=False,
         **kwargs
     ):
         """
@@ -946,6 +981,7 @@ def types(
             cursor_max,
             None,
             progress_bar,
+            warn,
             **kwargs
         )
 

habanero/request.py

@@ -1,6 +1,7 @@
 import requests
 import json
 import re
+import warnings
 
 from .filterhandler import filter_handler
 from .habanero_utils import (
@@ -37,9 +38,10 @@ def request(
     cursor_max=None,
     agency=False,
     progress_bar=False,
+    should_warn=False,
     **kwargs
 ):
-
+    warning_thrown = False
     url = url + path
 
     if cursor_max.__class__.__name__ != "NoneType":
@@ -92,6 +94,7 @@ def request(
             ids = ids.split()
         if ids.__class__.__name__ == "int":
             ids = [ids]
+        # should_warn = len(ids) > 1
         coll = []
         for i in range(len(ids)):
             if works:
@@ -114,7 +117,7 @@ def request(
                     None,
                     progress_bar,
                     **kwargs
-                ).do_request()
+                ).do_request(should_warn = should_warn)
                 coll.append(res)
             else:
                 if agency:
@@ -124,21 +127,40 @@ def request(
 
                 endpt = endpt.strip("/")
 
-                try:
-                    r = requests.get(
-                        endpt, params=payload, headers=make_ua(mailto, ua_string)
-                    )
+                r = requests.get(
+                    endpt, params=payload, headers=make_ua(mailto, ua_string)
+                )
+                if r.status_code > 201 and should_warn:
+                    warning_thrown = True
+                    mssg = '%s on %s: %s' % (r.status_code, ids[i], r.reason)
+                    warnings.warn(mssg)
+                else:
                     r.raise_for_status()
-                except requests.exceptions.HTTPError:
-                    if is_json(r):
-                        raise RequestError(r.status_code, parse_json_err(r))
-                    else:
-                        r.raise_for_status()
-                except requests.exceptions.RequestException as e:
-                    raise e
-                check_json(r)
-                js = r.json()
-                coll.append(js)
+
+                # try:
+                #     r = requests.get(
+                #         endpt, params=payload, headers=make_ua(mailto, ua_string)
+                #     )
+                #     if r.status_code > 201 and should_warn:
+                #         warning_thrown = True
+                #         mssg = '%s on %s: %s' % (r.status_code, ids[i], r.reason)
+                #         warnings.warn(mssg)
+                #     else:
+                #         r.raise_for_status()
+                # except requests.exceptions.HTTPError:
+                #     if is_json(r):
+                #         raise RequestError(r.status_code, parse_json_err(r))
+                #     else:
+                #         r.raise_for_status()
+                # except requests.exceptions.RequestException as e:
+                #     raise e
+
+                if warning_thrown:
+                    coll.append(None)
+                else:
+                    check_json(r)
+                    js = r.json()
+                    coll.append(js)
 
         if len(coll) == 1:
             coll = coll[0]

habanero/request_class.py

@@ -3,6 +3,7 @@
 import re
 import math
 from tqdm import tqdm
+import warnings
 
 from .filterhandler import filter_handler
 from .habanero_utils import (
@@ -69,7 +70,7 @@ def _url(self):
         tmpurl = self.url + self.path
         return tmpurl.strip("/")
 
-    def do_request(self):
+    def do_request(self, should_warn = False):
         filt = filter_handler(self.filter)
         if self.select.__class__ is list:
             self.select = ",".join(self.select)
@@ -100,13 +101,15 @@ def do_request(self):
         # rename query filters
         payload = rename_query_filters(payload)
 
-        js = self._req(payload=payload)
+        js = self._req(payload=payload, should_warn = should_warn)
+        if js is None:
+            return js
         cu = js["message"].get("next-cursor")
         max_avail = js["message"]["total-results"]
-        res = self._redo_req(js, payload, cu, max_avail)
+        res = self._redo_req(js, payload, cu, max_avail, should_warn)
         return res
 
-    def _redo_req(self, js, payload, cu, max_avail):
+    def _redo_req(self, js, payload, cu, max_avail, should_warn):
         if cu.__class__.__name__ != "NoneType" and self.cursor_max > len(
             js["message"]["items"]
         ):
@@ -129,7 +132,7 @@ def _redo_req(self, js, payload, cu, max_avail):
                 and total < max_avail
             ):
                 payload["cursor"] = cu
-                out = self._req(payload=payload)
+                out = self._req(payload=payload, should_warn = should_warn)
                 cu = out["message"].get("next-cursor")
                 res.append(out)
                 total = sum([len(z["message"]["items"]) for z in res])
@@ -141,7 +144,7 @@ def _redo_req(self, js, payload, cu, max_avail):
         else:
             return js
 
-    def _req(self, payload):
+    def _req(self, payload, should_warn):
         try:
             r = requests.get(
                 self._url(),
@@ -154,7 +157,12 @@ def _req(self, payload):
                 f = r.json()
                 raise RequestError(r.status_code, f["message"][0]["message"])
             except:
-                r.raise_for_status()
+                if should_warn:
+                    mssg = '%s: %s' % (r.status_code, r.reason)
+                    warnings.warn(mssg)
+                    return None
+                else:
+                    r.raise_for_status()
         except requests.exceptions.RequestException as e:
             print(e)
         check_json(r)

test/cassettes/test-funders/test_funders_bad_id_warn.yaml

@@ -0,0 +1,53 @@
+interactions:
+- request:
+    body: null
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate
+      Connection:
+      - keep-alive
+      User-Agent:
+      - python-requests/2.25.1 habanero/0.7.6
+      X-USER-AGENT:
+      - python-requests/2.25.1 habanero/0.7.6
+    method: GET
+    uri: https://api.crossref.org/funders/10.13039/notarealdoi
+  response:
+    body:
+      string: Resource not found.
+    headers:
+      access-control-allow-headers:
+      - X-Requested-With, Accept, Accept-Encoding, Accept-Charset, Accept-Language,
+        Accept-Ranges, Cache-Control
+      access-control-allow-origin:
+      - '*'
+      access-control-expose-headers:
+      - Link
+      connection:
+      - close
+      content-length:
+      - '19'
+      content-type:
+      - application/json
+      date:
+      - Sun, 07 Nov 2021 15:36:17 GMT
+      permissions-policy:
+      - interest-cohort=()
+      server:
+      - Jetty(9.4.40.v20210413)
+      x-api-pool:
+      - public
+      x-rate-limit-interval:
+      - 1s
+      x-rate-limit-limit:
+      - '50'
+      x-ratelimit-interval:
+      - 1s
+      x-ratelimit-limit:
+      - '50'
+    status:
+      code: 404
+      message: Not Found
+version: 1