apply md5 fips patch (#295)

Author: kulgan

.gitignore

@@ -82,3 +82,4 @@ ignore/
 
 # pyenv
 .python-version
+report.xml

src/gdc_client/download/client.py

@@ -1,4 +1,3 @@
-import hashlib
 from io import BytesIO
 import logging
 import os
@@ -158,7 +157,7 @@ def _md5_members(self, members):
             member_uuid = m.split("/")[0]
             log.debug(f"Validating checksum for {member_uuid}...")
 
-            md5sum = hashlib.md5()
+            md5sum = utils.md5()
             filename = os.path.join(self.base_directory, m)
             with open(filename, "rb") as f:
                 md5sum.update(f.read())

src/gdc_client/parcel/client.py

@@ -9,7 +9,6 @@
 from gdc_client.parcel import const
 from gdc_client.parcel import utils
 from gdc_client.parcel.download_stream import DownloadStream
-from gdc_client.parcel.portability import colored
 from gdc_client.parcel.portability import Process
 from gdc_client.parcel.segment import SegmentProducer
 

src/gdc_client/parcel/utils.py

@@ -7,7 +7,6 @@
 # ***************************************************************************************
 
 from contextlib import contextmanager
-from functools import partial
 import hashlib
 import logging
 import mmap
@@ -26,6 +25,7 @@
 
 from gdc_client.exceptions import MD5ValidationError
 from gdc_client.parcel.download_stream import DownloadStream
+from gdc_client.parcel import utils
 
 # Logging
 log = logging.getLogger("utils")
@@ -180,13 +180,13 @@ def calculate_segments(start, stop, block):
 
 
 def md5sum(block):
-    m = hashlib.md5()
+    m = utils.md5()
     m.update(block)
     return m.hexdigest()
 
 
 def md5sum_whole_file(fname):
-    hash_md5 = hashlib.md5()
+    hash_md5 = utils.md5()
 
     with open(fname, "rb") as f:
         for chunk in iter(lambda: f.read(4096), b""):
@@ -233,3 +233,8 @@ def mmap_open(path):
 
 def STRIP(comment):
     return " ".join(comment.split())
+
+
+def md5():
+    md5_kwargs = {} if sys.version_info < (3, 9) else {"usedforsecurity": False}
+    return hashlib.md5(**md5_kwargs)

src/gdc_client/utils.py

@@ -1,9 +1,4 @@
-try:
-    # Python3
-    from urllib.parse import urlencode
-except ImportError:
-    # Python2
-    from urllib import urlencode
+from urllib.parse import urlencode
 
 
 def build_url(path, *params, **kwparams):

tests/conftest.py

@@ -1,4 +1,3 @@
-import hashlib
 from io import BytesIO
 from multiprocessing import Process
 import tarfile
@@ -9,16 +8,17 @@
 from moto import mock_aws
 import pytest
 
+from gdc_client.parcel import utils
 from gdc_client.parcel.const import HTTP_CHUNK_SIZE
 
 
 def md5(iterable: Iterable):
-    md5 = hashlib.md5()
+    md5_fn = utils.md5()
 
     for chunk in iterable:
-        md5.update(chunk.encode("utf-8"))
+        md5_fn.update(chunk.encode("utf-8"))
 
-    return md5.hexdigest()
+    return md5_fn.hexdigest()
 
 
 def make_tarfile(

tests/test_download_client.py

@@ -11,7 +11,7 @@
 from gdc_client.parcel.const import HTTP_CHUNK_SIZE, SAVE_INTERVAL
 from gdc_client.parcel.download_stream import DownloadStream
 
-from conftest import make_tarfile, md5, uuids
+from conftest import make_tarfile, uuids
 from gdc_client.download.client import GDCHTTPDownloadClient, fix_url
 from gdc_client.download.parser import download
 from gdc_client.query.index import GDCIndexClient