Expressive handler for fsspec block_size ValueError

ci/py3.7.yml

@@ -10,7 +10,7 @@ dependencies:
   - codecov
   - dask
   - distributed
-  - fsspec>=2021.04.0
+  - fsspec>=2021.6.0
   - h5netcdf
   - h5py
   - hdf5

ci/py3.8.yml

@@ -10,7 +10,7 @@ dependencies:
   - codecov
   - dask
   - distributed
-  - fsspec>=2021.04.0
+  - fsspec>=2021.6.0
   - h5netcdf
   - h5py
   - hdf5

ci/py3.9.yml

@@ -10,7 +10,7 @@ dependencies:
   - codecov
   - dask
   - distributed
-  - fsspec>=2021.04.0
+  - fsspec>=2021.6.0
   - h5netcdf
   - h5py
   - hdf5

pangeo_forge_recipes/storage.py

@@ -11,6 +11,7 @@
 from typing import Any, Iterator, Optional, Sequence, Union
 
 import fsspec
+from fsspec.implementations.http import BlockSizeError
 
 logger = logging.getLogger(__name__)
 
@@ -29,7 +30,13 @@ def _copy_btw_filesystems(input_opener, output_opener, BLOCK_SIZE=10_000_000):
         with output_opener as target:
             while True:
                 logger.debug("_copy_btw_filesystems reading data")
-                data = source.read(BLOCK_SIZE)
+                try:
+                    data = source.read(BLOCK_SIZE)
+                except BlockSizeError as e:
+                    raise ValueError(
+                        "Server does not permit random access to this file via Range requests. "
+                        'Try re-instantiating recipe with `fsspec_open_kwargs={"block_size": 0}`'
+                    ) from e
                 if not data:
                     break
                 logger.debug(f"_copy_btw_filesystems copying block of {len(data)} bytes")