Fix pickling error when jobstate file doesnt exist and fix threading …

…error when lock file exists then disappears (#4575)

Co-authored-by: Brandon Walker <walkerbd@dali1.dali.hpc.ncats.nih.gov>
Co-authored-by: Adam Novak <anovak@soe.ucsc.edu>

src/toil/fileStores/nonCachingFileStore.py

@@ -303,6 +303,10 @@ def _getAllJobStates(cls, coordination_dir: str) -> Iterator[Dict[str, str]]:
                     # This is a FileNotFoundError.
                     # job finished & deleted its jobState file since the jobState files were discovered
                     continue
+                elif e.errno == 5:
+                    # This is a OSError: [Errno 5] Input/output error (jobStatefile seems to disappear 
+                    # on network file system sometimes)
+                    continue
                 else:
                     raise
 

src/toil/lib/threading.py

@@ -533,7 +533,12 @@ def leave(self) -> Iterator[bool]:
                 # There is someone claiming to be here. Are they alive?
                 full_path = os.path.join(self.lockfileDir, item)
 
-                fd = os.open(full_path, os.O_RDONLY)
+                try:
+                    fd = os.open(full_path, os.O_RDONLY)
+                except OSError as e:
+                    # suddenly file doesnt exist on network file system?
+                    continue
+
                 try:
                     fcntl.lockf(fd, fcntl.LOCK_SH | fcntl.LOCK_NB)
                 except OSError as e: