diff --git a/mcserver/tasks.py b/mcserver/tasks.py
index 20b804a..6f0391d 100644
--- a/mcserver/tasks.py
+++ b/mcserver/tasks.py
@@ -225,8 +225,15 @@ def os_remove_with_retry(path, max_retries=5, backoff=0.1):
         try:
             os.remove(path)
             return
-        
-        except (FileNotFoundError, PermissionError, OSError) as e:
+            
+        except FileNotFoundError as e:
+            # In case we've made it here, we've made a best effort to delete
+            # and it's likely gone.
+            if attempt == max_retries - 1:
+                return
+            time.sleep(backoff * (2**attempt))
+
+        except (PermissionError, OSError) as e:
             if attempt == max_retries - 1:
                 raise
             time.sleep(backoff * (2**attempt))