fix(drive): ensure modified date matches after upload (#4)

* fix(drive): ensure modified date matches after upload Sometimes, the modified date of a newly uploaded file wouldn't match the local file's modified date. Added a check to ensure that the modified date matches before returning. Also added logging to a local file.
P403n1x87 · May 17, 2020 · f97ec1c · f97ec1c
1 parent 87340af
commit f97ec1c
Show file tree

Hide file tree

Showing 13 changed files with 381 additions and 50 deletions.
diff --git a/README.md b/README.md
@@ -16,8 +16,8 @@ He's both useless and useful, until you try him for the first time :).</p>
     <img src="https://travis-ci.com/P403n1x87/erwin.svg?token=fzW2yzQyjwys4tWf9anS&branch=master"
          alt="Travis CI Build Status">
   </a>
-  <img src="https://img.shields.io/badge/version-0.2.0--beta-blue.svg"
-       alt="Version 0.2.0-beta">
+  <img src="https://img.shields.io/badge/version-0.2.1--beta-blue.svg"
+       alt="Version 0.2.1-beta">
   <a href="https://github.com/P403n1x87/erwin/blob/master/LICENSE.md">
     <img src="https://img.shields.io/badge/license-GPLv3-ff69b4.svg"
          alt="LICENSE">

diff --git a/erwin/__init__.py b/erwin/__init__.py
@@ -0,0 +1,24 @@
+# This file is part of "erwin" which is released under GPL.
+#
+# See file LICENCE or go to http://www.gnu.org/licenses/ for full license
+# details.
+#
+# Erwin is a cloud storage synchronisation service.
+#
+# Copyright (c) 2020 Gabriele N. Tornetta <phoenix1987@gmail.com>.
+# All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+APP_NAME = "erwin"
+AUTHOR = "p403n1x87"
diff --git a/erwin/__main__.py b/erwin/__main__.py
@@ -1,3 +1,25 @@
+# This file is part of "erwin" which is released under GPL.
+#
+# See file LICENCE or go to http://www.gnu.org/licenses/ for full license
+# details.
+#
+# Erwin is a cloud storage synchronisation service.
+#
+# Copyright (c) 2020 Gabriele N. Tornetta <phoenix1987@gmail.com>.
+# All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 from queue import Queue
 from shutil import copyfile
 import threading

diff --git a/erwin/config.py b/erwin/config.py
@@ -1,13 +1,36 @@
+# This file is part of "erwin" which is released under GPL.
+#
+# See file LICENCE or go to http://www.gnu.org/licenses/ for full license
+# details.
+#
+# Erwin is a cloud storage synchronisation service.
+#
+# Copyright (c) 2020 Gabriele N. Tornetta <phoenix1987@gmail.com>.
+# All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 from appdirs import user_config_dir
+import os
 import os.path
 import signal
 import yaml
 
+from erwin import APP_NAME
 from erwin.fs import FSNotReady, State
 from erwin.logging import LOGGER
 
 
-APP_NAME = "erwin"
 CONFIG_DIR = user_config_dir(APP_NAME)
 CONFIG_FILE = os.path.join(CONFIG_DIR, "config.yml")
 

diff --git a/erwin/flow.py b/erwin/flow.py
@@ -1,3 +1,25 @@
+# This file is part of "erwin" which is released under GPL.
+#
+# See file LICENCE or go to http://www.gnu.org/licenses/ for full license
+# details.
+#
+# Erwin is a cloud storage synchronisation service.
+#
+# Copyright (c) 2020 Gabriele N. Tornetta <phoenix1987@gmail.com>.
+# All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 from threading import RLock
 from time import sleep
 

diff --git a/erwin/fs/__init__.py b/erwin/fs/__init__.py
@@ -1,3 +1,25 @@
+# This file is part of "erwin" which is released under GPL.
+#
+# See file LICENCE or go to http://www.gnu.org/licenses/ for full license
+# details.
+#
+# Erwin is a cloud storage synchronisation service.
+#
+# Copyright (c) 2020 Gabriele N. Tornetta <phoenix1987@gmail.com>.
+# All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 from abc import ABC, abstractmethod
 from collections import defaultdict
 from copy import deepcopy
@@ -16,20 +38,20 @@ def wait(source_file, dest_fs, dst):
     while True:
         dest_file = dest_fs.search(dst)
         if source_file & dest_file:
-            break
+            LOGGER.debug(f"Destination file {dst} on {dest_fs} became available.")
+            return dest_file
         sleep(0.001)
-    return dest_file
+        LOGGER.trace(f"File mismatch: expected {source_file} but got {dest_file}")
 
 
 def wait_dir(dest_fs, dst):
     LOGGER.debug(f"Waiting for destination directory {dst} on {dest_fs}.")
     while True:
         dest_file = dest_fs.search(dst)
         if dest_file:
-            break
+            LOGGER.debug(f"Destination directory {dst} on {dest_fs} became available.")
+            return dest_file
         sleep(0.001)
-    LOGGER.debug(f"Destination directory {dst} on {dest_fs} became available.")
-    return dest_file
 
 
 def wait_removed(dest_fs, path):

diff --git a/erwin/fs/drive.py b/erwin/fs/drive.py
@@ -1,3 +1,25 @@
+# This file is part of "erwin" which is released under GPL.
+#
+# See file LICENCE or go to http://www.gnu.org/licenses/ for full license
+# details.
+#
+# Erwin is a cloud storage synchronisation service.
+#
+# Copyright (c) 2020 Gabriele N. Tornetta <phoenix1987@gmail.com>.
+# All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 from collections import defaultdict
 import datetime
 from httplib2 import Http, ServerNotFoundError
@@ -45,6 +67,10 @@
 ]
 
 
+class UnknownParent(Exception):
+    pass
+
+
 def suppresserror(f):
     def wrapper(*args, **kwargs):
         try:
@@ -222,8 +248,11 @@ def _get_paths(self, file, partial_path="", path_list=None):
             path_list.append(partial_path)
             return path_list
 
-        for parent in parents:
-            self._get_paths(self._file_map[parent], partial_path, path_list)
+        try:
+            for parent in parents:
+                self._get_paths(self._file_map[parent], partial_path, path_list)
+        except KeyError as e:
+            raise UnknownParent() from e
 
         return path_list
 
@@ -248,7 +277,6 @@ def get_children(file):
 
         return {"file": parent, "children": get_children(parent)}
 
-    # @suppresserror
     def _get_changes(self):
         start_token = self._changes_token or (
             self._drive.changes()
@@ -270,7 +298,9 @@ def _get_changes(self):
         moved = []
         removed = []
 
-        for change in changes:
+        while changes:
+            change = changes.pop(0)
+
             file_id = change["fileId"]
             if not file_id or "file" not in change:
                 continue
@@ -282,43 +312,55 @@ def _get_changes(self):
 
             new_file = self._to_file(dfile) if not dfile["trashed"] else None
 
-            old_dfile = self._file_map.get(file_id, None)
-            if old_dfile:
-                old_file = self._to_file(old_dfile)
-                if new_file:
-                    if old_file.parents == new_file.parents and old_file == new_file:
-                        continue
-
-                    if old_file.parents != new_file.parents and (
-                        old_file == new_file
-                        or old_file.is_folder
-                        and new_file.is_folder
-                    ):
-                        src, dst = self._path(old_dfile), self._path(dfile)
-                        moved.append((src, dst))
-                        self.state.move(src, dst)
-                    elif old_file.parents == new_file.parents and old_file != new_file:
-                        path = self._path(dfile)
-                        added.append((new_file, path))
-                        self.state.add(new_file, path)
+            try:
+                old_dfile = self._file_map.get(file_id, None)
+                if old_dfile:
+                    old_file = self._to_file(old_dfile)
+                    if new_file:
+                        if (
+                            old_file.parents == new_file.parents
+                            and old_file == new_file
+                        ):
+                            continue
+
+                        if old_file.parents != new_file.parents and (
+                            old_file == new_file
+                            or old_file.is_folder
+                            and new_file.is_folder
+                        ):
+                            src, dst = self._path(old_dfile), self._path(dfile)
+                            moved.append((src, dst))
+                            self.state.move(src, dst)
+                        elif (
+                            old_file.parents == new_file.parents
+                            and old_file != new_file
+                        ):
+                            path = self._path(dfile)
+                            added.append((new_file, path))
+                            self.state.add(new_file, path)
+                        else:
+                            old_path = self._path(old_dfile)
+                            new_path = self._path(dfile)
+                            removed.append(old_path)
+                            added.append((new_file, new_path))
                     else:
-                        old_path = self._path(old_dfile)
-                        new_path = self._path(dfile)
-                        removed.append(old_path)
-                        added.append((new_file, new_path))
-                else:
-                    path = self._path(old_dfile)
-                    removed.append(path)
-                    self.state.remove(path)
-                    del self._file_map[file_id]
+                        path = self._path(old_dfile)
+                        removed.append(path)
+                        self.state.remove(path)
+                        del self._file_map[file_id]
+
+                elif new_file:
+                    path = self._path(dfile)
+                    added.append((new_file, path))
+                    self.state.add(new_file, path)
 
-            elif new_file:
-                path = self._path(dfile)
-                added.append((new_file, path))
-                self.state.add(new_file, path)
+                if not dfile["trashed"]:
+                    self._file_map[file_id] = dfile
 
-            if not dfile["trashed"]:
-                self._file_map[file_id] = dfile
+            except UnknownParent:
+                # Changes are not received in the "right" order so we try
+                # to deal with the current change again later on.
+                changes.append(change)
 
         return Delta(added, moved, removed)
 
@@ -517,7 +559,8 @@ def write(self, stream, path, modified_date):
             folder, name = os.path.split(path)
             parent = self.search(folder)
             if not parent:
-                raise RuntimeError("Destination folder does not exist.")
+                self.makedirs(folder)
+                parent = self.search(folder)
 
             new_file = self._to_file(
                 self._drive.files()
@@ -538,6 +581,23 @@ def write(self, stream, path, modified_date):
                 .execute()
             )
 
+        # Ensure that modified date matches
+        while new_file.modified_date != modified_date:
+            sleep(0.001)
+            new_file = self._to_file(
+                self._drive.files()
+                .update(
+                    fileId=new_file._id,
+                    body={
+                        "modifiedTime": datetime.datetime.strftime(
+                            modified_date, "%Y-%m-%dT%H:%M:%S.%fZ"
+                        )
+                    },
+                    fields=GoogleDriveFS.FILE_FIELDS,
+                )
+                .execute()
+            )
+
         with STATE_LOCK:
             self.state.add(new_file, path)
 
@@ -584,13 +644,19 @@ def move(self, src, dst):
         head, tail = os.path.split(dst)
         dst_dir = self.search(head)
         if not dst_dir:
-            raise RuntimeError("Destination folder does not exist.")
+            self.makedirs(head)
+            dst_dir = self.search(head)
 
         dest_file = self._to_file(
             self._drive.files()
             .update(
                 fileId=file._id,
-                body={"name": tail},
+                body={
+                    "name": tail,
+                    "modifiedTime": datetime.datetime.strftime(
+                        file.modified_date, "%Y-%m-%dT%H:%M:%S.%fZ"
+                    ),
+                },
                 addParents=dst_dir._id,
                 removeParents=",".join(file.parents),
                 fields=GoogleDriveFS.FILE_FIELDS,