enh: show correct upload speed if some resources were uploaded in a p…

…revious session
DCOR-dev · Aug 24, 2021 · 4a45ab6 · 4a45ab6
1 parent dc469b7
commit 4a45ab6
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 3 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,4 +1,6 @@
 0.6.2
+ - enh: show correct upload speed if some resources were uploaded in
+   a previeous session
  - fix: set job.state to "error" if the daemon encountered one
  - fix: catch ConnectionError in update_job_status
 0.6.1

diff --git a/dcoraid/upload/job.py b/dcoraid/upload/job.py
@@ -75,6 +75,7 @@ def __init__(self, api, dataset_id, resource_paths,
         self.resource_names = resource_names
         self.task_id = task_id
         self.paths_uploaded = []
+        self.paths_uploaded_before = []
         self.state = None
         self.set_state("init")
         self.traceback = None
@@ -209,7 +210,12 @@ def get_rate(self, resolution=3.05):
             Mean upload rate in bytes per second
         """
         cur_time = time.perf_counter()
-        cur_bytes = sum(self.file_bytes_uploaded)
+        # get bytes of files that have been uploaded
+        cur_bytes = 0
+        for ii, path in enumerate(self.paths):
+            if path not in self.paths_uploaded_before:
+                # only count files from the current session
+                cur_bytes += self.file_bytes_uploaded[ii]
 
         delta_time = cur_time - self._last_time
         delta_bytes = cur_bytes - self._last_bytes
@@ -232,7 +238,7 @@ def get_rate(self, resolution=3.05):
         else:
             # finished
             tdelt = (self.end_time - self.start_time)
-            rate = sum(self.file_bytes_uploaded) / tdelt
+            rate = cur_bytes / tdelt
         self._last_rate = rate
         return rate
 
@@ -367,8 +373,11 @@ def task_upload_resources(self):
                         # We are currently retrying an upload. If the
                         # resource exists, we have already uploaded it.
                         self.file_bytes_uploaded[ii] = self.file_sizes[ii]
+                        self.paths_uploaded.append(path)
+                        self.paths_uploaded_before.append(path)
                         continue
-                    if not exists:
+                    else:
+                        # Normal upload.
                         try:
                             dataset.add_resource(
                                 dataset_id=self.dataset_id,