Improved code formatting after using flake8

commons.py

@@ -48,26 +48,27 @@ def setup_logging(save_dir, console="debug",
     logger = logging.getLogger('')
     logger.setLevel(logging.DEBUG)
 
-    if info_filename != None:
+    if info_filename is not None:
         info_file_handler = logging.FileHandler(join(save_dir, info_filename))
         info_file_handler.setLevel(logging.INFO)
         info_file_handler.setFormatter(base_formatter)
         logger.addHandler(info_file_handler)
 
-    if debug_filename != None:
+    if debug_filename is not None:
         debug_file_handler = logging.FileHandler(join(save_dir, debug_filename))
         debug_file_handler.setLevel(logging.DEBUG)
         debug_file_handler.setFormatter(base_formatter)
         logger.addHandler(debug_file_handler)
 
-    if console != None:
+    if console is not None:
         console_handler = logging.StreamHandler()
-        if console == "debug": console_handler.setLevel(logging.DEBUG)
-        if console == "info":  console_handler.setLevel(logging.INFO)
+        if console == "debug":
+            console_handler.setLevel(logging.DEBUG)
+        if console == "info":
+            console_handler.setLevel(logging.INFO)
         console_handler.setFormatter(base_formatter)
         logger.addHandler(console_handler)
 
     def exception_handler(type_, value, tb):
         logger.info("\n" + "".join(traceback.format_exception(type, value, tb)))
     sys.excepthook = exception_handler
-

datasets_ws.py

@@ -9,15 +9,15 @@
 from PIL import Image
 from os.path import join
 import torch.utils.data as data
-import torchvision.transforms as transforms
+import torchvision.transforms as T
 from torch.utils.data.dataset import Subset
 from sklearn.neighbors import NearestNeighbors
 from torch.utils.data.dataloader import DataLoader
 
 
-base_transform = transforms.Compose([
-    transforms.ToTensor(),
-    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+base_transform = T.Compose([
+    T.ToTensor(),
+    T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
 ])
 
 
@@ -26,7 +26,7 @@ def path_to_pil_img(path):
 
 
 def collate_fn(batch):
-    """Creates mini-batch tensors from the list of tuples (images, 
+    """Creates mini-batch tensors from the list of tuples (images,
         triplets_local_indexes, triplets_global_indexes).
         triplets_local_indexes are the indexes referring to each triplet within images.
         triplets_global_indexes are the global indexes of each image.
@@ -41,8 +41,8 @@ def collate_fn(batch):
         triplets_local_indexes: torch tensor of shape (batch_size*10, 3).
         triplets_global_indexes: torch tensor of shape (batch_size, 12).
     """
-    images                  = torch.cat([e[0] for e in batch])
-    triplets_local_indexes  = torch.cat([e[1][None] for e in batch])
+    images = torch.cat([e[0] for e in batch])
+    triplets_local_indexes = torch.cat([e[1][None] for e in batch])
     triplets_global_indexes = torch.cat([e[2][None] for e in batch])
     for i, (local_indexes, global_indexes) in enumerate(zip(triplets_local_indexes, triplets_global_indexes)):
         local_indexes += len(global_indexes) * i  # Increment local indexes by offset (len(global_indexes) is 12)
@@ -52,11 +52,13 @@ def collate_fn(batch):
 class PCADataset(data.Dataset):
     def __init__(self, args, datasets_folder="dataset", dataset_folder="pitts30k/images/train"):
         dataset_folder_full_path = join(datasets_folder, dataset_folder)
-        if not os.path.exists(dataset_folder_full_path) :
+        if not os.path.exists(dataset_folder_full_path):
             raise FileNotFoundError(f"Folder {dataset_folder_full_path} does not exist")
         self.images_paths = sorted(glob(join(dataset_folder_full_path, "**", "*.jpg"), recursive=True))
+
     def __getitem__(self, index):
         return base_transform(path_to_pil_img(self.images_paths[index]))
+
     def __len__(self):
         return len(self.images_paths)
 
@@ -69,41 +71,44 @@ def __init__(self, args, datasets_folder="datasets", dataset_name="pitts30k", sp
         self.args = args
         self.dataset_name = dataset_name
         self.dataset_folder = join(datasets_folder, dataset_name, "images", split)
-        if not os.path.exists(self.dataset_folder): raise FileNotFoundError(f"Folder {self.dataset_folder} does not exist")
+        if not os.path.exists(self.dataset_folder):
+            raise FileNotFoundError(f"Folder {self.dataset_folder} does not exist")
 
         self.resize = args.resize
         self.test_method = args.test_method
 
         #### Read paths and UTM coordinates for all images.
         database_folder = join(self.dataset_folder, "database")
-        queries_folder  = join(self.dataset_folder, "queries")
-        if not os.path.exists(database_folder): raise FileNotFoundError(f"Folder {database_folder} does not exist")
-        if not os.path.exists(queries_folder) : raise FileNotFoundError(f"Folder {queries_folder} does not exist")
+        queries_folder = join(self.dataset_folder, "queries")
+        if not os.path.exists(database_folder):
+            raise FileNotFoundError(f"Folder {database_folder} does not exist")
+        if not os.path.exists(queries_folder):
+            raise FileNotFoundError(f"Folder {queries_folder} does not exist")
         self.database_paths = sorted(glob(join(database_folder, "**", "*.jpg"), recursive=True))
-        self.queries_paths  = sorted(glob(join(queries_folder, "**", "*.jpg"),  recursive=True))
+        self.queries_paths = sorted(glob(join(queries_folder, "**", "*.jpg"),  recursive=True))
         # The format must be path/to/file/@utm_easting@utm_northing@...@.jpg
-        self.database_utms = np.array([(path.split("@")[1], path.split("@")[2]) for path in self.database_paths]).astype(np.float)
-        self.queries_utms  = np.array([(path.split("@")[1], path.split("@")[2]) for path in self.queries_paths]).astype(np.float)
+        self.database_utms = np.array([(path.split("@")[1], path.split("@")[2]) for path in self.database_paths]).astype(float)
+        self.queries_utms = np.array([(path.split("@")[1], path.split("@")[2]) for path in self.queries_paths]).astype(float)
 
         # Find soft_positives_per_query, which are within val_positive_dist_threshold (deafult 25 meters)
         knn = NearestNeighbors(n_jobs=-1)
         knn.fit(self.database_utms)
-        self.soft_positives_per_query = knn.radius_neighbors(self.queries_utms, 
+        self.soft_positives_per_query = knn.radius_neighbors(self.queries_utms,
                                                              radius=args.val_positive_dist_threshold,
                                                              return_distance=False)
 
         self.images_paths = list(self.database_paths) + list(self.queries_paths)
 
         self.database_num = len(self.database_paths)
-        self.queries_num  = len(self.queries_paths)
+        self.queries_num = len(self.queries_paths)
 
     def __getitem__(self, index):
         img = path_to_pil_img(self.images_paths[index])
         img = base_transform(img)
         # With database images self.test_method should always be "hard_resize"
         if self.test_method == "hard_resize":
             # self.test_method=="hard_resize" is the default, resizes all images to the same size.
-            img = transforms.functional.resize(img, self.resize)
+            img = T.functional.resize(img, self.resize)
         else:
             img = self._test_query_transform(img)
         return img, index
@@ -113,32 +118,34 @@ def _test_query_transform(self, img):
         C, H, W = img.shape
         if self.test_method == "single_query":
             # self.test_method=="single_query" is used when queries have varying sizes, and can't be stacked in a batch.
-            processed_img = transforms.functional.resize(img, min(self.resize))
+            processed_img = T.functional.resize(img, min(self.resize))
         elif self.test_method == "central_crop":
             # Take the biggest central crop of size self.resize. Preserves ratio.
             scale = max(self.resize[0]/H, self.resize[1]/W)
             processed_img = torch.nn.functional.interpolate(img.unsqueeze(0), scale_factor=scale).squeeze(0)
-            processed_img = transforms.functional.center_crop(processed_img, self.resize)
+            processed_img = T.functional.center_crop(processed_img, self.resize)
             assert processed_img.shape[1:] == torch.Size(self.resize), f"{processed_img.shape[1:]} {self.resize}"
         elif self.test_method == "five_crops" or self.test_method == 'nearest_crop' or self.test_method == 'maj_voting':
             # Get 5 square crops with size==shorter_side (usually 480). Preserves ratio and allows batches.
             shorter_side = min(self.resize)
-            processed_img = transforms.functional.resize(img, shorter_side)
-            processed_img = torch.stack(transforms.functional.five_crop(processed_img, shorter_side))
+            processed_img = T.functional.resize(img, shorter_side)
+            processed_img = torch.stack(T.functional.five_crop(processed_img, shorter_side))
             assert processed_img.shape == torch.Size([5, 3, shorter_side, shorter_side]), \
                 f"{processed_img.shape} {torch.Size([5, 3, shorter_side, shorter_side])}"
         return processed_img
 
     def __len__(self):
         return len(self.images_paths)
+
     def __repr__(self):
-        return  (f"< {self.__class__.__name__}, {self.dataset_name} - #database: {self.database_num}; #queries: {self.queries_num} >")
+        return f"< {self.__class__.__name__}, {self.dataset_name} - #database: {self.database_num}; #queries: {self.queries_num} >"
+
     def get_positives(self):
         return self.soft_positives_per_query
 
 
 class TripletsDataset(BaseDataset):
-    """Dataset used for training, it is used to compute the triplets 
+    """Dataset used for training, it is used to compute the triplets
     with TripletsDataset.compute_triplets() with various mining methods.
     If is_inference == True, uses methods of the parent class BaseDataset,
     this is used for example when computing the cache, because we compute features
@@ -153,21 +160,17 @@ def __init__(self, args, datasets_folder="datasets", dataset_name="pitts30k", sp
             self.neg_cache = [np.empty((0,), dtype=np.int32) for _ in range(self.queries_num)]
         self.is_inference = False
 
-        identity_transform = transforms.Lambda(lambda x: x)
-        self.resized_transform = transforms.Compose([
-            transforms.Resize(self.resize) if self.resize is not None else identity_transform,
+        identity_transform = T.Lambda(lambda x: x)
+        self.resized_transform = T.Compose([
+            T.Resize(self.resize) if self.resize is not None else identity_transform,
             base_transform
         ])
 
-        self.query_transform = transforms.Compose([
-                transforms.ColorJitter(brightness=args.brightness)       if args.brightness          != None else identity_transform,
-                transforms.ColorJitter(contrast=args.contrast)           if args.contrast            != None else identity_transform,
-                transforms.ColorJitter(saturation=args.saturation)       if args.saturation          != None else identity_transform,
-                transforms.ColorJitter(hue=args.hue)                     if args.hue                 != None else identity_transform,
-                transforms.RandomPerspective(args.rand_perspective)      if args.rand_perspective    != None else identity_transform,
-                transforms.RandomResizedCrop(size=self.resize, scale=(1-args.random_resized_crop, 1))  \
-                                                                         if args.random_resized_crop != None else identity_transform,
-                transforms.RandomRotation(degrees=args.random_rotation)  if args.random_rotation     != None else identity_transform,
+        self.query_transform = T.Compose([
+                T.ColorJitter(args.brightness, args.contrast, args.saturation, args.hue),
+                T.RandomPerspective(args.rand_perspective),
+                T.RandomResizedCrop(size=self.resize, scale=(1-args.random_resized_crop, 1)),
+                T.RandomRotation(degrees=args.random_rotation),
                 self.resized_transform,
         ])
 
@@ -185,7 +188,7 @@ def __init__(self, args, datasets_folder="datasets", dataset_name="pitts30k", sp
                          "within the training set. They won't be considered as they're useless for training.")
         # Remove queries without positives
         self.hard_positives_per_query = np.delete(self.hard_positives_per_query, queries_without_any_hard_positive)
-        self.queries_paths            = np.delete(self.queries_paths,            queries_without_any_hard_positive)
+        self.queries_paths = np.delete(self.queries_paths, queries_without_any_hard_positive)
 
         # Recompute images_paths and queries_num because some queries might have been removed
         self.images_paths = list(self.database_paths) + list(self.queries_paths)
@@ -214,14 +217,14 @@ def __getitem__(self, index):
         if self.is_inference:
             # At inference time return the single image. This is used for caching or computing NetVLAD's clusters
             return super().__getitem__(index)
-        query_index, best_positive_index, neg_indexes = torch.split(self.triplets_global_indexes[index], (1,1,self.negs_num_per_query))
-        query     =  self.query_transform(path_to_pil_img(self.queries_paths[query_index]))
-        positive  =  self.resized_transform(path_to_pil_img(self.database_paths[best_positive_index]))
+        query_index, best_positive_index, neg_indexes = torch.split(self.triplets_global_indexes[index], (1, 1, self.negs_num_per_query))
+        query = self.query_transform(path_to_pil_img(self.queries_paths[query_index]))
+        positive = self.resized_transform(path_to_pil_img(self.database_paths[best_positive_index]))
         negatives = [self.resized_transform(path_to_pil_img(self.database_paths[i])) for i in neg_indexes]
         images = torch.stack((query, positive, *negatives), 0)
-        triplets_local_indexes = torch.empty((0,3), dtype=torch.int)
+        triplets_local_indexes = torch.empty((0, 3), dtype=torch.int)
         for neg_num in range(len(neg_indexes)):
-            triplets_local_indexes = torch.cat((triplets_local_indexes, torch.tensor([0,1,2+neg_num]).reshape(1,3)))
+            triplets_local_indexes = torch.cat((triplets_local_indexes, torch.tensor([0, 1, 2 + neg_num]).reshape(1, 3)))
         return images, triplets_local_indexes, self.triplets_global_indexes[index]
 
     def __len__(self):
@@ -244,9 +247,9 @@ def compute_triplets(self, args, model):
     def compute_cache(args, model, subset_ds, cache_shape):
         """Compute the cache containing features of images, which is used to
         find best positive and hardest negatives."""
-        subset_dl = DataLoader(dataset=subset_ds, num_workers=args.num_workers, 
+        subset_dl = DataLoader(dataset=subset_ds, num_workers=args.num_workers,
                                batch_size=args.infer_batch_size, shuffle=False,
-                               pin_memory=(args.device=="cuda"))
+                               pin_memory=(args.device == "cuda"))
         model = model.eval()
 
         # RAMEfficient2DMatrix can be replaced by np.zeros, but using
@@ -286,7 +289,6 @@ def get_hardest_negatives_indexes(self, args, cache, query_features, neg_samples
         neg_indexes = neg_samples[neg_nums].astype(np.int32)
         return neg_indexes
 
-
     def compute_triplets_random(self, args, model):
         self.triplets_global_indexes = []
         # Take 1000 random queries
@@ -314,7 +316,6 @@ def compute_triplets_random(self, args, model):
         # self.triplets_global_indexes is a tensor of shape [1000, 12]
         self.triplets_global_indexes = torch.tensor(self.triplets_global_indexes)
 
-
     def compute_triplets_full(self, args, model):
         self.triplets_global_indexes = []
         # Take 1000 random queries
@@ -344,7 +345,6 @@ def compute_triplets_full(self, args, model):
         # self.triplets_global_indexes is a tensor of shape [1000, 12]
         self.triplets_global_indexes = torch.tensor(self.triplets_global_indexes)
 
-
     def compute_triplets_partial(self, args, model):
         self.triplets_global_indexes = []
         # Take 1000 random queries
@@ -391,13 +391,14 @@ def __init__(self, shape, dtype=np.float32):
         self.shape = shape
         self.dtype = dtype
         self.matrix = [None] * shape[0]
+
     def __setitem__(self, indexes, vals):
         assert vals.shape[1] == self.shape[1], f"{vals.shape[1]} {self.shape[1]}"
         for i, val in zip(indexes, vals):
             self.matrix[i] = val.astype(self.dtype, copy=False)
+
     def __getitem__(self, index):
         if hasattr(index, "__len__"):
             return np.array([self.matrix[i] for i in index])
         else:
             return self.matrix[index]
-

eval.py

@@ -2,7 +2,7 @@
 """
 With this script you can evaluate checkpoints or test models from two popular
 landmark retrieval github repos.
-The first is https://github.com/naver/deep-image-retrieval from Naver labs, 
+The first is https://github.com/naver/deep-image-retrieval from Naver labs,
 provides ResNet-50 and ResNet-101 trained with AP on Google Landmarks 18 clean.
 $ python eval.py --off_the_shelf=naver --l2=none --backbone=resnet101conv5 --aggregation=gem --fc_output_dim=2048
 
@@ -107,4 +107,3 @@
 logging.info(f"Recalls on {test_ds}: {recalls_str}")
 
 logging.info(f"Finished in {str(datetime.now() - start_time)[:-7]}")
-