Add S3 File and Folder Operations in S3FileHandler Class

torch_snippets/_modidx.py

@@ -413,6 +413,22 @@
                                                                                    'torch_snippets/registry.py'),
                                          'torch_snippets.registry.tryeval': ('registry.html#tryeval', 'torch_snippets/registry.py')},
             'torch_snippets.s3_loader': {},
+            'torch_snippets.s3_loader2': { 'torch_snippets.s3_loader2.S3FileHandler': ( 's3_loader2.html#s3filehandler',
+                                                                                        'torch_snippets/s3_loader2.py'),
+                                           'torch_snippets.s3_loader2.S3FileHandler.__init__': ( 's3_loader2.html#s3filehandler.__init__',
+                                                                                                 'torch_snippets/s3_loader2.py'),
+                                           'torch_snippets.s3_loader2.S3FileHandler.download_s3_file': ( 's3_loader2.html#s3filehandler.download_s3_file',
+                                                                                                         'torch_snippets/s3_loader2.py'),
+                                           'torch_snippets.s3_loader2.S3FileHandler.download_s3_folder': ( 's3_loader2.html#s3filehandler.download_s3_folder',
+                                                                                                           'torch_snippets/s3_loader2.py'),
+                                           'torch_snippets.s3_loader2.S3FileHandler.inmemory_download_s3': ( 's3_loader2.html#s3filehandler.inmemory_download_s3',
+                                                                                                             'torch_snippets/s3_loader2.py'),
+                                           'torch_snippets.s3_loader2.S3FileHandler.list_s3_buckets': ( 's3_loader2.html#s3filehandler.list_s3_buckets',
+                                                                                                        'torch_snippets/s3_loader2.py'),
+                                           'torch_snippets.s3_loader2.S3FileHandler.list_s3_objects': ( 's3_loader2.html#s3filehandler.list_s3_objects',
+                                                                                                        'torch_snippets/s3_loader2.py'),
+                                           'torch_snippets.s3_loader2.S3FileHandler.upload_file_to_s3': ( 's3_loader2.html#s3filehandler.upload_file_to_s3',
+                                                                                                          'torch_snippets/s3_loader2.py')},
             'torch_snippets.scp': {},
             'torch_snippets.sklegos': { 'torch_snippets.sklegos.Cat2Num': ('sklegos.html#cat2num', 'torch_snippets/sklegos.py'),
                                         'torch_snippets.sklegos.Cat2Num.__init__': ( 'sklegos.html#cat2num.__init__',

torch_snippets/adapters.py

@@ -1,17 +1,8 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/adapters.ipynb.
 
 # %% auto 0
-__all__ = [
-    "np_2_b64",
-    "b64_2_np",
-    "b64_2_file",
-    "bytes_2_file",
-    "file_2_bytes",
-    "csvs_2_cvat",
-    "cvat_2_csvs",
-    "df_2_yolo",
-    "yolo_2_df",
-]
+__all__ = ['np_2_b64', 'b64_2_np', 'b64_2_file', 'bytes_2_file', 'file_2_bytes', 'csvs_2_cvat', 'cvat_2_csvs', 'df_2_yolo',
+           'yolo_2_df']
 
 # %% ../nbs/adapters.ipynb 2
 import base64, cv2, numpy as np
@@ -20,7 +11,6 @@
 from .paths import P, stems, stem, parent, makedir
 from .markup import write_xml, read_xml, AttrDict
 
-
 # %% ../nbs/adapters.ipynb 3
 def np_2_b64(image: np.ndarray) -> str:
     """Convert a numpy image to base64 string"""
@@ -75,7 +65,6 @@ def file_2_bytes(fpath):
         output = f.read()
     return output
 
-
 # %% ../nbs/adapters.ipynb 4
 def _process(
     df: pd.DataFrame, label_column="readable_label", default_label="Background"
@@ -296,7 +285,6 @@ def cvat_2_csvs(xmlfile, csvs_folder):
         except Exception as e:
             Warn(f'{e} @ {item["@name"]}')
 
-
 # %% ../nbs/adapters.ipynb 5
 def df_2_yolo(df, h, w, class2id, class_column):
     yolo_data = []

torch_snippets/bb_utils.py

@@ -1,28 +1,9 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/bounding_boxes.ipynb.
 
 # %% auto 0
-__all__ = [
-    "randint",
-    "BB",
-    "df2bbs",
-    "bbs2df",
-    "bbfy",
-    "jitter",
-    "compute_eps",
-    "enlarge_bbs",
-    "shrink_bbs",
-    "iou",
-    "compute_distance_matrix",
-    "compute_distances",
-    "split_bb_to_xyXY",
-    "combine_xyXY_to_bb",
-    "is_absolute",
-    "is_relative",
-    "to_relative",
-    "to_absolute",
-    "merge_by_bb",
-    "isin",
-]
+__all__ = ['randint', 'BB', 'df2bbs', 'bbs2df', 'bbfy', 'jitter', 'compute_eps', 'enlarge_bbs', 'shrink_bbs', 'iou',
+           'compute_distance_matrix', 'compute_distances', 'split_bb_to_xyXY', 'combine_xyXY_to_bb', 'is_absolute',
+           'is_relative', 'to_relative', 'to_absolute', 'merge_by_bb', 'isin']
 
 # %% ../nbs/bounding_boxes.ipynb 2
 import numpy as np
@@ -168,7 +149,6 @@ def distances(self, other_bbs, threshold=None, direction=None):
             raise NotImplementedError("")
         return sorted(other_bbs, key=lambda obj: self.l2(obj[1]))
 
-
 # %% ../nbs/bounding_boxes.ipynb 8
 def df2bbs(df):
     """
@@ -303,7 +283,6 @@ def shrink_bbs(bbs, eps=0.2):
         for (x, y, X, Y), (h, w) in zip(bbs, shs)
     ]
 
-
 # %% ../nbs/bounding_boxes.ipynb 9
 def iou(bboxes1, bboxes2):
     """
@@ -384,7 +363,6 @@ def compute_distances(df1, df2, shrink_factors=(1, 1)):
     distances = compute_distance_matrix(bbs1, bbs2)
     return distances
 
-
 # %% ../nbs/bounding_boxes.ipynb 10
 def split_bb_to_xyXY(df):
     """
@@ -526,7 +504,6 @@ def to_absolute(df, height, width, force=False):
         df = combine_xyXY_to_bb(df)
     return df
 
-
 # %% ../nbs/bounding_boxes.ipynb 18
 def merge_by_bb(df1, df2, suffixes=("_x", "_y"), iou_threshold=0.1):
     """Merge df1 columns to df2 by using iou

torch_snippets/bokeh_loader.py

@@ -1,7 +1,9 @@
+"""For rapid prototyping bokeh charts"""
+
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/bokeh_plotting.ipynb.
 
 # %% auto 0
-__all__ = ["parse_sz", "get_bplot"]
+__all__ = ['parse_sz', 'get_bplot']
 
 # %% ../nbs/bokeh_plotting.ipynb 2
 from bokeh.io import output_notebook, show as bshow
@@ -23,7 +25,6 @@
 from bokeh.palettes import Spectral7
 import numpy as np
 
-
 # %% ../nbs/bokeh_plotting.ipynb 3
 def parse_sz(size):
     """

torch_snippets/charts.py

@@ -1,16 +1,7 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/charts.ipynb.
 
 # %% auto 0
-__all__ = [
-    "alt",
-    "Chart",
-    "CM",
-    "radar",
-    "confusion_matrix",
-    "spider",
-    "upsetaltair_top_level_configuration",
-    "UpSetAltair",
-]
+__all__ = ['alt', 'Chart', 'CM', 'radar', 'confusion_matrix', 'spider', 'upsetaltair_top_level_configuration', 'UpSetAltair']
 
 # %% ../nbs/charts.ipynb 2
 import altair as alt
@@ -22,7 +13,6 @@
 
 init_plt()
 
-
 # %% ../nbs/charts.ipynb 5
 def confusion_matrix(df=None, truth=None, pred=None, mapping=None, save_to=None):
     """
@@ -96,7 +86,6 @@ def confusion_matrix(df=None, truth=None, pred=None, mapping=None, save_to=None)
 
 CM = confusion_matrix
 
-
 # %% ../nbs/charts.ipynb 12
 def spider(
     df,
@@ -193,7 +182,6 @@ def spider(
 
 radar = spider
 
-
 # %% ../nbs/charts.ipynb 14
 # Top-level altair configuration
 def upsetaltair_top_level_configuration(

torch_snippets/decorators.py

@@ -1,7 +1,7 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/decorators.ipynb.
 
 # %% auto 0
-__all__ = ["format", "warn_on_fail", "timeit", "io", "check_kwargs_not_none"]
+__all__ = ['format', 'warn_on_fail', 'timeit', 'io', 'check_kwargs_not_none']
 
 # %% ../nbs/decorators.ipynb 2
 from functools import wraps
@@ -11,7 +11,6 @@
 from .markup import AD
 import time
 
-
 # %% ../nbs/decorators.ipynb 3
 def format(input):
     if isinstance(input, (list, tuple, set, L)):

torch_snippets/imgaug_loader.py

@@ -1,17 +1,7 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/imgaug_loader.ipynb.
 
 # %% auto 0
-__all__ = [
-    "do",
-    "bw",
-    "rotate",
-    "pad",
-    "get_size",
-    "rescale",
-    "crop",
-    "imgaugbbs2bbs",
-    "bbs2imgaugbbs",
-]
+__all__ = ['do', 'bw', 'rotate', 'pad', 'get_size', 'rescale', 'crop', 'imgaugbbs2bbs', 'bbs2imgaugbbs']
 
 # %% ../nbs/imgaug_loader.ipynb 2
 import imgaug.augmenters as iaa
@@ -24,7 +14,6 @@
     to_absolute,
 )
 
-
 # %% ../nbs/imgaug_loader.ipynb 3
 def do(img, bbs=None, aug=None, cval=255):
     """

torch_snippets/inspector.py

@@ -1,13 +1,14 @@
+"""View statistics of tensors and other python containers"""
+
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/inspector.ipynb.
 
 # %% auto 0
-__all__ = ["inspect"]
+__all__ = ['inspect']
 
 # %% ../nbs/inspector.ipynb 2
 from .loader import *
 from .registry import AttrDict
 
-
 # %% ../nbs/inspector.ipynb 3
 def inspect(*arrays, **kwargs):
     """

torch_snippets/interactive_show.py

@@ -1,18 +1,8 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/interactive_show.ipynb.
 
 # %% auto 0
-__all__ = [
-    "COLORS",
-    "to_networkx",
-    "plot_image",
-    "plot_graph",
-    "tonp",
-    "tolist",
-    "convert_to_nx",
-    "viz2",
-    "df2graph_nodes",
-    "ishow",
-]
+__all__ = ['COLORS', 'to_networkx', 'plot_image', 'plot_graph', 'tonp', 'tolist', 'convert_to_nx', 'viz2', 'df2graph_nodes',
+           'ishow']
 
 # %% ../nbs/interactive_show.ipynb 1
 from pathlib import Path
@@ -44,7 +34,6 @@
 
 output_notebook()
 
-
 # %% ../nbs/interactive_show.ipynb 2
 def to_networkx(
     data,

torch_snippets/ipython.py

@@ -1,22 +1,10 @@
+"""Utilities specific to jupyter"""
+
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/jupyter_notebook.ipynb.
 
 # %% auto 0
-__all__ = [
-    "save_notebook",
-    "backup_this_notebook",
-    "backup_all_notebooks",
-    "backup_folders_of_nbs",
-    "display_dfs_side_by_side",
-    "show_big_dataframe",
-    "h1",
-    "h2",
-    "h3",
-    "h4",
-    "h5",
-    "h6",
-    "store_scrap",
-    "shutdown_current_notebook",
-]
+__all__ = ['save_notebook', 'backup_this_notebook', 'backup_all_notebooks', 'backup_folders_of_nbs', 'display_dfs_side_by_side',
+           'show_big_dataframe', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'store_scrap', 'shutdown_current_notebook']
 
 # %% ../nbs/jupyter_notebook.ipynb 2
 import os, sys, json, time, hashlib
@@ -30,7 +18,6 @@
 # %% ../nbs/jupyter_notebook.ipynb 3
 # | export
 
-
 # %% ../nbs/jupyter_notebook.ipynb 4
 def save_notebook(file_path):
     from IPython.display import display, Javascript
@@ -124,7 +111,6 @@ def backup_folders_of_nbs(src, dest):
     for f in Glob(f"{dest}/*/changelog.md"):
         f.rm(confirm_prompt=False)
 
-
 # %% ../nbs/jupyter_notebook.ipynb 6
 def display_dfs_side_by_side(*args, titles=cycle([""]), max_rows=50):
     from IPython.display import display_html
@@ -151,7 +137,6 @@ def show_big_dataframe(df, max_rows=30):
     ):
         show(df, max_rows=max_rows, frame_count=2)
 
-
 # %% ../nbs/jupyter_notebook.ipynb 7
 def h1(text):
     from IPython.display import Markdown
@@ -188,7 +173,6 @@ def h6(text):
 
     show(Markdown(f"###### {text}"))
 
-
 # %% ../nbs/jupyter_notebook.ipynb 8
 @contextmanager
 def store_scrap(at):
@@ -203,7 +187,6 @@ def store_scrap(at):
     )
     return scrap
 
-
 # %% ../nbs/jupyter_notebook.ipynb 9
 # Function to shut down the current notebook session
 def shutdown_current_notebook(delay: int = None):

torch_snippets/load_defaults.py

@@ -1,7 +1,7 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/load_defautls.ipynb.
 
 # %% auto 0
-__all__ = ["ifnone", "exists", "loadifexists"]
+__all__ = ['ifnone', 'exists', 'loadifexists']
 
 # %% ../nbs/load_defautls.ipynb 2
 from .loader import os