fix(datasets): move storage to gcs

Author: AmitMY

setup.py

@@ -11,7 +11,7 @@
 setup(
     name="sign-language-datasets",
     packages=packages,
-    version="0.1.8",
+    version="0.2.0",
     description="TFDS Datasets for sign language",
     author="Amit Moryossef",
     author_email="amitmoryossef@gmail.com",

sign_language_datasets/datasets/autsl/autsl.py

@@ -12,7 +12,7 @@
 from tqdm import tqdm
 
 from ..warning import dataset_warning
-from ...datasets.config import SignDatasetConfig
+from ...datasets.config import SignDatasetConfig, cloud_bucket_file
 from ...utils.features import PoseFeature
 
 _DESCRIPTION = """
@@ -44,16 +44,16 @@
 _TRAIN_LABELS = "http://158.109.8.102/AuTSL/data/train/train_labels.csv"
 
 _VALID_VIDEOS = "http://158.109.8.102/AuTSL/data/validation/val_set_bjhfy68.zip"  # 3 files
-_VALID_LABELS = "https://nlp.biu.ac.il/~amit/datasets/public/autsl_validation_labels.csv"
+_VALID_LABELS = cloud_bucket_file("public/autsl_validation_labels.csv")
 
 _TEST_VIDEOS = "http://158.109.8.102/AuTSL/data/test/test_set_xsaft57.zip"  # 3 files
-_TEST_LABELS = "https://nlp.biu.ac.il/~amit/datasets/public/autsl_test_labels.csv"
+_TEST_LABELS = cloud_bucket_file("public/autsl_test_labels.csv")
 
 _CLASSES = "https://data.chalearnlap.cvc.uab.cat/AuTSL/data/SignList_ClassId_TR_EN.csv"
 
 _POSE_URLS = {
-    "holistic": "https://nlp.biu.ac.il/~amit/datasets/poses/holistic/autsl.tar.gz",
-    "openpose": "https://nlp.biu.ac.il/~amit/datasets/poses/openpose/autsl.tar.gz",
+    "holistic": cloud_bucket_file("poses/holistic/autsl.tar.gz"),
+    "openpose": cloud_bucket_file("poses/openpose/autsl.tar.gz")
 }
 _POSE_HEADERS = {
     "holistic": path.join(path.dirname(path.realpath(__file__)), "holistic.poseheader"),

sign_language_datasets/datasets/autsl/checksums.tsv

@@ -24,7 +24,3 @@ http://158.109.8.102/AuTSL/data/train/train_set_vfbha39.zip.018   	688988403	4cc
 http://158.109.8.102/AuTSL/data/validation/val_set_bjhfy68.zip.001	1048576000	007a96f8e15055e370adaa3b7e8ca9e3b76abd269f677bc33808fc739401ce49	val_set_bjhfy68.zip.001
 http://158.109.8.102/AuTSL/data/validation/val_set_bjhfy68.zip.002	1048576000	c138b73f87f952008fe21b18466bc924d539532fd348c9a7e775761c27294591	val_set_bjhfy68.zip.002
 http://158.109.8.102/AuTSL/data/validation/val_set_bjhfy68.zip.003	1028644777	ed1549f54ddc5b4cf71d89743888daacde11f28616e52d9b5133bccf2f9fd0b1	val_set_bjhfy68.zip.003
-https://nlp.biu.ac.il/~amit/datasets/poses/holistic/autsl.tar.gz	14820092818	4e80233393dcf83c4d98cbde80e7e9bc202bb4d6782f71e343d148ab212b3f9f	autsl.tar.gz
-https://nlp.biu.ac.il/~amit/datasets/poses/openpose/autsl.tar.gz	1107613563	8678569831453dd9befe3b53a2856e320d052d76f63c5e4c0d45c6add0f9feda	autsl.tar.gz
-https://nlp.biu.ac.il/~amit/datasets/public/autsl_test_labels.csv	86676	a3fe15717484beb2565fd812aba720afcccb36c43173f891ce31163af9137651	autsl_test_labels.csv
-https://nlp.biu.ac.il/~amit/datasets/public/autsl_validation_labels.csv	102264	12e74248fb6199ad41fa58aa85b151966e6da31912ec109b790c6c6d627c773b	autsl_validation_labels.csv

sign_language_datasets/datasets/config.py

@@ -3,6 +3,9 @@
 import cv2
 import tensorflow_datasets as tfds
 
+def cloud_bucket_file(file_name: str) -> str:
+    escaped_file_name = file_name.replace("/", "%2F")
+    return f"https://firebasestorage.googleapis.com/v0/b/sign-language-datasets/o/{escaped_file_name}?alt=media"
 
 class SignDatasetConfig(tfds.core.BuilderConfig):
     """General BuilderConfig for sign language datasets."""

sign_language_datasets/datasets/dgs_corpus/create_index.py

@@ -4,6 +4,8 @@
 import re
 import urllib.request
 
+from sign_language_datasets.datasets.config import cloud_bucket_file
+
 corpus_path = "https://www.sign-lang.uni-hamburg.de/meinedgs/"
 
 index_data = {}
@@ -33,12 +35,12 @@
 
         # Add holistic
         for c in ["a", "b"]:
-            holistic_path = "https://nlp.biu.ac.il/~amit/datasets/poses/holistic/dgs_corpus/" + tr_id + "_" + c + ".pose"
+            holistic_path = cloud_bucket_file(f"poses/holistic/dgs_corpus/{tr_id}_{c}.pose")
             index_data[tr_id]["holistic_" + c] = holistic_path if index_data[tr_id]["video_" + c] is not None else None
 
         # Make sure parsing worked
         if index_data[tr_id]["openpose"] is not None:
             assert index_data[tr_id]["openpose"].endswith(".json.gz")
 
-with open("data.json", "w") as f:
+with open("dgs.json", "w") as f:
     json.dump(index_data, f)

sign_language_datasets/datasets/dgs_corpus/dgs.json

@@ -21,7 +21,7 @@
 from pose_format.pose import Pose
 
 from ..warning import dataset_warning
-from ...datasets.config import SignDatasetConfig
+from ...datasets.config import SignDatasetConfig, cloud_bucket_file
 from ...utils.features import PoseFeature
 
 try:
@@ -49,7 +49,7 @@
 _HOMEPAGE = "https://www.sign-lang.uni-hamburg.de/meinedgs/"
 
 # This `dgs.json` file was created using `create_index.py`
-INDEX_URL = "https://nlp.biu.ac.il/~amit/datasets/dgs.json"
+INDEX_PATH = path.join(path.dirname(path.realpath(__file__)), "dgs.json")
 
 _POSE_HEADERS = {
     "holistic": path.join(path.dirname(path.realpath(__file__)), "holistic.poseheader"),
@@ -279,9 +279,7 @@ def _split_generators(self, dl_manager: tfds.download.DownloadManager):
         """Returns SplitGenerators."""
         dataset_warning(self)
 
-        index_path = dl_manager.download(INDEX_URL)
-
-        with open(index_path, "r", encoding="utf-8") as f:
+        with open(INDEX_PATH, "r", encoding="utf-8") as f:
             index_data = json.load(f)
 
         # No need to download HTML pages