Merge pull request #15 from bricksdont/dgs_split

Dgs split

Author: AmitMY

.gitignore

@@ -9,4 +9,5 @@ sign_language_datasets/datasets/ncslgr
 .coverage
 build/
 dist/
-sign_language_datasets.egg-info/
+sign_language_datasets.egg-info/
+.DS_Store

setup.py

@@ -11,7 +11,7 @@
 setup(
     name="sign-language-datasets",
     packages=packages,
-    version="0.0.12",
+    version="0.0.13",
     description="TFDS Datasets for sign language",
     author="Amit Moryossef",
     author_email="amitmoryossef@gmail.com",

sign_language_datasets/datasets/config.py

@@ -14,6 +14,7 @@ def __init__(
         include_pose: Optional[str] = None,
         fps: Optional[float] = None,
         resolution: Optional[Tuple[int, int]] = None,
+        split: Optional[str] = None,
         extra: dict = {},
         **kwargs,
     ):
@@ -24,6 +25,7 @@ def __init__(
           include_pose: str, what pose data to include.
           fps: float, what pose data to include.
           resolution: (int, int), what resolution of videos to load.
+          split: specify a known split identifier (optional)
           **kwargs: keyword arguments forwarded to super.
         """
         super(SignDatasetConfig, self).__init__(**kwargs)
@@ -33,6 +35,7 @@ def __init__(
 
         self.fps = fps
         self.resolution = resolution
+        self.split = split
         self.extra = extra
 
     def ffmpeg_args(self):

sign_language_datasets/datasets/dgs_corpus/dgs_corpus.py

@@ -10,7 +10,7 @@
 import tensorflow_datasets as tfds
 
 from os import path
-from typing import Dict, Any, Set, Optional
+from typing import Dict, Any, Set, Optional, List
 from pose_format.utils.openpose import load_openpose, OpenPoseFrames
 from pose_format.pose import Pose
 
@@ -44,6 +44,10 @@
     "openpose": path.join(path.dirname(path.realpath(__file__)), "openpose.poseheader"),
 }
 
+_KNOWN_SPLITS = {
+    "3.0.0-uzh-document": path.join(path.dirname(path.realpath(__file__)), "splits", "split.3.0.0-uzh-document.json"),
+}
+
 
 def convert_dgs_dict_to_openpose_frames(input_dict: Dict[str, Any]) -> OpenPoseFrames:
     """
@@ -98,6 +102,32 @@ def get_openpose(openpose_path: str, fps: int, people: Optional[Set] = None,
     return poses
 
 
+def load_split(split_name: str) -> Dict[str, List[str]]:
+    """
+    Loads a split from the file system. What is loaded must be a JSON object with the following structure:
+
+    {"train": ..., "dev": ..., "test": ...}
+
+    :param split_name: An identifier for a predefined split or a filepath to a custom split file.
+    :return: The split loaded as a dictionary.
+    """
+    if split_name not in _KNOWN_SPLITS.keys():
+        # assume that the supplied string is a path on the file system
+        if not path.exists(split_name):
+            raise ValueError("Split '%s' is not a known data split identifier and does not exist as a file either.\n"
+                             "Known split identifiers are: %s" % (split_name, str(_KNOWN_SPLITS)))
+
+        split_path = split_name
+    else:
+        # the supplied string is an identifier for a predefined split
+        split_path = _KNOWN_SPLITS[split_name]
+
+    with open(split_path) as infile:
+        split = json.load(infile)  # type: Dict[str, List[str]]
+
+    return split
+
+
 class DgsCorpus(tfds.core.GeneratorBasedBuilder):
     """DatasetBuilder for dgs_corpus dataset."""
 
@@ -193,7 +223,19 @@ def _split_generators(self, dl_manager: tfds.download.DownloadManager):
             _id: {k: local_paths[v] if v is not None else None for k, v in datum.items()} for _id, datum in index_data.items()
         }
 
-        return [tfds.core.SplitGenerator(name=tfds.Split.TRAIN, gen_kwargs={"data": processed_data})]
+        if self._builder_config.split is not None:
+            split = load_split(self._builder_config.split)
+
+            train_data = {key: value for key, value in processed_data.items() if key in split["train"]}
+            dev_data = {key: value for key, value in processed_data.items() if key in split["dev"]}
+            test_data = {key: value for key, value in processed_data.items() if key in split["test"]}
+
+            return [tfds.core.SplitGenerator(name=tfds.Split.TRAIN, gen_kwargs={"data": train_data}),
+                    tfds.core.SplitGenerator(name=tfds.Split.VALIDATION, gen_kwargs={"data": dev_data}),
+                    tfds.core.SplitGenerator(name=tfds.Split.TEST, gen_kwargs={"data": test_data})]
+
+        else:
+            return [tfds.core.SplitGenerator(name=tfds.Split.TRAIN, gen_kwargs={"data": processed_data})]
 
     def _generate_examples(self, data):
         """ Yields examples. """

sign_language_datasets/datasets/dgs_corpus/splits/create_document_split.py

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+"""dgs_document_split.ipynb
+
+Automatically generated by Colaboratory.
+
+Original file is located at
+    https://colab.research.google.com/drive/19pHmLuIEAKFn4BqVr7cwNRaVxQHWKNI7
+"""
+
+# ! pip install sign-language-datasets==0.0.12
+
+import json
+
+import numpy as np
+
+import tensorflow_datasets as tfds
+import sign_language_datasets.datasets
+from sign_language_datasets.datasets.config import SignDatasetConfig
+from sign_language_datasets.datasets.dgs_corpus.dgs_utils import get_elan_sentences
+
+from typing import Optional, Tuple
+
+np.random.seed(1)
+
+# Videos 1177918 and 1432043 have 25 fps, start and end frame won't match
+
+INCORRECT_FRAMERATE = ["1432043", "1177918"]
+
+
+def get_split_indexes(total_size: int, dev_size: int, test_size) -> Tuple[np.array, np.array, np.array]:
+    train_indexes = np.arange(total_size, dtype=np.int32)
+
+    np.random.shuffle(train_indexes)
+
+    # high inclusive
+
+    dev_indexes = np.random.choice(train_indexes, size=(dev_size,), replace=False)
+
+    remaining_train_indexes = np.asarray([i for i in train_indexes if i not in dev_indexes])
+
+    test_indexes = np.random.choice(remaining_train_indexes, size=(test_size,), replace=False)
+
+    remaining_train_indexes = np.asarray([i for i in remaining_train_indexes if i not in test_indexes])
+
+    return remaining_train_indexes, dev_indexes, test_indexes
+
+
+config = SignDatasetConfig(name="only-annotations", version="1.0.0", include_video=False, include_pose=None)
+dgs_corpus = tfds.load('dgs_corpus', builder_kwargs=dict(config=config))
+
+
+def get_split(dev_size: int, test_size: int):
+    ids = np.array([datum["id"].numpy().decode("utf-8") for datum in dgs_corpus["train"] if
+                    datum["id"] not in INCORRECT_FRAMERATE])
+
+    train_indexes, dev_indexes, test_indexes = get_split_indexes(len(ids), dev_size=dev_size, test_size=test_size)
+
+    print("Number of entire files in each split:")
+    print(str({"train": len(train_indexes), "dev": len(dev_indexes), "test": len(test_indexes)}))
+
+    return {"dgs_corpus_version": "3.0.0",
+            "train": list(ids[train_indexes]),
+            "dev": list(ids[dev_indexes]),
+            "test": list(ids[test_indexes])}
+
+
+split = get_split(dev_size=10, test_size=10)
+
+with open('split.json', 'w') as outfile:
+    json.dump(split, outfile, indent=4)
+
+# ! cat split.json
+
+"""## compute sentence statistics for this split"""
+
+
+def get_split_name_from_id(_id: str) -> str:
+    for key in split.keys():
+        if _id in split[key]:
+            return key
+
+    return "none"
+
+
+sentences_found = {"train": 0, "dev": 0, "test": 0, "none": 0}
+
+for datum in dgs_corpus["train"]:
+
+    _id = datum["id"].numpy().decode('utf-8')
+
+    split_name = get_split_name_from_id(_id)
+
+    elan_path = datum["paths"]["eaf"].numpy().decode('utf-8')
+    sentences = get_elan_sentences(elan_path)
+
+    for sentence in sentences:
+        gloss_sequence = " ".join([s["gloss"] for s in sentence["glosses"]])
+        german_sentence = sentence["german"]
+
+        if gloss_sequence != "" and german_sentence != "":
+            sentences_found[split_name] += 1
+
+print(sentences_found)