diff --git a/.gitignore b/.gitignore index 0b0c6bf..c49449b 100644 --- a/.gitignore +++ b/.gitignore @@ -143,3 +143,6 @@ dmypy.json #VS Code .vscode/ + +tests/data/tmp_* +docs/data/tmp_* diff --git a/.readthedocs.yml b/.readthedocs.yml deleted file mode 100644 index 758d263..0000000 --- a/.readthedocs.yml +++ /dev/null @@ -1,14 +0,0 @@ -# Required -version: 2 - -# Image to use -build: - image: testing - -# Configuration -python: - version: 3.9 - install: - - requirements: docs/requirements.txt - - method: pip - path: . diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index b5259ae..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,7 +0,0 @@ -# Changelog - - - -## v0.1.0 (13/12/2022) - -- First release of `tsdf`! \ No newline at end of file diff --git a/docs/basic_reading_and_writing.ipynb b/docs/basic_reading_and_writing.ipynb new file mode 100644 index 0000000..a6f7c63 --- /dev/null +++ b/docs/basic_reading_and_writing.ipynb @@ -0,0 +1,237 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Basic reading and writing\n", + "\n", + "These are some examples on how to read and write TSDF data into and from a numpy array, using the `tsdf` library." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import tsdf" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Load some data" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Data type:\t int16\n", + "Data shape:\t (10, 3)\n" + ] + } + ], + "source": [ + "# The name of the data\n", + "metadata_path = \"data/example_meta.json\"\n", + "binary_filename = \"example_binary.bin\"\n", + "\n", + "# Multiple metadata files (one for each binary) are loaded into a dictionary\n", + "# mapping the binary file name to the metadata object\n", + "metadata_dict = tsdf.load_metadata_from_path(metadata_path)\n", + "\n", + "# Retrieve the metadata object we want, using the name of the binary as key\n", + "metadata = metadata_dict[binary_filename]\n", + "\n", + "# Load the data\n", + "data = tsdf.load_binary_from_metadata(metadata)\n", + "\n", + "# Print some info\n", + "print(f\"Data type:\\t {data.dtype}\")\n", + "print(f\"Data shape:\\t {data.shape}\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Perform basic data processing" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Processed data type:\t float32\n", + "Data shape:\t\t (10, 3)\n" + ] + } + ], + "source": [ + "# Perform an operation, resulting in a different data type\n", + "processed_data_1 = (data / 10).astype('float32')\n", + "\n", + "# Print some info\n", + "print(f\"Processed data type:\\t {processed_data_1.dtype}\")\n", + "print(f\"Data shape:\\t\\t {processed_data_1.shape}\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Write the processed data \n", + "Write the processed data in binary format. The call returns the corresponding metadata object." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "File written to data/tmp_example_processed.bin\n" + ] + } + ], + "source": [ + "# The new name of the file\n", + "output_bin_filename = \"tmp_example_processed.bin\"\n", + "\n", + "# Write the data to a new binary file\n", + "processed_metadata_1 = tsdf.write_binary_file(\n", + " \"data\",\n", + " output_bin_filename,\n", + " processed_data_1,\n", + " metadata.get_plain_tsdf_dict_copy(),\n", + " )\n", + "\n", + "print(f\"File written to data/{output_bin_filename}\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Write the TSDF metadata file" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "File written to data/tmp_example_processed_meta.json\n" + ] + } + ], + "source": [ + "# Write new metadata file\n", + "output_meta_filename = \"tmp_example_processed_meta.json\"\n", + "tsdf.write_metadata([processed_metadata_1], output_meta_filename)\n", + "print(f\"File written to data/{output_meta_filename}\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Write a metadata file that combines multiple binary files" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "File written to data/tmp_example_processed_2.bin\n", + "File written to data/tmp_example_processed_2_meta.json\n" + ] + } + ], + "source": [ + "# Preprocess the original data to generate another data source\n", + "processed_data_2 = (data * 1000).astype(\"int32\")\n", + "\n", + "# Adjust the metadata slightly\n", + "updated_metadata = metadata.get_plain_tsdf_dict_copy()\n", + "updated_metadata.pop(\"scale_factors\") # remove the 'scale_factors'\n", + "\n", + "# Save the new binary file\n", + "output_bin_filename_2 = \"tmp_example_processed_2.bin\"\n", + "processed_metadata_2 = tsdf.write_binary_file(\n", + " \"data\",\n", + " output_bin_filename_2,\n", + " processed_data_2,\n", + " updated_metadata,\n", + ")\n", + "print(f\"File written to data/{output_bin_filename_2}\")\n", + "\n", + "# Write a metadata file that combines the two binary files\n", + "output_meta_filename_2 = \"tmp_example_processed_2_meta.json\"\n", + "tsdf.write_metadata([processed_metadata_1, processed_metadata_2],\n", + " output_meta_filename_2)\n", + "print(f\"File written to data/{output_meta_filename_2}\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "tsdf-zVA6tG---py3.9", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "d1e978847a77d4ff49203fd09f0f7925f58560bf1007438482d75cb657018d9b" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/converting_legacy_data.ipynb b/docs/converting_legacy_data.ipynb new file mode 100644 index 0000000..58da127 --- /dev/null +++ b/docs/converting_legacy_data.ipynb @@ -0,0 +1,70 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Transform legacy (TSDB) format to the current TSDF v0.1\n", + "Transform one file (or all files within the given directory) from TSDB to TSDF format." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from tsdf.legacy_tsdf_utils import (\n", + " generate_tsdf_metadata_from_tsdb,\n", + " convert_file_tsdb_to_tsdf,\n", + " convert_files_tsdb_to_tsdf,\n", + ")\n", + "\n", + "data_dir = 'data'" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# Path to the metadata file\n", + "path_to_file = os.path.join(data_dir, \"ppp_format_meta_legacy.json\")\n", + "path_to_new_file = os.path.join(data_dir, \"tmp_ppp_format_meta.json\")\n", + "\n", + "# Generate a TSDF metadata file from TSDB\n", + "generate_tsdf_metadata_from_tsdb(path_to_file, path_to_new_file)\n", + "\n", + "# Convert a TSDB metadata file to TSDB format\n", + "# convert_metadata_tsdb_to_tsdf(path_to_file)\n", + "\n", + "# Convert all metadata files in the directory from TSDB to TSDF format\n", + "# convert_metadatas_tsdb_to_tsdf(path_to_dir)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "tsdf-zVA6tG---py3.9", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/data/example_binary.bin b/docs/data/example_binary.bin new file mode 100644 index 0000000..be558be Binary files /dev/null and b/docs/data/example_binary.bin differ diff --git a/docs/data/example_meta.json b/docs/data/example_meta.json new file mode 100644 index 0000000..1c0bd7d --- /dev/null +++ b/docs/data/example_meta.json @@ -0,0 +1,30 @@ +{ + "subject_id": "dummy", + "study_id": "dummy", + "device_id": "dummy", + "endianness": "little", + "metadata_version": "0.1", + "start_datetime_unix_ms": 1571135957025, + "start_iso8601": "2019-10-15T10:39:17.025000+00:00", + "end_datetime_unix_ms": 1571168851826, + "end_iso8601": "2019-10-15T19:47:31.826000+00:00", + "file_name": "example_binary.bin", + "channels": [ + "x", + "y", + "z" + ], + "units": [ + "m/s/s", + "m/s/s", + "m/s/s" + ], + "scale_factors": [ + 0.00469378, + 0.00469378, + 0.00469378 + ], + "data_type": "int", + "bits": 16, + "rows": 10 +} \ No newline at end of file diff --git a/docs/data/ppp_format_meta_legacy.json b/docs/data/ppp_format_meta_legacy.json new file mode 100644 index 0000000..cf238b5 --- /dev/null +++ b/docs/data/ppp_format_meta_legacy.json @@ -0,0 +1,58 @@ +{ + "project_id": "PPP", + "device_id": "Verily Study Watch", + "subject_id": "77", + "source_file_name": "WatchData.IMU.Week10.raw", + "endianness": "little", + "metadata_version": "0.1", + "start_datetime_unix_ms": 1571135957025, + "start_datetime_iso8601": "2019-10-15T10:39:17.025000+00:00", + "end_datetime_unix_ms": 1571168851826, + "end_datetime_iso8601": "2019-10-15T19:47:31.826000+00:00", + "datasets": [ + { + "file_name": "ppp_format_time.bin", + "quantities": "time", + "time_encode": "difference", + "units": "ms", + "scale_factors": [ + 1 + ], + "datatype": "float", + "bits": 32, + "columns": 1, + "rows": 17 + }, + { + "file_name": "ppp_format_samples.bin", + "quantities": [ + "acceleration_x", + "acceleration_y", + "acceleration_z", + "rotation_x", + "rotation_y", + "rotation_z" + ], + "units": [ + "m/s/s", + "m/s/s", + "m/s/s", + "deg/s", + "deg/s", + "deg/s" + ], + "scale_factors": [ + 0.00469378, + 0.00469378, + 0.00469378, + 0.06097561, + 0.06097561, + 0.06097561 + ], + "datatype": "int", + "bits": 16, + "columns": 6, + "rows": 17 + } + ] +} \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index a2e19d0..85081a5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,9 +4,6 @@ A package to work with TSDF data in Python. This implementation is based on the ## What is TSDF data? -`tsdf` stands for _`time series data format`_. -It is a unified, standardized format for storing all types of physiological sensor data. It was originally introduced in this [preprint](https://arxiv.org/abs/2211.11294). - TSDF provides a unified, user-friendly format for both numerical sensor data and metadata, utilizing raw binary data and JSON-format text files for measurements/timestamps and metadata, respectively. It defines essential metadata fields to enhance data interpretability and exchangeability, aiming to bolster scientific reproducibility in studies reliant on digital biosensor data as a critical evidence base across various disease domains. diff --git a/docs/installation.md b/docs/installation.md index abaf607..8e46e30 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,7 +1,5 @@ # Installation -## Installing tsdf - The package is available in [PyPI](https://pypi.org/project/tsdf/). The latest stable release can be installed using: ```bash @@ -27,4 +25,4 @@ Otherwise you can install it manually by following these steps: ## What now? Now you can import functions from the `tsdf` Python package. -See some examples in the next section. +See some examples in the [usage section](/tsdf/basic_reading_and_writing). diff --git a/docs/processing_example.ipynb b/docs/processing_example.ipynb deleted file mode 100644 index 017e94f..0000000 --- a/docs/processing_example.ipynb +++ /dev/null @@ -1,354 +0,0 @@ -{ - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Usage Examples\n", - "\n", - "These are some examples on how to read and write TSDF data into and from a numpy array, using the `tsdf` library." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## How to run these examples\n", - "\n", - "### Imports\n" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "import os\n", - "import numpy as np\n", - "import tsdf" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Set data location" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "data_dir = \"../tests/data\"" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Process an existing binary file and write the new data\n", - "Read and process an existing binary data (accompanied by the TSDF metadata), process the data and save it in the new format, with the corresponding TSDF metadata file.\n", - "\n", - "### Load dummy data" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Data type used for storing:\t int16\n", - "Data dimensions:\t\t (10, 3)\n", - "Number of rows:\t\t\t 10\n" - ] - } - ], - "source": [ - "# The name of the data\n", - "data_name = \"example_10_3_int16\"\n", - "\n", - "# Multiple metadata files (for each binary) are loaded into a dictionary\n", - "# mapping the binary file name to the metadata object\n", - "metadata_dict = tsdf.load_metadata_from_path(f\"{data_dir}/{data_name}_meta.json\")\n", - "\n", - "# Retrieve the metadata object we want, using the name of the binary as key\n", - "metadata = metadata_dict[f\"{data_name}.bin\"]\n", - "\n", - "# Load the data\n", - "data = metadata.load_binary()\n", - "\n", - "# Print some info\n", - "print(f\"Data type used for storing:\\t {data.dtype}\")\n", - "print(f\"Data dimensions:\\t\\t {data.shape}\")\n", - "print(f\"Number of rows:\\t\\t\\t {data.shape[0]}\")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Perform basic data processing" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Data type used for storing:\t float32\n", - "Data dimensions:\t\t (10, 3)\n", - "Number of rows:\t\t\t 10\n" - ] - } - ], - "source": [ - "# Perform an operation, resulting in a different data type\n", - "processed_data_1 = (data / 10).astype('float32')\n", - "\n", - "# Print some info\n", - "print(f\"Data type used for storing:\\t {processed_data_1.dtype}\")\n", - "print(f\"Data dimensions:\\t\\t {processed_data_1.shape}\")\n", - "print(f\"Number of rows:\\t\\t\\t {processed_data_1.shape[0]}\")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Write the processed data \n", - "Write the processed data in binary format. The call returns the corresponding metadata object." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "# The new name of the file\n", - "processed_data_name_1 = \"tmp_test_example_10_3_int16_to_float32\"\n", - "\n", - "# Write the data to a new binary file\n", - "processed_metadata_1 = tsdf.write_binary_file(\n", - " data_dir,\n", - " f\"{processed_data_name_1}.bin\",\n", - " processed_data_1,\n", - " metadata.get_plain_tsdf_dict_copy(),\n", - " )" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Write the TSDF metadata file" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "# Write new metadata file\n", - "tsdf.write_metadata([processed_metadata_1], f\"{processed_data_name_1}_meta.json\")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Write a metadata file that combines multiple binary files" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "# Preprocess the original data to generate another data source\n", - "processed_data_2 = (data * 1000).astype(\"int32\")\n", - "\n", - "# Adjust the metadata slightly\n", - "updated_metadata = metadata.get_plain_tsdf_dict_copy()\n", - "updated_metadata.pop(\"scale_factors\") # remove the 'scale_factors'\n", - "\n", - "# Save the new binary file\n", - "processed_data_name_2 = \"tmp_test_example_10_3_int16_to_int32\"\n", - "processed_metadata_2 = tsdf.write_binary_file(\n", - " data_dir,\n", - " f\"{processed_data_name_2}_.bin\",\n", - " processed_data_2,\n", - " updated_metadata,\n", - ")\n", - "\n", - "# Write a metadata file that combines the two binary files\n", - "tsdf.write_metadata(\n", - " [processed_metadata_1, processed_metadata_2],\n", - " \"tmp_test_example_10_3_int16_to_int_n_float_meta.json\",\n", - ")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Generate and save data from scratch" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "# Generate random data\n", - "rs = np.random.RandomState(seed=42)\n", - "data_1 = rs.rand(17, 1).astype(np.float32)\n", - "data_2 = rs.rand(15, 2).astype(np.int16)\n", - "data_3 = rs.rand(10, 3).astype(np.int16)\n", - "\n", - "# Define the metadata\n", - "new_metadata = {\n", - " \"subject_id\": \"example\",\n", - " \"study_id\": \"example\",\n", - " \"device_id\": \"example\",\n", - " \"endianness\": \"little\",\n", - " \"metadata_version\": \"0.1\",\n", - " \"start_datetime_unix_ms\": 1571135957025,\n", - " \"start_iso8601\": \"2019-10-15T10:39:17.025000+00:00\",\n", - " \"end_datetime_unix_ms\": 1571168851826,\n", - " \"end_iso8601\": \"2019-10-15T19:47:31.826000+00:00\",\n", - " \"channels\": [\"x\", \"y\", \"z\"],\n", - " \"units\": [\"m/s/s\", \"m/s/s\", \"m/s/s\"]\n", - "}\n", - "\n", - "# Write the three binary files based on the provided metadata\n", - "file_prefix = \"tmp_test\"\n", - "new_meta_1 = tsdf.write_binary_file(data_dir, f\"{file_prefix}_1.bin\", data_1, new_metadata)\n", - "new_meta_2 = tsdf.write_binary_file(data_dir, f\"{file_prefix}_2.bin\", data_2, new_metadata)\n", - "new_meta_3 = tsdf.write_binary_file(data_dir, f\"{file_prefix}_3.bin\", data_3, new_metadata)\n", - "\n", - "# Write the metadata file, which references the three binary files\n", - "tsdf.write_metadata([new_meta_1, new_meta_2, new_meta_3], f\"{file_prefix}_meta.json\")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Transform legacy (TSDB) format to the current TSDF v0.1\n", - "Transform one file (or all files within the given directory) from TSDB to TSDF format." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "from tsdf.legacy_tsdf_utils import (\n", - " generate_tsdf_metadata_from_tsdb,\n", - " convert_file_tsdb_to_tsdf,\n", - " convert_files_tsdb_to_tsdf,\n", - ")\n", - "\n", - "# Path to the metadata file\n", - "path_to_file = os.path.join(data_dir, \"ppp_format_meta_legacy.json\")\n", - "path_to_new_file = os.path.join(data_dir, \"tmp_ppp_format_meta.json\")\n", - "\n", - "# Generate a TSDF metadata file from TSDB\n", - "generate_tsdf_metadata_from_tsdb(path_to_file, path_to_new_file)\n", - "\n", - "# Convert a TSDB metadata file to TSDB format\n", - "# convert_metadata_tsdb_to_tsdf(path_to_file)\n", - "\n", - "# Convert all metadata files in the directory from TSDB to TSDF format\n", - "# convert_metadatas_tsdb_to_tsdf(path_to_dir)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Validate TSDF file\n", - "\n", - "Files can be validated using the validator module, which is also callable from the command line. The validator checks the metadata file and inspects whether the binary file is consistent with the metadata. This snippet shows how to use the validator from code." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Successfully loaded binary file ppp_format_time.bin, resulting shape: (17,)\n", - "Successfully loaded binary file ppp_format_samples.bin, resulting shape: (17, 6)\n" - ] - } - ], - "source": [ - "# Import the validator\n", - "from tsdf import validator\n", - "\n", - "# Verify the metadata file\n", - "path_to_metadata_file = os.path.join(data_dir, \"ppp_format_meta.json\")\n", - "validator.validate_tsdf_format(path_to_metadata_file)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "tsdf-zVA6tG---py3.9", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.13" - }, - "orig_nbformat": 4, - "vscode": { - "interpreter": { - "hash": "d1e978847a77d4ff49203fd09f0f7925f58560bf1007438482d75cb657018d9b" - } - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs/tsdf_fields_table.md b/docs/tsdf_fields_table.md index 2cc6b89..f8d9849 100644 --- a/docs/tsdf_fields_table.md +++ b/docs/tsdf_fields_table.md @@ -25,7 +25,7 @@ TSDF metadata is represented as a dictionary. In this section, we will comprehen -## TSDF optional fields +## TSDF domain-specific fields | Field | Type | Description | diff --git a/docs/validating_files.ipynb b/docs/validating_files.ipynb new file mode 100644 index 0000000..29f51bb --- /dev/null +++ b/docs/validating_files.ipynb @@ -0,0 +1,76 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Validate TSDF file\n", + "\n", + "Files can be validated using the validator module, which is also callable from the command line. The validator checks the metadata file and inspects whether the binary file is consistent with the metadata.\n", + "\n", + "The validator can be called from the command line as follows:\n", + "\n", + "```bash\n", + "pip install tsdf # in case you didn't already do this\n", + "validate-tsdf file-to-check.json\n", + "```\n", + "\n", + "The snippet below shows how to use the validator from code." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from tsdf import validator\n", + "\n", + "data_dir = \"../tests/data\"" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Successfully loaded binary file ppp_format_time.bin, resulting shape: (17,)\n", + "Successfully loaded binary file ppp_format_samples.bin, resulting shape: (17, 6)\n" + ] + } + ], + "source": [ + "# Verify the metadata file\n", + "path_to_metadata_file = os.path.join(data_dir, \"ppp_format_meta.json\")\n", + "result = validator.validate_tsdf_format(path_to_metadata_file)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "tsdf-zVA6tG---py3.9", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/writing_data_from_scratch.ipynb b/docs/writing_data_from_scratch.ipynb new file mode 100644 index 0000000..9184df9 --- /dev/null +++ b/docs/writing_data_from_scratch.ipynb @@ -0,0 +1,95 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Writing data from scratch\n", + "\n", + "When your data is not based on existing data, you will have to create a metadata object yourself." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import tsdf\n", + "\n", + "data_dir = \"data\"" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Files written to data/\n" + ] + } + ], + "source": [ + "# Generate random data\n", + "rs = np.random.RandomState(seed=42)\n", + "data_1 = rs.rand(17, 1).astype(np.float32)\n", + "data_2 = rs.rand(15, 2).astype(np.int16)\n", + "data_3 = rs.rand(10, 3).astype(np.int16)\n", + "\n", + "# Define the metadata\n", + "basic_metadata = {\n", + " \"subject_id\": \"example\",\n", + " \"study_id\": \"example\",\n", + " \"device_id\": \"example\",\n", + " \"endianness\": \"little\",\n", + " \"metadata_version\": \"0.1\",\n", + " \"start_datetime_unix_ms\": 1571135957025,\n", + " \"start_iso8601\": \"2019-10-15T10:39:17.025000+00:00\",\n", + " \"end_datetime_unix_ms\": 1571168851826,\n", + " \"end_iso8601\": \"2019-10-15T19:47:31.826000+00:00\",\n", + " \"channels\": [\"x\", \"y\", \"z\"],\n", + " \"units\": [\"m/s/s\", \"m/s/s\", \"m/s/s\"]\n", + "}\n", + "\n", + "# Write the three binary files based on the provided metadata.\n", + "# The new_meta variables will contain the basic_metadata, combined\n", + "# with the fields derived from the binary data (data type, bit depth, etc.)\n", + "file_prefix = \"tmp_test\"\n", + "new_meta_1 = tsdf.write_binary_file(data_dir, f\"{file_prefix}_1.bin\", data_1, basic_metadata)\n", + "new_meta_2 = tsdf.write_binary_file(data_dir, f\"{file_prefix}_2.bin\", data_2, basic_metadata)\n", + "new_meta_3 = tsdf.write_binary_file(data_dir, f\"{file_prefix}_3.bin\", data_3, basic_metadata)\n", + "\n", + "# Write the metadata file, which combines the metadata, in turn referencing the three binary files\n", + "tsdf.write_metadata([new_meta_1, new_meta_2, new_meta_3], f\"{file_prefix}_meta.json\")\n", + "print(f\"Files written to {data_dir}/\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "tsdf-zVA6tG---py3.9", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/mkdocs.yml b/mkdocs.yml index b148011..cbe76fa 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -11,7 +11,16 @@ theme: readthedocs nav: - Introduction: index.md - - TSDF fields: tsdf_fields_table.md - Installation: installation.md - - Example: processing_example.ipynb - - Contact: contact.md + + - The format: + - TSDF fields: tsdf_fields_table.md + + - Usage: + - Basic reading and writing: basic_reading_and_writing.ipynb + - Writing data from scratch: writing_data_from_scratch.ipynb + - Converting legacy data: converting_legacy_data.ipynb + - Validating files: validating_files.ipynb + + - About: + - Contact: contact.md diff --git a/poetry.lock b/poetry.lock index 1398346..5826409 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,9 +1,10 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry and should not be changed by hand. [[package]] name = "anyio" version = "4.0.0" description = "High level compatibility layer for multiple asynchronous event loop implementations" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -25,6 +26,7 @@ trio = ["trio (>=0.22)"] name = "appnope" version = "0.1.3" description = "Disable App Nap on macOS >= 10.9" +category = "dev" optional = false python-versions = "*" files = [ @@ -36,6 +38,7 @@ files = [ name = "argon2-cffi" version = "23.1.0" description = "Argon2 for Python" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -56,6 +59,7 @@ typing = ["mypy"] name = "argon2-cffi-bindings" version = "21.2.0" description = "Low-level CFFI bindings for Argon2" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -93,6 +97,7 @@ tests = ["pytest"] name = "arrow" version = "1.2.3" description = "Better dates & times for Python" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -107,6 +112,7 @@ python-dateutil = ">=2.7.0" name = "asttokens" version = "2.4.0" description = "Annotate AST trees with source code positions" +category = "dev" optional = false python-versions = "*" files = [ @@ -124,6 +130,7 @@ test = ["astroid", "pytest"] name = "async-lru" version = "2.0.4" description = "Simple LRU cache for asyncio" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -138,6 +145,7 @@ typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -156,6 +164,7 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte name = "babel" version = "2.12.1" description = "Internationalization utilities" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -167,6 +176,7 @@ files = [ name = "backcall" version = "0.2.0" description = "Specifications for callback functions passed in to an API" +category = "dev" optional = false python-versions = "*" files = [ @@ -178,6 +188,7 @@ files = [ name = "beautifulsoup4" version = "4.12.2" description = "Screen-scraping library" +category = "dev" optional = false python-versions = ">=3.6.0" files = [ @@ -196,6 +207,7 @@ lxml = ["lxml"] name = "bleach" version = "6.0.0" description = "An easy safelist-based HTML-sanitizing tool." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -214,6 +226,7 @@ css = ["tinycss2 (>=1.1.0,<1.2)"] name = "certifi" version = "2023.7.22" description = "Python package for providing Mozilla's CA Bundle." +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -225,6 +238,7 @@ files = [ name = "cffi" version = "1.15.1" description = "Foreign Function Interface for Python calling C code." +category = "dev" optional = false python-versions = "*" files = [ @@ -301,6 +315,7 @@ pycparser = "*" name = "charset-normalizer" version = "3.2.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "dev" optional = false python-versions = ">=3.7.0" files = [ @@ -385,6 +400,7 @@ files = [ name = "click" version = "8.1.7" description = "Composable command line interface toolkit" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -399,6 +415,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." +category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -410,6 +427,7 @@ files = [ name = "comm" version = "0.1.4" description = "Jupyter Python Comm implementation, for usage in ipykernel, xeus-python etc." +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -429,6 +447,7 @@ typing = ["mypy (>=0.990)"] name = "contourpy" version = "1.1.0" description = "Python library for calculating contours of 2D quadrilateral grids" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -439,6 +458,7 @@ files = [ {file = "contourpy-1.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18a64814ae7bce73925131381603fff0116e2df25230dfc80d6d690aa6e20b37"}, {file = "contourpy-1.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90c81f22b4f572f8a2110b0b741bb64e5a6427e0a198b2cdc1fbaf85f352a3aa"}, {file = "contourpy-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:53cc3a40635abedbec7f1bde60f8c189c49e84ac180c665f2cd7c162cc454baa"}, + {file = "contourpy-1.1.0-cp310-cp310-win32.whl", hash = "sha256:9b2dd2ca3ac561aceef4c7c13ba654aaa404cf885b187427760d7f7d4c57cff8"}, {file = "contourpy-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:1f795597073b09d631782e7245016a4323cf1cf0b4e06eef7ea6627e06a37ff2"}, {file = "contourpy-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0b7b04ed0961647691cfe5d82115dd072af7ce8846d31a5fac6c142dcce8b882"}, {file = "contourpy-1.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:27bc79200c742f9746d7dd51a734ee326a292d77e7d94c8af6e08d1e6c15d545"}, @@ -447,6 +467,7 @@ files = [ {file = "contourpy-1.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5cec36c5090e75a9ac9dbd0ff4a8cf7cecd60f1b6dc23a374c7d980a1cd710e"}, {file = "contourpy-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f0cbd657e9bde94cd0e33aa7df94fb73c1ab7799378d3b3f902eb8eb2e04a3a"}, {file = "contourpy-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:181cbace49874f4358e2929aaf7ba84006acb76694102e88dd15af861996c16e"}, + {file = "contourpy-1.1.0-cp311-cp311-win32.whl", hash = "sha256:edb989d31065b1acef3828a3688f88b2abb799a7db891c9e282df5ec7e46221b"}, {file = "contourpy-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:fb3b7d9e6243bfa1efb93ccfe64ec610d85cfe5aec2c25f97fbbd2e58b531256"}, {file = "contourpy-1.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bcb41692aa09aeb19c7c213411854402f29f6613845ad2453d30bf421fe68fed"}, {file = "contourpy-1.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5d123a5bc63cd34c27ff9c7ac1cd978909e9c71da12e05be0231c608048bb2ae"}, @@ -455,6 +476,7 @@ files = [ {file = "contourpy-1.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:317267d915490d1e84577924bd61ba71bf8681a30e0d6c545f577363157e5e94"}, {file = "contourpy-1.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d551f3a442655f3dcc1285723f9acd646ca5858834efeab4598d706206b09c9f"}, {file = "contourpy-1.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e7a117ce7df5a938fe035cad481b0189049e8d92433b4b33aa7fc609344aafa1"}, + {file = "contourpy-1.1.0-cp38-cp38-win32.whl", hash = "sha256:108dfb5b3e731046a96c60bdc46a1a0ebee0760418951abecbe0fc07b5b93b27"}, {file = "contourpy-1.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:d4f26b25b4f86087e7d75e63212756c38546e70f2a92d2be44f80114826e1cd4"}, {file = "contourpy-1.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc00bb4225d57bff7ebb634646c0ee2a1298402ec10a5fe7af79df9a51c1bfd9"}, {file = "contourpy-1.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:189ceb1525eb0655ab8487a9a9c41f42a73ba52d6789754788d1883fb06b2d8a"}, @@ -463,6 +485,7 @@ files = [ {file = "contourpy-1.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:143dde50520a9f90e4a2703f367cf8ec96a73042b72e68fcd184e1279962eb6f"}, {file = "contourpy-1.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e94bef2580e25b5fdb183bf98a2faa2adc5b638736b2c0a4da98691da641316a"}, {file = "contourpy-1.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ed614aea8462735e7d70141374bd7650afd1c3f3cb0c2dbbcbe44e14331bf002"}, + {file = "contourpy-1.1.0-cp39-cp39-win32.whl", hash = "sha256:71551f9520f008b2950bef5f16b0e3587506ef4f23c734b71ffb7b89f8721999"}, {file = "contourpy-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:438ba416d02f82b692e371858143970ed2eb6337d9cdbbede0d8ad9f3d7dd17d"}, {file = "contourpy-1.1.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a698c6a7a432789e587168573a864a7ea374c6be8d4f31f9d87c001d5a843493"}, {file = "contourpy-1.1.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:397b0ac8a12880412da3551a8cb5a187d3298a72802b45a3bd1805e204ad8439"}, @@ -487,6 +510,7 @@ test-no-images = ["pytest", "pytest-cov", "wurlitzer"] name = "coverage" version = "7.3.1" description = "Code coverage measurement for Python" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -554,6 +578,7 @@ toml = ["tomli"] name = "cycler" version = "0.11.0" description = "Composable style cycles" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -565,6 +590,7 @@ files = [ name = "debugpy" version = "1.6.7.post1" description = "An implementation of the Debug Adapter Protocol for Python" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -592,6 +618,7 @@ files = [ name = "decorator" version = "5.1.1" description = "Decorators for Humans" +category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -603,6 +630,7 @@ files = [ name = "defusedxml" version = "0.7.1" description = "XML bomb protection for Python stdlib modules" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -614,6 +642,7 @@ files = [ name = "entrypoints" version = "0.4" description = "Discover and load entry points from installed packages." +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -625,6 +654,7 @@ files = [ name = "exceptiongroup" version = "1.1.3" description = "Backport of PEP 654 (exception groups)" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -639,6 +669,7 @@ test = ["pytest (>=6)"] name = "executing" version = "1.2.0" description = "Get the currently executing AST node of a frame, and other information" +category = "dev" optional = false python-versions = "*" files = [ @@ -653,6 +684,7 @@ tests = ["asttokens", "littleutils", "pytest", "rich"] name = "fastjsonschema" version = "2.18.0" description = "Fastest Python implementation of JSON schema" +category = "dev" optional = false python-versions = "*" files = [ @@ -663,23 +695,11 @@ files = [ [package.extras] devel = ["colorama", "json-spec", "jsonschema", "pylint", "pytest", "pytest-benchmark", "pytest-cache", "validictory"] -[[package]] -name = "flatten-json" -version = "0.1.13" -description = "Flatten JSON objects" -optional = false -python-versions = "*" -files = [ - {file = "flatten_json-0.1.13.tar.gz", hash = "sha256:ee352333e8293e957ccb1b4597a111fc4f6da88ab74b8cb3f8f51eed1e12f500"}, -] - -[package.dependencies] -six = "*" - [[package]] name = "fonttools" version = "4.42.1" description = "Tools to manipulate font files" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -737,6 +757,7 @@ woff = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "zopfli (>=0.1.4)"] name = "fqdn" version = "1.5.1" description = "Validates fully-qualified domain names against RFC 1123, so that they are acceptable to modern bowsers" +category = "dev" optional = false python-versions = ">=2.7, !=3.0, !=3.1, !=3.2, !=3.3, !=3.4, <4" files = [ @@ -748,6 +769,7 @@ files = [ name = "ghp-import" version = "2.1.0" description = "Copy your docs directly to the gh-pages branch." +category = "dev" optional = false python-versions = "*" files = [ @@ -765,6 +787,7 @@ dev = ["flake8", "markdown", "twine", "wheel"] name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" +category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -776,6 +799,7 @@ files = [ name = "importlib-metadata" version = "6.8.0" description = "Read metadata from Python packages" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -795,6 +819,7 @@ testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs name = "importlib-resources" version = "6.0.1" description = "Read resources from Python packages" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -813,6 +838,7 @@ testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -824,6 +850,7 @@ files = [ name = "ipykernel" version = "6.25.2" description = "IPython Kernel for Jupyter" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -837,7 +864,7 @@ comm = ">=0.1.1" debugpy = ">=1.6.5" ipython = ">=7.23.1" jupyter-client = ">=6.1.12" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0.0 || >=5.1.0" matplotlib-inline = ">=0.1" nest-asyncio = "*" packaging = "*" @@ -857,6 +884,7 @@ test = ["flaky", "ipyparallel", "pre-commit", "pytest (>=7.0)", "pytest-asyncio" name = "ipython" version = "8.15.0" description = "IPython: Productive Interactive Computing" +category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -897,6 +925,7 @@ test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.21)", "pa name = "ipython-genutils" version = "0.2.0" description = "Vestigial utilities from IPython" +category = "dev" optional = false python-versions = "*" files = [ @@ -908,6 +937,7 @@ files = [ name = "ipywidgets" version = "8.1.0" description = "Jupyter interactive widgets" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -929,6 +959,7 @@ test = ["ipykernel", "jsonschema", "pytest (>=3.6.0)", "pytest-cov", "pytz"] name = "isoduration" version = "20.11.0" description = "Operations with ISO 8601 durations" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -943,6 +974,7 @@ arrow = ">=0.15.0" name = "jedi" version = "0.19.0" description = "An autocompletion tool for Python that can be used for text editors." +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -962,6 +994,7 @@ testing = ["Django (<3.1)", "attrs", "colorama", "docopt", "pytest (<7.0.0)"] name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -979,6 +1012,7 @@ i18n = ["Babel (>=2.7)"] name = "json5" version = "0.9.14" description = "A Python implementation of the JSON5 data format." +category = "dev" optional = false python-versions = "*" files = [ @@ -993,6 +1027,7 @@ dev = ["hypothesis"] name = "jsonpointer" version = "2.4" description = "Identify specific nodes in a JSON document (RFC 6901)" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" files = [ @@ -1004,6 +1039,7 @@ files = [ name = "jsonschema" version = "4.19.0" description = "An implementation of JSON Schema validation for Python" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1033,6 +1069,7 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- name = "jsonschema-specifications" version = "2023.7.1" description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1047,6 +1084,7 @@ referencing = ">=0.28.0" name = "jupyter" version = "1.0.0" description = "Jupyter metapackage. Install all the Jupyter components in one go." +category = "dev" optional = false python-versions = "*" files = [ @@ -1067,6 +1105,7 @@ qtconsole = "*" name = "jupyter-client" version = "8.3.1" description = "Jupyter protocol implementation and client libraries" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1076,7 +1115,7 @@ files = [ [package.dependencies] importlib-metadata = {version = ">=4.8.3", markers = "python_version < \"3.10\""} -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0.0 || >=5.1.0" python-dateutil = ">=2.8.2" pyzmq = ">=23.0" tornado = ">=6.2" @@ -1090,6 +1129,7 @@ test = ["coverage", "ipykernel (>=6.14)", "mypy", "paramiko", "pre-commit", "pyt name = "jupyter-console" version = "6.6.3" description = "Jupyter terminal console" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1101,7 +1141,7 @@ files = [ ipykernel = ">=6.14" ipython = "*" jupyter-client = ">=7.0.0" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0.0 || >=5.1.0" prompt-toolkit = ">=3.0.30" pygments = "*" pyzmq = ">=17" @@ -1114,6 +1154,7 @@ test = ["flaky", "pexpect", "pytest"] name = "jupyter-core" version = "5.3.1" description = "Jupyter core package. A base package on which Jupyter projects rely." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1134,6 +1175,7 @@ test = ["ipykernel", "pre-commit", "pytest", "pytest-cov", "pytest-timeout"] name = "jupyter-events" version = "0.7.0" description = "Jupyter Event System library" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1159,6 +1201,7 @@ test = ["click", "pre-commit", "pytest (>=7.0)", "pytest-asyncio (>=0.19.0)", "p name = "jupyter-lsp" version = "2.2.0" description = "Multi-Language Server WebSocket proxy for Jupyter Notebook/Lab server" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1174,6 +1217,7 @@ jupyter-server = ">=1.1.2" name = "jupyter-server" version = "2.7.3" description = "The backend—i.e. core services, APIs, and REST endpoints—to Jupyter web applications." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1186,7 +1230,7 @@ anyio = ">=3.1.0" argon2-cffi = "*" jinja2 = "*" jupyter-client = ">=7.4.4" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0.0 || >=5.1.0" jupyter-events = ">=0.6.0" jupyter-server-terminals = "*" nbconvert = ">=6.4.4" @@ -1210,6 +1254,7 @@ test = ["flaky", "ipykernel", "pre-commit", "pytest (>=7.0)", "pytest-console-sc name = "jupyter-server-terminals" version = "0.4.4" description = "A Jupyter Server Extension Providing Terminals." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1229,6 +1274,7 @@ test = ["coverage", "jupyter-server (>=2.0.0)", "pytest (>=7.0)", "pytest-cov", name = "jupyterlab" version = "4.0.5" description = "JupyterLab computational environment" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1261,6 +1307,7 @@ test = ["coverage", "pytest (>=7.0)", "pytest-check-links (>=0.7)", "pytest-cons name = "jupyterlab-pygments" version = "0.2.2" description = "Pygments theme using JupyterLab CSS variables" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1272,6 +1319,7 @@ files = [ name = "jupyterlab-server" version = "2.24.0" description = "A set of server components for JupyterLab and JupyterLab like applications." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1298,6 +1346,7 @@ test = ["hatch", "ipykernel", "jupyterlab-server[openapi]", "openapi-spec-valida name = "jupyterlab-widgets" version = "3.0.8" description = "Jupyter interactive widgets for JupyterLab" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1309,6 +1358,7 @@ files = [ name = "jupytext" version = "1.15.1" description = "Jupyter notebooks as Markdown documents, Julia, Python or R scripts" +category = "dev" optional = false python-versions = "~=3.6" files = [ @@ -1331,6 +1381,7 @@ toml = ["toml"] name = "kiwisolver" version = "1.4.5" description = "A fast implementation of the Cassowary constraint solver" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1444,6 +1495,7 @@ files = [ name = "lxml" version = "4.9.3" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" files = [ @@ -1551,6 +1603,7 @@ source = ["Cython (>=0.29.35)"] name = "markdown" version = "3.4.4" description = "Python implementation of John Gruber's Markdown." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1569,6 +1622,7 @@ testing = ["coverage", "pyyaml"] name = "markdown-it-py" version = "3.0.0" description = "Python port of markdown-it. Markdown parsing, done right!" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1593,6 +1647,7 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] name = "markupsafe" version = "2.1.3" description = "Safely add untrusted strings to HTML/XML markup." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1616,6 +1671,16 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -1652,6 +1717,7 @@ files = [ name = "matplotlib" version = "3.7.2" description = "Python plotting package" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1714,6 +1780,7 @@ python-dateutil = ">=2.7" name = "matplotlib-inline" version = "0.1.6" description = "Inline Matplotlib backend for Jupyter" +category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1728,6 +1795,7 @@ traitlets = "*" name = "mdit-py-plugins" version = "0.4.0" description = "Collection of plugins for markdown-it-py" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1747,6 +1815,7 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] name = "mdurl" version = "0.1.2" description = "Markdown URL utilities" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1758,6 +1827,7 @@ files = [ name = "mergedeep" version = "1.3.4" description = "A deep merge function for 🐍." +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1769,6 +1839,7 @@ files = [ name = "mistune" version = "0.8.4" description = "The fastest markdown parser in pure Python" +category = "dev" optional = false python-versions = "*" files = [ @@ -1780,6 +1851,7 @@ files = [ name = "mkdocs" version = "1.5.2" description = "Project documentation with Markdown." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1811,6 +1883,7 @@ min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4)", "ghp-imp name = "mkdocs-jupyter" version = "0.22.0" description = "Use Jupyter in mkdocs websites" +category = "dev" optional = false python-versions = ">=3.7.1,<4" files = [ @@ -1828,6 +1901,7 @@ Pygments = ">=2.12.0,<3.0.0" name = "mkdocs-material" version = "8.5.11" description = "Documentation that simply works" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1848,6 +1922,7 @@ requests = ">=2.26" name = "mkdocs-material-extensions" version = "1.1.1" description = "Extension pack for Python Markdown and MkDocs Material." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1855,67 +1930,11 @@ files = [ {file = "mkdocs_material_extensions-1.1.1.tar.gz", hash = "sha256:9c003da71e2cc2493d910237448c672e00cefc800d3d6ae93d2fc69979e3bd93"}, ] -[[package]] -name = "mypy" -version = "1.5.1" -description = "Optional static typing for Python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "mypy-1.5.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f33592ddf9655a4894aef22d134de7393e95fcbdc2d15c1ab65828eee5c66c70"}, - {file = "mypy-1.5.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:258b22210a4a258ccd077426c7a181d789d1121aca6db73a83f79372f5569ae0"}, - {file = "mypy-1.5.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9ec1f695f0c25986e6f7f8778e5ce61659063268836a38c951200c57479cc12"}, - {file = "mypy-1.5.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:abed92d9c8f08643c7d831300b739562b0a6c9fcb028d211134fc9ab20ccad5d"}, - {file = "mypy-1.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:a156e6390944c265eb56afa67c74c0636f10283429171018446b732f1a05af25"}, - {file = "mypy-1.5.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6ac9c21bfe7bc9f7f1b6fae441746e6a106e48fc9de530dea29e8cd37a2c0cc4"}, - {file = "mypy-1.5.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:51cb1323064b1099e177098cb939eab2da42fea5d818d40113957ec954fc85f4"}, - {file = "mypy-1.5.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:596fae69f2bfcb7305808c75c00f81fe2829b6236eadda536f00610ac5ec2243"}, - {file = "mypy-1.5.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:32cb59609b0534f0bd67faebb6e022fe534bdb0e2ecab4290d683d248be1b275"}, - {file = "mypy-1.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:159aa9acb16086b79bbb0016145034a1a05360626046a929f84579ce1666b315"}, - {file = "mypy-1.5.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f6b0e77db9ff4fda74de7df13f30016a0a663928d669c9f2c057048ba44f09bb"}, - {file = "mypy-1.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:26f71b535dfc158a71264e6dc805a9f8d2e60b67215ca0bfa26e2e1aa4d4d373"}, - {file = "mypy-1.5.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fc3a600f749b1008cc75e02b6fb3d4db8dbcca2d733030fe7a3b3502902f161"}, - {file = "mypy-1.5.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:26fb32e4d4afa205b24bf645eddfbb36a1e17e995c5c99d6d00edb24b693406a"}, - {file = "mypy-1.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:82cb6193de9bbb3844bab4c7cf80e6227d5225cc7625b068a06d005d861ad5f1"}, - {file = "mypy-1.5.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4a465ea2ca12804d5b34bb056be3a29dc47aea5973b892d0417c6a10a40b2d65"}, - {file = "mypy-1.5.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9fece120dbb041771a63eb95e4896791386fe287fefb2837258925b8326d6160"}, - {file = "mypy-1.5.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d28ddc3e3dfeab553e743e532fb95b4e6afad51d4706dd22f28e1e5e664828d2"}, - {file = "mypy-1.5.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:57b10c56016adce71fba6bc6e9fd45d8083f74361f629390c556738565af8eeb"}, - {file = "mypy-1.5.1-cp38-cp38-win_amd64.whl", hash = "sha256:ff0cedc84184115202475bbb46dd99f8dcb87fe24d5d0ddfc0fe6b8575c88d2f"}, - {file = "mypy-1.5.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8f772942d372c8cbac575be99f9cc9d9fb3bd95c8bc2de6c01411e2c84ebca8a"}, - {file = "mypy-1.5.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5d627124700b92b6bbaa99f27cbe615c8ea7b3402960f6372ea7d65faf376c14"}, - {file = "mypy-1.5.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:361da43c4f5a96173220eb53340ace68cda81845cd88218f8862dfb0adc8cddb"}, - {file = "mypy-1.5.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:330857f9507c24de5c5724235e66858f8364a0693894342485e543f5b07c8693"}, - {file = "mypy-1.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:c543214ffdd422623e9fedd0869166c2f16affe4ba37463975043ef7d2ea8770"}, - {file = "mypy-1.5.1-py3-none-any.whl", hash = "sha256:f757063a83970d67c444f6e01d9550a7402322af3557ce7630d3c957386fa8f5"}, - {file = "mypy-1.5.1.tar.gz", hash = "sha256:b031b9601f1060bf1281feab89697324726ba0c0bae9d7cd7ab4b690940f0b92"}, -] - -[package.dependencies] -mypy-extensions = ">=1.0.0" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typing-extensions = ">=4.1.0" - -[package.extras] -dmypy = ["psutil (>=4.0)"] -install-types = ["pip"] -reports = ["lxml"] - -[[package]] -name = "mypy-extensions" -version = "1.0.0" -description = "Type system extensions for programs checked with the mypy type checker." -optional = false -python-versions = ">=3.5" -files = [ - {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, - {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, -] - [[package]] name = "nbclient" version = "0.8.0" description = "A client library for executing notebooks. Formerly nbconvert's ExecutePreprocessor." +category = "dev" optional = false python-versions = ">=3.8.0" files = [ @@ -1925,7 +1944,7 @@ files = [ [package.dependencies] jupyter-client = ">=6.1.12" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0.0 || >=5.1.0" nbformat = ">=5.1" traitlets = ">=5.4" @@ -1938,6 +1957,7 @@ test = ["flaky", "ipykernel (>=6.19.3)", "ipython", "ipywidgets", "nbconvert (>= name = "nbconvert" version = "6.5.4" description = "Converting Jupyter Notebooks" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1975,6 +1995,7 @@ webpdf = ["pyppeteer (>=1,<1.1)"] name = "nbformat" version = "5.9.2" description = "The Jupyter Notebook format" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1996,6 +2017,7 @@ test = ["pep440", "pre-commit", "pytest", "testpath"] name = "nest-asyncio" version = "1.5.7" description = "Patch asyncio to allow nested event loops" +category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -2007,6 +2029,7 @@ files = [ name = "notebook" version = "7.0.3" description = "Jupyter Notebook - A web-based notebook environment for interactive computing" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2030,6 +2053,7 @@ test = ["importlib-resources (>=5.0)", "ipykernel", "jupyter-server[test] (>=2.4 name = "notebook-shim" version = "0.2.3" description = "A shim layer for notebook traits and config" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2047,6 +2071,7 @@ test = ["pytest", "pytest-console-scripts", "pytest-jupyter", "pytest-tornasync" name = "numpy" version = "1.25.2" description = "Fundamental package for array computing in Python" +category = "main" optional = false python-versions = ">=3.9" files = [ @@ -2081,6 +2106,7 @@ files = [ name = "overrides" version = "7.4.0" description = "A decorator to automatically detect mismatch when overriding a method." +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2092,6 +2118,7 @@ files = [ name = "packaging" version = "23.1" description = "Core utilities for Python packages" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2099,58 +2126,11 @@ files = [ {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, ] -[[package]] -name = "pandas" -version = "1.5.3" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.8" -files = [ - {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3749077d86e3a2f0ed51367f30bf5b82e131cc0f14260c4d3e499186fccc4406"}, - {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:972d8a45395f2a2d26733eb8d0f629b2f90bebe8e8eddbb8829b180c09639572"}, - {file = "pandas-1.5.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:50869a35cbb0f2e0cd5ec04b191e7b12ed688874bd05dd777c19b28cbea90996"}, - {file = "pandas-1.5.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3ac844a0fe00bfaeb2c9b51ab1424e5c8744f89860b138434a363b1f620f354"}, - {file = "pandas-1.5.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a0a56cef15fd1586726dace5616db75ebcfec9179a3a55e78f72c5639fa2a23"}, - {file = "pandas-1.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:478ff646ca42b20376e4ed3fa2e8d7341e8a63105586efe54fa2508ee087f328"}, - {file = "pandas-1.5.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6973549c01ca91ec96199e940495219c887ea815b2083722821f1d7abfa2b4dc"}, - {file = "pandas-1.5.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c39a8da13cede5adcd3be1182883aea1c925476f4e84b2807a46e2775306305d"}, - {file = "pandas-1.5.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc"}, - {file = "pandas-1.5.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae"}, - {file = "pandas-1.5.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f2b952406a1588ad4cad5b3f55f520e82e902388a6d5a4a91baa8d38d23c7f6"}, - {file = "pandas-1.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:bc4c368f42b551bf72fac35c5128963a171b40dce866fb066540eeaf46faa003"}, - {file = "pandas-1.5.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:14e45300521902689a81f3f41386dc86f19b8ba8dd5ac5a3c7010ef8d2932813"}, - {file = "pandas-1.5.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9842b6f4b8479e41968eced654487258ed81df7d1c9b7b870ceea24ed9459b31"}, - {file = "pandas-1.5.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:26d9c71772c7afb9d5046e6e9cf42d83dd147b5cf5bcb9d97252077118543792"}, - {file = "pandas-1.5.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fbcb19d6fceb9e946b3e23258757c7b225ba450990d9ed63ccceeb8cae609f7"}, - {file = "pandas-1.5.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:565fa34a5434d38e9d250af3c12ff931abaf88050551d9fbcdfafca50d62babf"}, - {file = "pandas-1.5.3-cp38-cp38-win32.whl", hash = "sha256:87bd9c03da1ac870a6d2c8902a0e1fd4267ca00f13bc494c9e5a9020920e1d51"}, - {file = "pandas-1.5.3-cp38-cp38-win_amd64.whl", hash = "sha256:41179ce559943d83a9b4bbacb736b04c928b095b5f25dd2b7389eda08f46f373"}, - {file = "pandas-1.5.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c74a62747864ed568f5a82a49a23a8d7fe171d0c69038b38cedf0976831296fa"}, - {file = "pandas-1.5.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c4c00e0b0597c8e4f59e8d461f797e5d70b4d025880516a8261b2817c47759ee"}, - {file = "pandas-1.5.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a50d9a4336a9621cab7b8eb3fb11adb82de58f9b91d84c2cd526576b881a0c5a"}, - {file = "pandas-1.5.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd05f7783b3274aa206a1af06f0ceed3f9b412cf665b7247eacd83be41cf7bf0"}, - {file = "pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f69c4029613de47816b1bb30ff5ac778686688751a5e9c99ad8c7031f6508e5"}, - {file = "pandas-1.5.3-cp39-cp39-win32.whl", hash = "sha256:7cec0bee9f294e5de5bbfc14d0573f65526071029d036b753ee6507d2a21480a"}, - {file = "pandas-1.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:dfd681c5dc216037e0b0a2c821f5ed99ba9f03ebcf119c7dac0e9a7b960b9ec9"}, - {file = "pandas-1.5.3.tar.gz", hash = "sha256:74a3fd7e5a7ec052f183273dc7b0acd3a863edf7520f5d3a1765c04ffdb3b0b1"}, -] - -[package.dependencies] -numpy = [ - {version = ">=1.20.3", markers = "python_version < \"3.10\""}, - {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, - {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, -] -python-dateutil = ">=2.8.1" -pytz = ">=2020.1" - -[package.extras] -test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] - [[package]] name = "pandocfilters" version = "1.5.0" description = "Utilities for writing pandoc filters in python" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -2162,6 +2142,7 @@ files = [ name = "parso" version = "0.8.3" description = "A Python Parser" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2177,6 +2158,7 @@ testing = ["docopt", "pytest (<6.0.0)"] name = "pathspec" version = "0.11.2" description = "Utility library for gitignore style pattern matching of file paths." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2188,6 +2170,7 @@ files = [ name = "pexpect" version = "4.8.0" description = "Pexpect allows easy control of interactive console applications." +category = "dev" optional = false python-versions = "*" files = [ @@ -2202,6 +2185,7 @@ ptyprocess = ">=0.5" name = "pickleshare" version = "0.7.5" description = "Tiny 'shelve'-like database with concurrency support" +category = "dev" optional = false python-versions = "*" files = [ @@ -2213,6 +2197,7 @@ files = [ name = "pillow" version = "10.0.0" description = "Python Imaging Library (Fork)" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2282,6 +2267,7 @@ tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "pa name = "platformdirs" version = "3.10.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2297,6 +2283,7 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-co name = "pluggy" version = "1.3.0" description = "plugin and hook calling mechanisms for python" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2312,6 +2299,7 @@ testing = ["pytest", "pytest-benchmark"] name = "prometheus-client" version = "0.17.1" description = "Python client for the Prometheus monitoring system." +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2326,6 +2314,7 @@ twisted = ["twisted"] name = "prompt-toolkit" version = "3.0.39" description = "Library for building powerful interactive command lines in Python" +category = "dev" optional = false python-versions = ">=3.7.0" files = [ @@ -2340,6 +2329,7 @@ wcwidth = "*" name = "psutil" version = "5.9.5" description = "Cross-platform lib for process and system monitoring in Python." +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -2366,6 +2356,7 @@ test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] name = "ptyprocess" version = "0.7.0" description = "Run a subprocess in a pseudo terminal" +category = "dev" optional = false python-versions = "*" files = [ @@ -2377,6 +2368,7 @@ files = [ name = "pure-eval" version = "0.2.2" description = "Safely evaluate AST nodes without side effects" +category = "dev" optional = false python-versions = "*" files = [ @@ -2391,6 +2383,7 @@ tests = ["pytest"] name = "pycparser" version = "2.21" description = "C parser in Python" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -2402,6 +2395,7 @@ files = [ name = "pygments" version = "2.16.1" description = "Pygments is a syntax highlighting package written in Python." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2416,6 +2410,7 @@ plugins = ["importlib-metadata"] name = "pymdown-extensions" version = "10.3" description = "Extension pack for Python Markdown." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2434,6 +2429,7 @@ extra = ["pygments (>=2.12)"] name = "pyparsing" version = "3.0.9" description = "pyparsing module - Classes and methods to define and execute parsing grammars" +category = "dev" optional = false python-versions = ">=3.6.8" files = [ @@ -2448,6 +2444,7 @@ diagrams = ["jinja2", "railroad-diagrams"] name = "pytest" version = "7.4.1" description = "pytest: simple powerful testing with Python" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2470,6 +2467,7 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no name = "pytest-cov" version = "4.1.0" description = "Pytest plugin for measuring coverage." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2484,10 +2482,26 @@ pytest = ">=4.6" [package.extras] testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] +[[package]] +name = "pytest-datadir" +version = "1.4.1" +description = "pytest plugin for test data directories and files" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-datadir-1.4.1.tar.gz", hash = "sha256:9f7a3c4def6ac4cac3cc8181139ab53bd2667231052bd40cb07081748d4420f0"}, + {file = "pytest_datadir-1.4.1-py3-none-any.whl", hash = "sha256:095f441782b1b907587eca7227fdbae94be43f1c96b4b2cbcc6801a4645be1af"}, +] + +[package.dependencies] +pytest = ">=5.0" + [[package]] name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" +category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -2502,6 +2516,7 @@ six = ">=1.5" name = "python-json-logger" version = "2.0.7" description = "A python library adding a json log formatter" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2509,21 +2524,11 @@ files = [ {file = "python_json_logger-2.0.7-py3-none-any.whl", hash = "sha256:f380b826a991ebbe3de4d897aeec42760035ac760345e57b812938dc8b35e2bd"}, ] -[[package]] -name = "pytz" -version = "2023.3.post1" -description = "World timezone definitions, modern and historical" -optional = false -python-versions = "*" -files = [ - {file = "pytz-2023.3.post1-py2.py3-none-any.whl", hash = "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7"}, - {file = "pytz-2023.3.post1.tar.gz", hash = "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b"}, -] - [[package]] name = "pywin32" version = "306" description = "Python for Window Extensions" +category = "dev" optional = false python-versions = "*" files = [ @@ -2547,6 +2552,7 @@ files = [ name = "pywinpty" version = "2.0.11" description = "Pseudo terminal support for Windows from Python." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2561,6 +2567,7 @@ files = [ name = "pyyaml" version = "6.0.1" description = "YAML parser and emitter for Python" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2620,6 +2627,7 @@ files = [ name = "pyyaml-env-tag" version = "0.1" description = "A custom YAML tag for referencing environment variables in YAML files. " +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2634,6 +2642,7 @@ pyyaml = "*" name = "pyzmq" version = "25.1.1" description = "Python bindings for 0MQ" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2739,6 +2748,7 @@ cffi = {version = "*", markers = "implementation_name == \"pypy\""} name = "qtconsole" version = "5.4.4" description = "Jupyter Qt console" +category = "dev" optional = false python-versions = ">= 3.7" files = [ @@ -2765,6 +2775,7 @@ test = ["flaky", "pytest", "pytest-qt"] name = "qtpy" version = "2.4.0" description = "Provides an abstraction layer on top of the various Qt bindings (PyQt5/6 and PySide2/6)." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2782,6 +2793,7 @@ test = ["pytest (>=6,!=7.0.0,!=7.0.1)", "pytest-cov (>=3.0.0)", "pytest-qt"] name = "referencing" version = "0.30.2" description = "JSON Referencing + Python" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2797,6 +2809,7 @@ rpds-py = ">=0.7.0" name = "requests" version = "2.31.0" description = "Python HTTP for Humans." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2818,6 +2831,7 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "rfc3339-validator" version = "0.1.4" description = "A pure python RFC3339 validator" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -2832,6 +2846,7 @@ six = "*" name = "rfc3986-validator" version = "0.1.1" description = "Pure python rfc3986 validator" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -2843,6 +2858,7 @@ files = [ name = "rpds-py" version = "0.10.2" description = "Python bindings to Rust's persistent data structures (rpds)" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2949,6 +2965,7 @@ files = [ name = "send2trash" version = "1.8.2" description = "Send file to trash natively under Mac OS X, Windows and Linux" +category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" files = [ @@ -2965,6 +2982,7 @@ win32 = ["pywin32"] name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -2976,6 +2994,7 @@ files = [ name = "sniffio" version = "1.3.0" description = "Sniff out which async library your code is running under" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2987,6 +3006,7 @@ files = [ name = "soupsieve" version = "2.5" description = "A modern CSS selector implementation for Beautiful Soup." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2998,6 +3018,7 @@ files = [ name = "stack-data" version = "0.6.2" description = "Extract data from python stack frames and tracebacks for informative displays" +category = "dev" optional = false python-versions = "*" files = [ @@ -3017,6 +3038,7 @@ tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"] name = "terminado" version = "0.17.1" description = "Tornado websocket backend for the Xterm.js Javascript terminal emulator library." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3037,6 +3059,7 @@ test = ["pre-commit", "pytest (>=7.0)", "pytest-timeout"] name = "tinycss2" version = "1.2.1" description = "A tiny CSS parser" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3055,6 +3078,7 @@ test = ["flake8", "isort", "pytest"] name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" +category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -3066,6 +3090,7 @@ files = [ name = "tomli" version = "2.0.1" description = "A lil' TOML parser" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3077,6 +3102,7 @@ files = [ name = "tornado" version = "6.3.3" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." +category = "dev" optional = false python-versions = ">= 3.8" files = [ @@ -3097,6 +3123,7 @@ files = [ name = "traitlets" version = "5.9.0" description = "Traitlets Python configuration system" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3112,6 +3139,7 @@ test = ["argcomplete (>=2.0)", "pre-commit", "pytest", "pytest-mock"] name = "typing-extensions" version = "4.7.1" description = "Backported and Experimental Type Hints for Python 3.7+" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3123,6 +3151,7 @@ files = [ name = "uri-template" version = "1.3.0" description = "RFC 6570 URI Template Processor" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3137,6 +3166,7 @@ dev = ["flake8", "flake8-annotations", "flake8-bandit", "flake8-bugbear", "flake name = "urllib3" version = "2.0.4" description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3154,6 +3184,7 @@ zstd = ["zstandard (>=0.18.0)"] name = "watchdog" version = "3.0.0" description = "Filesystem events monitoring" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3193,6 +3224,7 @@ watchmedo = ["PyYAML (>=3.10)"] name = "wcwidth" version = "0.2.6" description = "Measures the displayed width of unicode strings in a terminal" +category = "dev" optional = false python-versions = "*" files = [ @@ -3204,6 +3236,7 @@ files = [ name = "webcolors" version = "1.13" description = "A library for working with the color formats defined by HTML and CSS." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3219,6 +3252,7 @@ tests = ["pytest", "pytest-cov"] name = "webencodings" version = "0.5.1" description = "Character encoding aliases for legacy web content" +category = "dev" optional = false python-versions = "*" files = [ @@ -3230,6 +3264,7 @@ files = [ name = "websocket-client" version = "1.6.2" description = "WebSocket client for Python with low level API options" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -3246,6 +3281,7 @@ test = ["websockets"] name = "widgetsnbextension" version = "4.0.8" description = "Jupyter interactive widgets for Jupyter Notebook" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3257,6 +3293,7 @@ files = [ name = "zipp" version = "3.16.2" description = "Backport of pathlib-compatible object wrapper for zip files" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -3271,4 +3308,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "c8492034a4267e0f75c451d34ddc2b743e86e0d58ff41000f5e3f4bee14480c3" +content-hash = "e20b9ab0e24f8c778e5e3f3459237150a7c85de1860c7272c4b6adcbb7209f62" diff --git a/pyproject.toml b/pyproject.toml index b77a1a2..48da919 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,23 +17,18 @@ repository = "https://github.com/biomarkersParkinson/tsdf" [tool.poetry.dependencies] python = "^3.9" -flatten-json = "^0.1.13" numpy = "^1.24.1" -mypy = "^1.0.1" -mkdocs = "^1.4.2" -mkdocs-jupyter = "^0.22.0" - - -[tool.poetry.dev-dependencies] [tool.poetry.group.dev.dependencies] pytest = "^7.2.0" +pytest-cov = "^4.0.0" +pytest-datadir = "^1.4.1" jupyter = "^1.0.0" ipykernel = "^6.19.2" coverage = "^7.0.0" -pytest-cov = "^4.0.0" -pandas = "^1.5.2" matplotlib = "^3.6.3" +mkdocs = "^1.4.2" +mkdocs-jupyter = "^0.22.0" [tool.poetry.scripts] validate-tsdf = "tsdf.validator:main" diff --git a/src/tsdf/__init__.py b/src/tsdf/__init__.py index 8a1b636..c2b85b9 100644 --- a/src/tsdf/__init__.py +++ b/src/tsdf/__init__.py @@ -6,6 +6,7 @@ from .read_tsdf import ( load_metadata_file, load_metadata_from_path, + load_metadatas_from_dir, load_metadata_string, load_metadata_legacy_file ) @@ -21,3 +22,16 @@ ) from .tsdfmetadata import TSDFMetadata + +__all__ = [ + 'load_metadata_file', + 'load_metadata_from_path', + 'load_metadatas_from_dir', + 'load_metadata_string', + 'load_metadata_legacy_file', + 'write_metadata', + 'write_binary_file', + 'load_binary_from_metadata', + 'TSDFMetadata', + 'constants' +] diff --git a/src/tsdf/constants.py b/src/tsdf/constants.py index 11e6e9e..b6544b4 100644 --- a/src/tsdf/constants.py +++ b/src/tsdf/constants.py @@ -55,28 +55,3 @@ METADATA_NAMING_PATTERN = "**meta.json" """ Naming convention for the metadata files. ** allows for any prefix, including additional directories. """ - -class TestConstants: - """Class containing constants used for testing and demonstration.""" - - TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "..", "..", "tests", "data") - - """ Path to the test data directory. """ - TEST_OUTPUT_DATA_DIR = os.path.join(TEST_DATA_DIR, "outputs") - """ Path to the test output data directory. """ - - TEST_DATA_FILES = { - "flat": os.path.join(TEST_DATA_DIR, "flat_meta.json"), - "hierarchical": os.path.join(TEST_DATA_DIR, "hierarchical_meta.json"), - "wrongversion": os.path.join(TEST_DATA_DIR, "wrongversion_meta_fail.json"), - "missingkey": os.path.join(TEST_DATA_DIR, "missingkey_meta_fail.json"), - "example_10_3_int16": os.path.join(TEST_DATA_DIR, "example_10_3_int16_meta.json"), - "ppp": os.path.join(TEST_DATA_DIR, "ppp_format_meta.json"), - "legacy": os.path.join(TEST_DATA_DIR, "ppp_format_meta_legacy.json"), - } - """ Dictionary used for accessing test data files. """ - - METADATA_EXTENSION = "_meta.json" - """ Suffix and extension used to denote metadata files. """ - - BINARY_EXTENSION = ".bin" diff --git a/src/tsdf/read_binary.py b/src/tsdf/read_binary.py index 2f36de8..e8b74bb 100644 --- a/src/tsdf/read_binary.py +++ b/src/tsdf/read_binary.py @@ -11,17 +11,18 @@ def load_binary_from_metadata( - metadata_dir: str, metadata: 'tsdfmetadata.TSDFMetadata', start_row: int = 0, end_row: int = -1 + metadata: 'tsdfmetadata.TSDFMetadata', start_row: int = 0, end_row: int = -1 ) -> np.ndarray: """ - Use metadata properties to load and return numpy array from a binary file. - - :param metadata_dir: path to the directory containing the TSDF metadata files. + Use metadata properties to load and return numpy array from a binary file (located the same directory where the metadata is saved). + :param metadata: TSDFMetadata object. :param start_row: (optional) first row to load. :param end_row: (optional) last row to load. If -1, load all rows. :return: numpy array containing the data.""" + metadata_dir = metadata.file_dir_path + bin_path = os.path.join(metadata_dir, metadata.file_name) return _load_binary_file( bin_path, diff --git a/src/tsdf/tsdfmetadata.py b/src/tsdf/tsdfmetadata.py index e9f107a..b47fb0e 100644 --- a/src/tsdf/tsdfmetadata.py +++ b/src/tsdf/tsdfmetadata.py @@ -78,11 +78,3 @@ def get_plain_tsdf_dict_copy(self) -> Dict[str, Any]: if simple_dict.get("source_file_name") is not None: simple_dict.pop("source_file_name") return simple_dict - - def load_binary(self) -> ndarray: - """ - Load the binary file from the same directory where the metadata is saved. - - :return: binary file as a numpy array. - """ - return read_binary.load_binary_from_metadata(self.file_dir_path, self) diff --git a/src/tsdf/validator.py b/src/tsdf/validator.py index 3bb7a41..7cb7751 100644 --- a/src/tsdf/validator.py +++ b/src/tsdf/validator.py @@ -20,7 +20,7 @@ def validate_tsdf_format(file_path): # print(json.dumps(file_metadata.get_plain_tsdf_dict_copy(), indent=4)) # Load the binary data - binary_data = read_binary.load_binary_from_metadata(abs_dir, file_metadata) + binary_data = read_binary.load_binary_from_metadata(file_metadata) # Success message print(f"Successfully loaded binary file {file_name}, resulting shape: {binary_data.shape}") diff --git a/tests/data/outputs/.gitignore b/tests/data/outputs/.gitignore deleted file mode 100644 index a34ca7a..0000000 --- a/tests/data/outputs/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# The directory is used to store local test outputs. Ignore all generated files. -tmp_* diff --git a/tests/test_legacy_tsdf_utils.py b/tests/test_legacy_tsdf_utils.py index aec034c..3ebd217 100644 --- a/tests/test_legacy_tsdf_utils.py +++ b/tests/test_legacy_tsdf_utils.py @@ -1,39 +1,26 @@ -import os -import unittest -from tsdf import read_tsdf +import tsdf from tsdf import legacy_tsdf_utils -from tsdf.constants import TestConstants as CONST -class TestConversion(unittest.TestCase): +def test_conversion(shared_datadir): """Test whether the conversion from TSDB (legacy metadata format) to TSDF works. """ - def test_conversion(self): - path_to_file = os.path.join(CONST.TEST_DATA_DIR, "ppp_format_meta_legacy.json") - path_to_new_file = os.path.join( - CONST.TEST_OUTPUT_DATA_DIR, "tmp_test_ppp_format_meta.json" - ) + path_to_file = shared_datadir / "ppp_format_meta_legacy.json" + path_to_new_file = shared_datadir / "tmp_test_ppp_format_meta.json" + path_to_existing_tsdf_file = shared_datadir / "ppp_format_meta.json" - path_to_existing_tsdf_file = os.path.join( - CONST.TEST_DATA_DIR, "ppp_format_meta.json" - ) + # Generate a TSDF metadata file from TSDB + legacy_tsdf_utils.generate_tsdf_metadata_from_tsdb(path_to_file, path_to_new_file) - # Generate a TSDF metadata file from TSDB - legacy_tsdf_utils.generate_tsdf_metadata_from_tsdb(path_to_file, path_to_new_file) + # Load the generated metadata file + new_meta = tsdf.load_metadata_from_path(path_to_new_file) - # Load the generated metadata file - new_meta = read_tsdf.load_metadata_from_path(path_to_new_file) + # Load the existing metadata file + existing_meta = tsdf.load_metadata_from_path(path_to_existing_tsdf_file) - # Load the existing metadata file - existing_meta = read_tsdf.load_metadata_from_path(path_to_existing_tsdf_file) + # Compare the two metadata files (whether the mapped TSDFs fields are the same) + assert(new_meta["ppp_format_time.bin"].get_plain_tsdf_dict_copy() == + existing_meta["ppp_format_time.bin"].get_plain_tsdf_dict_copy()) - # Compare the two metadata files (whether the mapped TSDFs fields are the same) - self.assertEqual( - new_meta["ppp_format_time.bin"].get_plain_tsdf_dict_copy(), - existing_meta["ppp_format_time.bin"].get_plain_tsdf_dict_copy(), - ) - - self.assertEqual( - new_meta["ppp_format_samples.bin"].get_plain_tsdf_dict_copy(), - existing_meta["ppp_format_samples.bin"].get_plain_tsdf_dict_copy(), - ) + assert(new_meta["ppp_format_samples.bin"].get_plain_tsdf_dict_copy() == + existing_meta["ppp_format_samples.bin"].get_plain_tsdf_dict_copy()) diff --git a/tests/test_parse_metadata.py b/tests/test_parse_metadata.py index 8014daa..8c7c27d 100644 --- a/tests/test_parse_metadata.py +++ b/tests/test_parse_metadata.py @@ -1,41 +1,37 @@ import json -import unittest +import pytest from tsdf import parse_metadata from tsdf.tsdfmetadata import TSDFMetadataFieldError, TSDFMetadataFieldValueError -from tsdf.constants import TestConstants as CONST -class TestWrongFormatting(unittest.TestCase): - """Test whether the exceptions are thrown in case the metadata file is not well annotated.""" +def test_load_wrong_version(shared_datadir): + """Test that a file with a wrong version raises an exception.""" + + path = shared_datadir / "wrongversion_meta_fail.json" + with open(path, "r") as file: + data = json.load(file) + with pytest.raises(TSDFMetadataFieldValueError): + parse_metadata.read_data(data, path) # This should trigger an exception - def test_load_wrong_version(self): - """Test that a file with a wrong version raises an exception.""" - path = CONST.TEST_DATA_FILES["wrongversion"] - with open(path, "r") as file: - with self.assertRaises(TSDFMetadataFieldValueError) as context: - data = json.load(file) - parse_metadata.read_data(data, path) # This should trigger an exception +def test_load_missing_key(shared_datadir): + """Test that a file with a missing mandatory key raises an exception.""" - def test_load_missing_key(self): - """Test that a file with a missing mandatory key raises an exception.""" - path = CONST.TEST_DATA_FILES["missingkey"] - with open(path, "r") as file: - with self.assertRaises(TSDFMetadataFieldError) as context: - data = json.load(file) - parse_metadata.read_data(data, path) # This should trigger an exception + path = shared_datadir / "missingkey_meta_fail.json" + with open(path, "r") as file: + data = json.load(file) + with pytest.raises(TSDFMetadataFieldError): + parse_metadata.read_data(data, path) # This should trigger an exception -class TestTSDFMetadataParsing(unittest.TestCase): - """Test whether the TSDF objects are well specified are well defined.""" +def test_load_flat_structure(shared_datadir): + """Test parsing of a flat TSDF metadata file.""" - def test_load_flat_structure(self): - """Test parsing of a flat TSDF metadata file.""" - path = CONST.TEST_DATA_FILES["flat"] - with open(path, "r") as file: - data = json.load(file) - streams = parse_metadata.read_data(data, path) - first_stream = parse_metadata.get_file_metadata_at_index(streams, 0) - version: str = first_stream.metadata_version - for key, value in data.items(): - if parse_metadata.is_mandatory_type(key, version): - self.assertTrue(value == first_stream.__getattribute__(key)) + path = shared_datadir / "flat_meta.json" + with open(path, "r") as file: + data = json.load(file) + streams = parse_metadata.read_data(data, path) + first_stream = parse_metadata.get_file_metadata_at_index(streams, 0) + version: str = first_stream.metadata_version + for key, value in data.items(): + if parse_metadata.is_mandatory_type(key, version): + assert(value == first_stream.__getattribute__(key)) diff --git a/tests/test_read_binary.py b/tests/test_read_binary.py index 045ae7f..3cc29db 100644 --- a/tests/test_read_binary.py +++ b/tests/test_read_binary.py @@ -1,89 +1,51 @@ -import os -import unittest import numpy as np -from tsdf.constants import TestConstants as CONST -from tsdf import read_tsdf -from tsdf import read_binary +from pathlib import Path +import tsdf from tsdf import parse_metadata - - -def load_single_bin_file(dir_path: str, file_name: str) -> np.ndarray: - """ - Load a single binary file from the given directory path and file name. - - :param dir_path: The directory path where the binary file is located. - :param file_name: The name of the binary file without the extension. - - :returns: The binary data as a numpy array. - """ - path = os.path.join(dir_path, file_name + CONST.METADATA_EXTENSION) - metadata = read_tsdf.load_metadata_from_path(path) - data = read_binary.load_binary_from_metadata( - dir_path, metadata[file_name + CONST.BINARY_EXTENSION] - ) - return data - - -class TestBinaryFileReading(unittest.TestCase): - """Test reading of binary files based on the TSDF metadata.""" - - def test_load_binary_float32(self): - data = load_single_bin_file(CONST.TEST_DATA_DIR, "example_10_3_float32") - - self.assertEqual(data.shape, (10, 3)) - self.assertEqual(data.dtype, "float32") - - def test_load_binary_float64(self): - data = load_single_bin_file(CONST.TEST_DATA_DIR, "example_10_3_float64") - - self.assertEqual(data.shape, (10, 3)) - self.assertEqual(data.dtype, "float64") - - # def test_load_binary_float64_fail(self): - # """Should raise an exception on reading binary data""" - # path = os.path.join(CONST.TEST_DATA_DIR, "example_10_3_float64_meta_fail.json") - # metadata = load_tsdf.load_metadata_from_path(path) - # with self.assertRaises(Exception) as exc_context: - # io_binary.load_binary_from_metadata( - # CONST.TEST_DATA_DIR, io_metadata.get_file_metadata_at_index(metadata, 0) - # ) - # self.assertEqual( - # exc_context.exception.args[0], "Number of rows doesn't match file length." - # ) - - def test_load_binary_int16(self): - data = load_single_bin_file(CONST.TEST_DATA_DIR, "example_10_3_int16") - - self.assertEqual(data.shape, (10, 3)) - self.assertEqual(data.dtype, "int16") - - def test_load_like_ppp(self): - path = CONST.TEST_DATA_FILES["ppp"] - metadata = read_tsdf.load_metadata_from_path(path) - time_data = read_binary.load_binary_from_metadata( - CONST.TEST_DATA_DIR, parse_metadata.get_file_metadata_at_index(metadata, 0) - ) - self.assertEqual(time_data.shape, (17,)) - # time data should be loaded as float64 - self.assertEqual(time_data.dtype, "float32") - - sample_data = read_binary.load_binary_from_metadata( - CONST.TEST_DATA_DIR, parse_metadata.get_file_metadata_at_index(metadata, 1) - ) - self.assertEqual(sample_data.shape, (17, 6)) - # sample data should be loaded as int16 - self.assertEqual(sample_data.dtype, "int16") - - def test_random_access(self): - # TODO: test the new random access functionality - file_name = "example_10_3_int16" - path = os.path.join(CONST.TEST_DATA_DIR, file_name + CONST.METADATA_EXTENSION) - metadata = read_tsdf.load_metadata_from_path(path) - data = read_binary.load_binary_from_metadata( - CONST.TEST_DATA_DIR, - metadata[file_name + CONST.BINARY_EXTENSION], - 2, - 6, - ) - self.assertEqual(data.shape, (4, 3)) - self.assertEqual(data.dtype, "int16") +from utils import load_single_bin_file + + +def test_load_binary_float32(shared_datadir): + data = load_single_bin_file(shared_datadir, "example_10_3_float32") + assert(data.shape == (10, 3)) + assert(data.dtype == "float32") + +def test_load_binary_float64(shared_datadir): + data = load_single_bin_file(shared_datadir, "example_10_3_float64") + assert(data.shape == (10, 3)) + assert(data.dtype == "float64") + +# def test_load_binary_float64_fail(shared_datadir): +# """Should raise an exception on reading binary data""" +# path = shared_datadir / "example_10_3_float64_meta_fail.json" +# metadata = load_tsdf.load_metadata_from_path(path) +# with self.assertRaises(Exception) as exc_context: +# io_binary.load_binary_from_metadata( +# shared_datadir, io_metadata.get_file_metadata_at_index(metadata, 0) +# ) +# assert(exc_context.exception.args[0] == "Number of rows doesn't match file length.") + +def test_load_binary_int16(shared_datadir): + data = load_single_bin_file(shared_datadir, "example_10_3_int16") + assert(data.shape == (10, 3)) + assert(data.dtype == "int16") + +def test_load_like_ppp(shared_datadir): + metadata = tsdf.load_metadata_from_path(shared_datadir / "ppp_format_meta.json") + time_data = tsdf.load_binary_from_metadata(parse_metadata.get_file_metadata_at_index(metadata, 0)) + assert(time_data.shape == (17,)) + # time data should be loaded as float64 + assert(time_data.dtype == "float32") + + sample_data = tsdf.load_binary_from_metadata(parse_metadata.get_file_metadata_at_index(metadata, 1)) + assert(sample_data.shape == (17, 6)) + # sample data should be loaded as int16 + assert(sample_data.dtype == "int16") + +def test_random_access(shared_datadir): + # TODO: test the new random access functionality + name = "example_10_3_int16" + metadata = tsdf.load_metadata_from_path(shared_datadir / (name + "_meta.json")) + data = tsdf.load_binary_from_metadata(metadata[name + ".bin"], 2, 6) + assert(data.shape == (4, 3)) + assert(data.dtype == "int16") diff --git a/tests/test_read_tsdf.py b/tests/test_read_tsdf.py index 6f0a706..33aca70 100644 --- a/tests/test_read_tsdf.py +++ b/tests/test_read_tsdf.py @@ -1,34 +1,31 @@ -import unittest -from tsdf.constants import TestConstants as CONST -from tsdf import read_tsdf -class TestMetadataFileReading(unittest.TestCase): - """Test loading of the metadata file.""" +import tsdf - def test_load_metadata_file(self): - """Test that a json file gets loaded correctly.""" - with open(CONST.TEST_DATA_FILES["hierarchical"], "r") as file: - data = read_tsdf.load_metadata_file(file) - self.assertEqual(len(data), 4) - def test_load_metadata_legacy_file(self): - """Test that a json file gets loaded correctly.""" - with open(CONST.TEST_DATA_FILES["legacy"], "r") as file: - data = read_tsdf.load_metadata_legacy_file(file) - self.assertEqual(len(data), 2) +def test_load_metadata_file(shared_datadir): + """Test that a json file gets loaded correctly.""" + with open(shared_datadir / "hierarchical_meta.json", "r") as file: + data = tsdf.load_metadata_file(file) + assert(len(data) == 4) - def test_load_metadata_from_path(self): - """Test that a json file from a path gets loaded correctly.""" - data = read_tsdf.load_metadata_from_path(CONST.TEST_DATA_FILES["hierarchical"]) - self.assertEqual(len(data), 4) +def test_load_metadata_legacy_file(shared_datadir): + """Test that a json file gets loaded correctly.""" + with open(shared_datadir / "ppp_format_meta_legacy.json", "r") as file: + data = tsdf.load_metadata_legacy_file(file) + assert(len(data) == 2) - def test_load_metadata_string(self): - """Test that a json object gets loaded from a string correctly.""" - with open(CONST.TEST_DATA_FILES["hierarchical"], "r") as file: - json_string = file.read() - data = read_tsdf.load_metadata_string(json_string) - self.assertEqual(len(data), 4) +def test_load_metadata_from_path(shared_datadir): + """Test that a json file from a path gets loaded correctly.""" + data = tsdf.load_metadata_from_path(shared_datadir / "hierarchical_meta.json") + assert(len(data) == 4) - def test_load_metadatas_from_dir(self): - """Test that all metadata files gets loaded from a directory correctly.""" - data = read_tsdf.load_metadatas_from_dir(CONST.TEST_DATA_DIR) - self.assertEqual(len(data), 6) +def test_load_metadata_string(shared_datadir): + """Test that a json object gets loaded from a string correctly.""" + with open(shared_datadir / "hierarchical_meta.json", "r") as file: + json_string = file.read() + data = tsdf.load_metadata_string(json_string) + assert(len(data) == 4) + +def test_load_metadatas_from_dir(shared_datadir): + """Test that all metadata files gets loaded from a directory correctly.""" + data = tsdf.load_metadatas_from_dir(shared_datadir) + assert(len(data) == 6) diff --git a/tests/test_validator.py b/tests/test_validator.py index 4bf9340..9fee88d 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -1,13 +1,9 @@ -import unittest -from tsdf.constants import TestConstants as CONST from tsdf import validator -class TestValidator(unittest.TestCase): +def test_validate_valid_file(shared_datadir): + is_valid = validator.validate_tsdf_format(shared_datadir / "ppp_format_meta.json") + assert(is_valid) - def test_validate_valid_file(self): - result = validator.validate_tsdf_format(CONST.TEST_DATA_FILES["ppp"]) - self.assertTrue(result) - - def test_validate_invalid_file(self): - result = validator.validate_tsdf_format(CONST.TEST_DATA_FILES["missingkey"]) - self.assertFalse(result) +def test_validate_invalid_file(shared_datadir): + is_valid = validator.validate_tsdf_format(shared_datadir / "missingkey_meta_fail.json") + assert(not is_valid) diff --git a/tests/test_write_binary.py b/tests/test_write_binary.py index 0e0a381..aaa463e 100644 --- a/tests/test_write_binary.py +++ b/tests/test_write_binary.py @@ -1,32 +1,29 @@ -import os -import unittest import numpy as np -from tsdf.constants import TestConstants as CONST from tsdf import read_tsdf from tsdf import write_binary - -class TestBinaryFileWriting(unittest.TestCase): +def test_write_binary(shared_datadir): """Test writing of binary files from loaded data (e.g., NumPy array).""" + """Save a NumPy array as a binary file.""" + + test_meta_file_name = "flat_meta.json" + test_file_name = "tmp_test_output_1.bin" + + rs = np.random.RandomState(seed=42) + data_original = rs.rand(17, 1).astype(np.float32) + with open(shared_datadir / test_meta_file_name, "r") as file: + metadatas = read_tsdf.load_metadata_file(file) + write_binary.write_binary_file( + shared_datadir, + test_file_name, + data_original, + metadatas["audio_voice_089.raw"].get_plain_tsdf_dict_copy(), + ) - def test_write_binary(self): - """Save a NumPy array as a binary file.""" - test_file_name = "tmp_test_output_1.bin" - rs = np.random.RandomState(seed=42) - data_original = rs.rand(17, 1).astype(np.float32) - with open(CONST.TEST_DATA_FILES["flat"], "r") as file: - metadatas = read_tsdf.load_metadata_file(file) - write_binary.write_binary_file( - CONST.TEST_OUTPUT_DATA_DIR, - test_file_name, - data_original, - metadatas["audio_voice_089.raw"].get_plain_tsdf_dict_copy(), - ) - - # Read file again to check contents - path = os.path.join(CONST.TEST_OUTPUT_DATA_DIR, test_file_name) - with open(path, "rb") as fid: - data_written = np.fromfile(fid, dtype=" np.ndarray: - """ - Load a single binary file from the given directory path and file name. - - :param dir_path: The directory path where the binary file is located. - :param file_name: The name of the binary file without the extension. - - :returns: The binary data as a numpy array. - """ - path = os.path.join(dir_path, file_name + CONST.METADATA_EXTENSION) - metadata = read_tsdf.load_metadata_from_path(path) - data = read_binary.load_binary_from_metadata( - dir_path, metadata[file_name + CONST.BINARY_EXTENSION] +import tsdf +from tsdf import TSDFMetadata +from utils import load_single_bin_file + + +def test_save_metadata(shared_datadir): + """Test writing multiple binary files and combining their TSDF metadatas.""" + test_name = "tmp_test_save_metadata" + rs = np.random.RandomState(seed=42) + data_1 = rs.rand(17, 1).astype(np.float32) + data_2 = rs.rand(15, 2).astype(np.int16) + data_3 = rs.rand(10, 3).astype(np.int16) + + name = "example_10_3_int16" + metas = tsdf.load_metadata_from_path(shared_datadir / (name + "_meta.json")) + loaded_meta: TSDFMetadata = metas[name + ".bin"] + + new_meta_1 = tsdf.write_binary_file( + shared_datadir, + test_name + "_1.bin", + data_1, + loaded_meta.get_plain_tsdf_dict_copy(), + ) + new_meta_2 = tsdf.write_binary_file( + shared_datadir, + test_name + "_2.bin", + data_2, + loaded_meta.get_plain_tsdf_dict_copy(), ) - return data - -class TestMetadataFileWriting(unittest.TestCase): - """Test writing of metadata files based on loaded data.""" - - def test_save_metadata(self): - """Test writing multiple binary files and combining their TSDF metadatas.""" - test_name = "tmp_test_save_metadata" - rs = np.random.RandomState(seed=42) - data_1 = rs.rand(17, 1).astype(np.float32) - data_2 = rs.rand(15, 2).astype(np.int16) - data_3 = rs.rand(10, 3).astype(np.int16) - - use_case_name = "example_10_3_int16" - path = CONST.TEST_DATA_FILES[use_case_name] - loaded_meta: TSDFMetadata = read_tsdf.load_metadata_from_path(path)[ - use_case_name + CONST.BINARY_EXTENSION - ] - - new_meta_1 = write_binary.write_binary_file( - CONST.TEST_OUTPUT_DATA_DIR, - test_name + "_1.bin", - data_1, - loaded_meta.get_plain_tsdf_dict_copy(), - ) - new_meta_2 = write_binary.write_binary_file( - CONST.TEST_OUTPUT_DATA_DIR, - test_name + "_2.bin", - data_2, - loaded_meta.get_plain_tsdf_dict_copy(), - ) - - new_meta_3 = write_binary.write_binary_file( - CONST.TEST_OUTPUT_DATA_DIR, - test_name + "_3.bin", - data_3, - loaded_meta.get_plain_tsdf_dict_copy(), - ) - - # Combine two TSDF files - write_tsdf.write_metadata( - [new_meta_1, new_meta_2, new_meta_3], - test_name + CONST.METADATA_EXTENSION, - ) - - # Read the written metadata - - meta = read_tsdf.load_metadata_from_path( - os.path.join( - CONST.TEST_OUTPUT_DATA_DIR, test_name + CONST.METADATA_EXTENSION - ) - ) - self.assertEqual(len(meta), 3) - self.assertEqual(meta[test_name + "_1.bin"].rows, 17) - self.assertEqual(meta[test_name + "_2.bin"].rows, 15) - self.assertEqual(meta[test_name + "_3.bin"].rows, 10) - def test_bin_processing_and_writing_metadata(self): - """Test binary file reading, processing, and writing of the new binary and metadata files.""" - # Load existing TSDF metadata and the corresponding binary data - file_name = "example_10_3_int16" - path = os.path.join(CONST.TEST_DATA_DIR, file_name + CONST.METADATA_EXTENSION) - original_metadata = read_tsdf.load_metadata_from_path(path)[ - file_name + CONST.BINARY_EXTENSION - ] - original_data = read_binary.load_binary_from_metadata( - CONST.TEST_DATA_DIR, original_metadata - ) + new_meta_3 = tsdf.write_binary_file( + shared_datadir, + test_name + "_3.bin", + data_3, + loaded_meta.get_plain_tsdf_dict_copy(), + ) - # Perform light data processing - new_data = (original_data / 10).astype("float32") + # Combine two TSDF files + tsdf.write_metadata( + [new_meta_1, new_meta_2, new_meta_3], + test_name + "_meta.json", + ) - # Write new binary file - new_file_name = "tmp_test_example_10_3_int16_to_float32" - new_metadata = write_binary.write_binary_file( - CONST.TEST_OUTPUT_DATA_DIR, - new_file_name + CONST.BINARY_EXTENSION, - new_data, - original_metadata.get_plain_tsdf_dict_copy(), - ) + # Read the written metadata + meta = tsdf.load_metadata_from_path(shared_datadir / (test_name + "_meta.json")) + + assert(len(meta) == 3) + assert(meta[test_name + "_1.bin"].rows == 17) + assert(meta[test_name + "_2.bin"].rows == 15) + assert(meta[test_name + "_3.bin"].rows == 10) + +def test_bin_processing_and_writing_metadata(shared_datadir): + """Test binary file reading, processing, and writing of the new binary and metadata files.""" + # Load existing TSDF metadata and the corresponding binary data + name = "example_10_3_int16" + metas = tsdf.load_metadata_from_path(shared_datadir / (name + "_meta.json")) + original_metadata = metas[name + ".bin"] + original_data = tsdf.load_binary_from_metadata(original_metadata) + + # Perform light data processing + new_data = (original_data / 10).astype("float32") + + # Write new binary file + new_name = "tmp_test_example_10_3_int16_to_float32" + new_metadata = tsdf.write_binary_file( + shared_datadir, + new_name + ".bin", + new_data, + original_metadata.get_plain_tsdf_dict_copy(), + ) - # Write the new metadata file - write_tsdf.write_metadata([new_metadata], new_file_name + CONST.METADATA_EXTENSION) + # Write the new metadata file + tsdf.write_metadata([new_metadata], new_name + "_meta.json") - # Read file again to check contents - final_data = load_single_bin_file(CONST.TEST_OUTPUT_DATA_DIR, new_file_name) - self.assertEqual(final_data.shape, (10, 3)) - self.assertEqual(final_data.dtype, "float32") + # Read file again to check contents + final_data = load_single_bin_file(shared_datadir, new_name) + assert(final_data.shape == (10, 3)) + assert(final_data.dtype == "float32") diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..2710ac6 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,16 @@ +import numpy as np +from pathlib import Path +import tsdf + +def load_single_bin_file(data_dir: Path, name: str) -> np.ndarray: + """ + Load a single binary file from the given directory path and file name. + + :param dir_path: The directory path where the binary file is located. + :param file_name: The name of the binary file without the extension. + + :returns: The binary data as a numpy array. + """ + metadata = tsdf.load_metadata_from_path(data_dir / (name + "_meta.json")) + data = tsdf.load_binary_from_metadata(metadata[name + ".bin"]) + return data