From 6c68fd9570c55336ba6cfb604adc2d55b1238fe9 Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Sun, 9 Oct 2022 20:59:23 +0200 Subject: [PATCH 01/27] exposes ENMO calculation --- paat/features.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/paat/features.py b/paat/features.py index f1a5268..49ff4d1 100644 --- a/paat/features.py +++ b/paat/features.py @@ -22,7 +22,6 @@ -0.046015, 0.036283, -0.012977, -0.0046262, 0.012835, -0.0093762, 0.0034485, -0.00080972, -0.00019623]) -["Y", "X", "Z"] def calculate_vector_magnitude(data, minus_one=False, round_negative_to_zero=False, dtype=np.float32): r""" @@ -37,7 +36,7 @@ def calculate_vector_magnitude(data, minus_one=False, round_negative_to_zero=Fal round_negative_to_zero : Boolean (optional) If set to True, round negative values to zero dtype : np.dtype (optional) - set the data type of the return array. Standard float 16, but can be set to better precision + set the data type of the return array. Standard float 32, but can be set to better precision Returns @@ -78,6 +77,30 @@ def calculate_vector_magnitude(data, minus_one=False, round_negative_to_zero=Fal return vector_magnitude.reshape(data.shape[0], 1) +def calculate_enmo(data, dtype=np.float32): + """ + Calculate the Euclidean norm minus one from raw acceleration data. + This function is a wrapper of `calculate_vector_magnitude`. + + Parameters + ---------- + data : array_like + numpy array with acceleration data + dtype : np.dtype (optional) + set the data type of the return array. Standard float 32, but can be set to better precision + + + Returns + ------- + vector_magnitude : np.array (acceleration values, 1)(np.float) + numpy array with vector magnitude of the acceleration + """ + if isinstance(data, pd.DataFrame): + data = data[["Y", "X", "Z"]].values + + return calculate_vector_magnitude(data, minus_one=True, round_negative_to_zero=True) + + def calculate_frequency_features(data, win_len=60, win_step=60, sample_rate=100, nfft=512, nfilt=40): """ Calculate frequency features from raw acceleration signal. From ce18ef0217ad27572c00fa6396c82ac59696fd25 Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Sun, 9 Oct 2022 21:00:14 +0200 Subject: [PATCH 02/27] adds analysis and visualization functions --- paat/analysis.py | 24 ++++++++++++++++ paat/visualizations.py | 63 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 paat/analysis.py create mode 100644 paat/visualizations.py diff --git a/paat/analysis.py b/paat/analysis.py new file mode 100644 index 0000000..179a403 --- /dev/null +++ b/paat/analysis.py @@ -0,0 +1,24 @@ +from .estimates import calculate_pa_levels, create_activity_column +from .sleep import detect_sleep_weitz2022 +from .wear_time import etect_non_wear_time_syed2021 + + +def annotate(data, sample_freq): + + # Detect non-wear time + data.loc[:, "Non Wear Time"] = detect_non_wear_time_syed2021(data, sample_freq) + + # Detect sleep episodes + data.loc[:, "Time in bed"] = detect_sleep_triaxial_weitz2022(data, sample_freq)[:len(data)] + + # Classify moderate-to-vigorous and sedentary behavior + data.loc[:, ["MVPA", "SB"]] = calculate_pa_levels(data, sample_freq) + + # Merge the activity columns into one labelled column. columns indicates the + # importance of the columns, later names are more important and will be kept + data.loc[:, "Activity"] = create_activity_column(data, columns=["SB", "MVPA", "Time in bed", "Non Wear Time"]) + + #data = data.resample("1min").mean() + + # Remove the other columns after merging + return data[["X", "Y", "Z", "Activity"]] diff --git a/paat/visualizations.py b/paat/visualizations.py new file mode 100644 index 0000000..bb61062 --- /dev/null +++ b/paat/visualizations.py @@ -0,0 +1,63 @@ +import numpy as np +import matplotlib.pyplot as plt +from matplotlib.patches import Rectangle, Patch +import seaborn as sns + +from .features import calculate_enmo + +COLOR = {'Non Wear Time': "white", 'Time in bed': "blue", 'SB': "green", 'LPA': "yellow", 'MVPA': "red"} + + +def visualize(data, show_date=False, file_path=None): + + data["ENMO"] = calculate_enmo(data).copy() + data = data.resample("1min").apply({"X": "mean", "Y": "mean", "Z": "mean", "ENMO": "mean", "Activity": lambda x: x.value_counts().idxmax()}) + + n_days = len(data.groupby(data.index.day)) + ymax = data["ENMO"].max() * 1.05 + + fig = plt.figure(figsize=(10,n_days * 2)) + axes = fig.subplots(n_days,1) + + for ii, (_, day) in enumerate(data.groupby(data.index.day)): + + day["Time"] = np.arange(len(day)) + + ax = axes[ii] + + ax = sns.lineplot(x="Time", y="ENMO", data=day, color="black", linewidth=.75, ax=ax) + ax.set_ylabel("ENMO", fontsize=14) + + ticks, labels = list(zip(*[(tick, f"{tick // 60:0>2}:{tick % 60:0>2}") for tick in day["Time"] if (tick + 60) % 120 == 0])) + ax.set_xticks(ticks) + + ax.set_ylim((0, ymax)) + + if show_date: + ax.set_title(day.index[0].strftime('%d.%m.%Y')) + + ymin, ymax = ax.get_ylim() + height = ymax - ymin + + for _, row in day.iterrows(): + ax.add_patch(Rectangle((row["Time"], ymin), 1, height, alpha=.1, facecolor=COLOR[row["Activity"]])) + + + if ii == n_days - 1: + ax.set_xticklabels(labels) + ax.set_xlabel("Time", fontsize=14) + + fig.subplots_adjust(bottom=0.5, wspace=0.33) + handles = [Patch(color=value, label=key, alpha=.1) for key, value in COLOR.items()] + ax.legend(handles=handles,loc='upper center', + bbox_to_anchor=(0.5, -0.4),fancybox=False, shadow=False, ncol=5) + else: + ax.set_xticklabels("") + ax.set_xlabel("") + + plt.tight_layout() + + if file_path: + plt.savefig(file_path) + else: + plt.show() \ No newline at end of file From b5a33536cad24e1128e2f8b835162cd08b24e142 Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Sun, 9 Oct 2022 21:00:43 +0200 Subject: [PATCH 03/27] bumps version and registers pytest slow marker --- pyproject.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f38dc53..800cf21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "paat" -version = "1.0.0-beta.1" +version = "1.0.0-beta.2" description = "A comprehensive toolbox to analyse and model raw physical activity data" license = "MIT" @@ -68,6 +68,10 @@ docs = ["sphinx", "sphinx_rtd_theme", "numpydoc", "easydev", "nbsphinx"] [tool.pylint.message_control] disable = ["E1101", "C0330", "C0326"] +[tool.pytest.ini_options] +markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')", +] [build-system] requires = ["poetry-core>=1.0.0"] From 055f3d09fdc2efbf4824c78294e3f9c5159e417e Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Sun, 9 Oct 2022 21:17:28 +0200 Subject: [PATCH 04/27] cleans up code --- paat/__init__.py | 1 - paat/analysis.py | 18 ++++++++++++++++-- paat/features.py | 5 ++--- paat/visualizations.py | 23 +++++++++++------------ tests/test_debug.py | 1 + tests/test_io.py | 4 ++-- tests/test_pipeline.py | 2 +- 7 files changed, 33 insertions(+), 21 deletions(-) diff --git a/paat/__init__.py b/paat/__init__.py index 88622c1..682272f 100644 --- a/paat/__init__.py +++ b/paat/__init__.py @@ -40,7 +40,6 @@ __version__ = toml.load("pyproject.toml")["tool"]["poetry"]["version"] + "dev" - def sysinfo(): """ Prints system the dependency information diff --git a/paat/analysis.py b/paat/analysis.py index 179a403..668e361 100644 --- a/paat/analysis.py +++ b/paat/analysis.py @@ -4,6 +4,22 @@ def annotate(data, sample_freq): + """ + Function to annotate the raw acceleration data. Performs the standard + pipeline of paat's functions. + + Parameters + ---------- + data : DataFrame + a DataFrame containg the raw acceleration data + sample_freq : int + the sampling frequency in which the data was recorded + + Returns + ------- + data : DataFrame + the raw acceleration data plus a new column holding the activity labels + """ # Detect non-wear time data.loc[:, "Non Wear Time"] = detect_non_wear_time_syed2021(data, sample_freq) @@ -18,7 +34,5 @@ def annotate(data, sample_freq): # importance of the columns, later names are more important and will be kept data.loc[:, "Activity"] = create_activity_column(data, columns=["SB", "MVPA", "Time in bed", "Non Wear Time"]) - #data = data.resample("1min").mean() - # Remove the other columns after merging return data[["X", "Y", "Z", "Activity"]] diff --git a/paat/features.py b/paat/features.py index 49ff4d1..f3b5b07 100644 --- a/paat/features.py +++ b/paat/features.py @@ -233,14 +233,14 @@ def _calc_one_axis_brond_counts(acc, sample_freq, epoch_length, deadband=0.068, resampled_data = resampy.resample(acc, sample_freq, target_hz) # Step 1: Aliasing filter (0.01-7hz) - B2, A2 = signal.butter(4, np.array([0.01, 7])/(target_hz/2), btype='bandpass') + B2, A2 = signal.butter(4, np.array([0.01, 7]) / (target_hz / 2), btype='bandpass') dataf = signal.filtfilt(B2, A2, resampled_data) # Step 2: ActiGraph filter filtered = signal.lfilter(0.965 * B, A, dataf) # Step 3, 4 & 5: Downsample to 10hz, clip at peak (2.13g) and rectify - rectified = np.abs(np.clip(filtered[::3], a_min=-1*peak, a_max=peak)) + rectified = np.abs(np.clip(filtered[::3], a_min=-1 * peak, a_max=peak)) # Step 6 & 7: Dead-band and convert to 8bit resolution downsampled = np.where(rectified < deadband, 0, rectified) // adcResolution @@ -285,7 +285,6 @@ def calculate_brond_counts(data, sample_freq, epoch_length): if isinstance(epoch_length, str): epoch_length = pd.Timedelta(epoch_length).seconds - counts = pd.DataFrame({"Y": _calc_one_axis_brond_counts(data["Y"].values, sample_freq, epoch_length), "X": _calc_one_axis_brond_counts(data["X"].values, sample_freq, epoch_length), "Z": _calc_one_axis_brond_counts(data["Z"].values, sample_freq, epoch_length)}, diff --git a/paat/visualizations.py b/paat/visualizations.py index bb61062..96d1407 100644 --- a/paat/visualizations.py +++ b/paat/visualizations.py @@ -9,15 +9,15 @@ def visualize(data, show_date=False, file_path=None): - + data["ENMO"] = calculate_enmo(data).copy() data = data.resample("1min").apply({"X": "mean", "Y": "mean", "Z": "mean", "ENMO": "mean", "Activity": lambda x: x.value_counts().idxmax()}) - + n_days = len(data.groupby(data.index.day)) ymax = data["ENMO"].max() * 1.05 - - fig = plt.figure(figsize=(10,n_days * 2)) - axes = fig.subplots(n_days,1) + + fig = plt.figure(figsize=(10, n_days * 2)) + axes = fig.subplots(n_days, 1) for ii, (_, day) in enumerate(data.groupby(data.index.day)): @@ -32,32 +32,31 @@ def visualize(data, show_date=False, file_path=None): ax.set_xticks(ticks) ax.set_ylim((0, ymax)) - + if show_date: ax.set_title(day.index[0].strftime('%d.%m.%Y')) ymin, ymax = ax.get_ylim() height = ymax - ymin - for _, row in day.iterrows(): + for _, row in day.iterrows(): ax.add_patch(Rectangle((row["Time"], ymin), 1, height, alpha=.1, facecolor=COLOR[row["Activity"]])) - if ii == n_days - 1: ax.set_xticklabels(labels) ax.set_xlabel("Time", fontsize=14) fig.subplots_adjust(bottom=0.5, wspace=0.33) handles = [Patch(color=value, label=key, alpha=.1) for key, value in COLOR.items()] - ax.legend(handles=handles,loc='upper center', - bbox_to_anchor=(0.5, -0.4),fancybox=False, shadow=False, ncol=5) + ax.legend(handles=handles, loc='upper center', + bbox_to_anchor=(0.5, -0.4), fancybox=False, shadow=False, ncol=5) else: ax.set_xticklabels("") ax.set_xlabel("") plt.tight_layout() - + if file_path: plt.savefig(file_path) else: - plt.show() \ No newline at end of file + plt.show() diff --git a/tests/test_debug.py b/tests/test_debug.py index de9cd13..f83d529 100644 --- a/tests/test_debug.py +++ b/tests/test_debug.py @@ -1,4 +1,5 @@ import paat + def test_sysinfo(): paat.sysinfo() diff --git a/tests/test_io.py b/tests/test_io.py index 71fb1c5..58a5446 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -11,7 +11,8 @@ @pytest.fixture def unscaled_data(file_path): - return io.read_gt3x(file_path, rescale=False, pandas=True) + acc, _ = io.read_gt3x(file_path, rescale=False, pandas=True) + return acc def test_loading_data(file_path, load_gt3x_file): @@ -27,7 +28,6 @@ def test_against_actigraph_implementation(file_path, unscaled_data): ref = reader.to_pandas() npt.assert_almost_equal(unscaled_data[["X", "Y", "Z"]].values, ref[["X", "Y", "Z"]].values) - #assert np.allclose(data[["X", "Y", "Z"]].values, ref[["X", "Y", "Z"]].values) def test_exceptions(): diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 5df2632..c9c0068 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -28,7 +28,7 @@ def test_pipeline(): data.loc[:, "Activity"] = paat.create_activity_column(data, columns=["Non Wear Time", "Sleep", "MVPA", "SB"]) # Remove the other columns after merging - data = data[["X", "Y", "Z", "Activity"]] + data = data[["X", "Y", "Z", "Activity"]] # Get ActiLife counts counts = paat.calculate_actigraph_counts(data, sample_freq, "10s") From 905da9b5546eed0f6df553c78695594e16932848 Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Mon, 10 Oct 2022 09:59:21 +0200 Subject: [PATCH 05/27] invokes pkg_resources directly to resolve DeprecationWarning --- paat/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paat/__init__.py b/paat/__init__.py index 682272f..21e4e18 100644 --- a/paat/__init__.py +++ b/paat/__init__.py @@ -22,7 +22,7 @@ import platform from importlib import metadata -from pip._vendor import pkg_resources +import pkg_resources import toml from . import estimates, features, io, preprocessing, sleep, wear_time From 7cf6d7e2d71604c94e763671cfb467fc9852e62f Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Mon, 10 Oct 2022 09:59:49 +0200 Subject: [PATCH 06/27] adds numeric_only argument to resolve DeprecationWarning from pandas --- paat/features.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paat/features.py b/paat/features.py index f3b5b07..15e3057 100644 --- a/paat/features.py +++ b/paat/features.py @@ -315,6 +315,6 @@ def calculate_actigraph_counts(data, sample_freq, epoch_length): sec_per_epoch = pd.Timedelta(epoch_length).seconds counts = get_counts(data[["Y", "X", "Z"]].values, sample_freq, sec_per_epoch) - index = data.resample(epoch_length).mean().index + index = data.resample(epoch_length).mean(numeric_only=True).index counts = pd.DataFrame(counts, columns=["Y", "X", "Z"], index=index[:len(counts)]) return counts From 9425cf8b9a762b2cba3f53c4a81810d24121ccf2 Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Mon, 10 Oct 2022 10:10:20 +0200 Subject: [PATCH 07/27] changes workflow name --- .github/workflows/python-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index fbe2ba7..db4adf9 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -1,4 +1,4 @@ -name: tests +name: Run Unit Tests on: push: From 3110a0e78774654e8afc1bd08e82404088e549fa Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Mon, 10 Oct 2022 10:47:20 +0200 Subject: [PATCH 08/27] adds ignoring of artefact .backup files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 6c01de5..ea6cce0 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ docs/source/_static/copybutton.js poetry.lock .coverage coverage.xml + +*.backup \ No newline at end of file From 0b2b1789c8854a3e39f0e3f9fff769a5e11c814a Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Mon, 10 Oct 2022 11:07:58 +0200 Subject: [PATCH 09/27] exposes new functions on package level and updates readme --- README.rst | 15 +++++++++++ paat/__init__.py | 2 ++ paat/analysis.py | 4 +-- tests/{test_pipeline.py => test_readme.py} | 29 ++++++++++++++++------ 4 files changed, 40 insertions(+), 10 deletions(-) rename tests/{test_pipeline.py => test_readme.py} (64%) diff --git a/README.rst b/README.rst index 257d7dc..6a0be43 100644 --- a/README.rst +++ b/README.rst @@ -50,6 +50,21 @@ implemented while others are still work in progress. The following code snippet should give you a brief overview and idea on how to use this package. Further examples and more information on the functions can be found in the documentation. +.. code-block:: python + + # Load data from file + data, sample_freq = paat.read_gt3x(FILE_PATH_SIMPLE) + + # Annotate the acceleration data + data = paat.annotate(data, sample_freq) + + +The annotation function is a convenience function that applies the standard +workflow. However, *paat* gives you the freedom to adjust your pipeline as you +wish and combine it with other packages if required. Each step in *paat* is +defined as an own function, so you can adjust everything. For example, here is +a more detailed annotation pipeline: + .. code-block:: python # Load data from file diff --git a/paat/__init__.py b/paat/__init__.py index 21e4e18..b78b759 100644 --- a/paat/__init__.py +++ b/paat/__init__.py @@ -33,6 +33,8 @@ from .io import read_gt3x from .sleep import detect_sleep_weitz2022, detect_sleep_triaxial_weitz2022 from .wear_time import detect_non_wear_time_naive, detect_non_wear_time_hees2011, detect_non_wear_time_syed2021 +from .visualizations import visualize +from .analysis import annotate try: __version__ = metadata.version(__package__) diff --git a/paat/analysis.py b/paat/analysis.py index 668e361..a2a065a 100644 --- a/paat/analysis.py +++ b/paat/analysis.py @@ -1,6 +1,6 @@ from .estimates import calculate_pa_levels, create_activity_column -from .sleep import detect_sleep_weitz2022 -from .wear_time import etect_non_wear_time_syed2021 +from .sleep import detect_sleep_weitz2022, detect_sleep_triaxial_weitz2022 +from .wear_time import detect_non_wear_time_syed2021 def annotate(data, sample_freq): diff --git a/tests/test_pipeline.py b/tests/test_readme.py similarity index 64% rename from tests/test_pipeline.py rename to tests/test_readme.py index c9c0068..620782e 100644 --- a/tests/test_pipeline.py +++ b/tests/test_readme.py @@ -8,13 +8,29 @@ FILE_PATH_SIMPLE = os.path.join(TEST_ROOT, 'resources/10min_recording.gt3x') -def test_pipeline(): +def test_simple_example(): """ Test the Quickstart/Readme processing pipeline """ # Load data from file data, sample_freq = paat.read_gt3x(FILE_PATH_SIMPLE) + # Annotate the acceleration data + data = paat.annotate(data, sample_freq) + + # Get ActiLife counts + counts = paat.calculate_actigraph_counts(data, sample_freq, "10s") + data.loc[:, ["Y_counts", "X_counts", "Z_counts"]] = counts + + +def test_advanced_example(): + """ + Test the advanced example from the Readme + """ + + # Load data from file + data, sample_freq = paat.read_gt3x(FILE_PATH_SIMPLE) + # Detect non-wear time data.loc[:, "Non Wear Time"] = paat.detect_non_wear_time_syed2021(data, sample_freq) @@ -24,12 +40,9 @@ def test_pipeline(): # Classify moderate-to-vigorous and sedentary behavior data.loc[:, ["MVPA", "SB"]] = paat.calculate_pa_levels(data, sample_freq) - # Merge the activity columns into one labelled column - data.loc[:, "Activity"] = paat.create_activity_column(data, columns=["Non Wear Time", "Sleep", "MVPA", "SB"]) + # Merge the activity columns into one labelled column. columns indicates the + # importance of the columns, later names are more important and will be kept + data.loc[:, "Activity"] = paat.create_activity_column(data, columns=["SB", "MVPA", "Sleep", "Non Wear Time"]) # Remove the other columns after merging - data = data[["X", "Y", "Z", "Activity"]] - - # Get ActiLife counts - counts = paat.calculate_actigraph_counts(data, sample_freq, "10s") - data.loc[:, ["Y_counts", "X_counts", "Z_counts"]] = counts + data = data[["X", "Y", "Z", "Activity"]] \ No newline at end of file From 21ef19a7e072ebc731d132822055cf35dff0e7d8 Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Mon, 10 Oct 2022 16:03:11 +0200 Subject: [PATCH 10/27] improves overview plot --- paat/visualizations.py | 55 +++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/paat/visualizations.py b/paat/visualizations.py index 96d1407..3447b34 100644 --- a/paat/visualizations.py +++ b/paat/visualizations.py @@ -1,62 +1,67 @@ import numpy as np +import pandas as pd import matplotlib.pyplot as plt from matplotlib.patches import Rectangle, Patch import seaborn as sns from .features import calculate_enmo -COLOR = {'Non Wear Time': "white", 'Time in bed': "blue", 'SB': "green", 'LPA': "yellow", 'MVPA': "red"} +COLOR = {'Non Wear Time': "grey", 'Time in bed': "blue", 'SB': "green", 'LPA': "yellow", 'MVPA': "red"} -def visualize(data, show_date=False, file_path=None): +def visualize(data, show_date=False, title=None, file_path=None): data["ENMO"] = calculate_enmo(data).copy() data = data.resample("1min").apply({"X": "mean", "Y": "mean", "Z": "mean", "ENMO": "mean", "Activity": lambda x: x.value_counts().idxmax()}) n_days = len(data.groupby(data.index.day)) ymax = data["ENMO"].max() * 1.05 + min_per_day = 1440 - fig = plt.figure(figsize=(10, n_days * 2)) - axes = fig.subplots(n_days, 1) + fig = plt.figure(figsize=(8.27, 11.69)) #figsize=(10, n_days * 3)) + axes = fig.subplots(n_days, 1, sharex=True) + plt.subplots_adjust(hspace=.3) + + if title: + fig.suptitle(title, fontsize=14) for ii, (_, day) in enumerate(data.groupby(data.index.day)): - day["Time"] = np.arange(len(day)) + offset = (day.index[0] - pd.Timestamp(day.index[0].date())).total_seconds() // 60 + day["Time"] = np.arange(offset, offset + len(day)) ax = axes[ii] - ax = sns.lineplot(x="Time", y="ENMO", data=day, color="black", linewidth=.75, ax=ax) - ax.set_ylabel("ENMO", fontsize=14) - - ticks, labels = list(zip(*[(tick, f"{tick // 60:0>2}:{tick % 60:0>2}") for tick in day["Time"] if (tick + 60) % 120 == 0])) - ax.set_xticks(ticks) + if show_date: + ax.set_title(day.index[0].strftime('%A, %d.%m.%Y'), fontsize=12) + ax = sns.lineplot(x="Time", y="ENMO", data=day, color="black", linewidth=.75, ax=ax) + ax.set_ylabel("ENMO", fontsize=10) ax.set_ylim((0, ymax)) - - if show_date: - ax.set_title(day.index[0].strftime('%d.%m.%Y')) + ax.set_xlim((0, min_per_day)) ymin, ymax = ax.get_ylim() height = ymax - ymin + # Add background for _, row in day.iterrows(): ax.add_patch(Rectangle((row["Time"], ymin), 1, height, alpha=.1, facecolor=COLOR[row["Activity"]])) - if ii == n_days - 1: - ax.set_xticklabels(labels) - ax.set_xlabel("Time", fontsize=14) + ticks, labels = list(zip(*[(tick, f"{tick // 60:0>2}:{tick % 60:0>2}") for tick in range(min_per_day) if (tick + 60) % 120 == 0])) + ax.set_xticks(ticks) + ax.set_xticklabels("") + ax.set_xlabel("") - fig.subplots_adjust(bottom=0.5, wspace=0.33) - handles = [Patch(color=value, label=key, alpha=.1) for key, value in COLOR.items()] - ax.legend(handles=handles, loc='upper center', - bbox_to_anchor=(0.5, -0.4), fancybox=False, shadow=False, ncol=5) - else: - ax.set_xticklabels("") - ax.set_xlabel("") + ax.set_xticklabels(labels) + ax.set_xlabel("Time", fontsize=10) - plt.tight_layout() + # Add legend to last plot + #ax = axes[-1] + handles = [Patch(color=value, label=key, alpha=.1) for key, value in COLOR.items()] + ax.legend(handles=handles, loc='upper center', bbox_to_anchor=(0.5, -.9 * ymax), fancybox=False, shadow=False, ncol=5) + #ax.set_axis_off() if file_path: - plt.savefig(file_path) + plt.savefig(file_path, dpi=300) else: plt.show() From 93458b23dcba9f5de424e5e010dc3c6d1790ed58 Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Wed, 12 Oct 2022 13:56:11 +0200 Subject: [PATCH 11/27] exposes calculate_enmo function --- paat/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paat/__init__.py b/paat/__init__.py index b78b759..e237eec 100644 --- a/paat/__init__.py +++ b/paat/__init__.py @@ -29,7 +29,7 @@ # Expose API functions from .estimates import calculate_pa_levels, create_activity_column -from .features import calculate_actigraph_counts, calculate_vector_magnitude, calculate_brond_counts +from .features import calculate_actigraph_counts, calculate_vector_magnitude, calculate_brond_counts, calculate_enmo from .io import read_gt3x from .sleep import detect_sleep_weitz2022, detect_sleep_triaxial_weitz2022 from .wear_time import detect_non_wear_time_naive, detect_non_wear_time_hees2011, detect_non_wear_time_syed2021 From 63dc1c07411b8a3e7ad682e9483ff26eac074e65 Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Thu, 13 Oct 2022 15:38:20 +0200 Subject: [PATCH 12/27] adds matplotlib and seaborn as dependencies --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 800cf21..116d1a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,8 @@ torch = "^1.10.1" agcounts = "^0.1.1" toml = "^0.10.2" tables = "^3.7.0" +matplotlib = "^3.6.1" +seaborn = "^0.12.0" [tool.poetry.dev-dependencies] @@ -46,7 +48,6 @@ flake8 = "^4.0.1" sphinx = "^1.4" sphinx_rtd_theme = "1.0.0" notebook = "^6.4.10" -seaborn = "^0.11.2" numpydoc = "1.2" easydev = "0.9.35" pylint = "^2.0.0" From 78b4bf5cb302b286036f85016fc634a9ca6bc0ea Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Thu, 13 Oct 2022 15:52:02 +0200 Subject: [PATCH 13/27] fixes bug in readme --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 6a0be43..6a11ad8 100644 --- a/README.rst +++ b/README.rst @@ -53,7 +53,7 @@ examples and more information on the functions can be found in the documentation .. code-block:: python # Load data from file - data, sample_freq = paat.read_gt3x(FILE_PATH_SIMPLE) + data, sample_freq = paat.read_gt3x('path/to/gt3x/file') # Annotate the acceleration data data = paat.annotate(data, sample_freq) From 15918828824762780ea893987aa8f4eda1f777ec Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Thu, 13 Oct 2022 15:53:10 +0200 Subject: [PATCH 14/27] adds summary function --- paat/__init__.py | 2 +- paat/analysis.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/paat/__init__.py b/paat/__init__.py index e237eec..42b990b 100644 --- a/paat/__init__.py +++ b/paat/__init__.py @@ -34,7 +34,7 @@ from .sleep import detect_sleep_weitz2022, detect_sleep_triaxial_weitz2022 from .wear_time import detect_non_wear_time_naive, detect_non_wear_time_hees2011, detect_non_wear_time_syed2021 from .visualizations import visualize -from .analysis import annotate +from .analysis import annotate, summary try: __version__ = metadata.version(__package__) diff --git a/paat/analysis.py b/paat/analysis.py index a2a065a..bc81ecc 100644 --- a/paat/analysis.py +++ b/paat/analysis.py @@ -36,3 +36,24 @@ def annotate(data, sample_freq): # Remove the other columns after merging return data[["X", "Y", "Z", "Activity"]] + + +def summary(data, sample_freq): + """ + Create a daily summary of the DataFrame + + Parameters + ---------- + data : DataFrame + a DataFrame containg the raw acceleration data + sample_freq : int + the sampling frequency in which the data was recorded + level : str + a string indicating to which level the data should be aggregated + + Returns + ------- + agg : DataFrame + the aggregated data holding the minutes spend in each activity + """ + return pd.get_dummies(data["Activity"]).resample("D").sum() / (sample_freq * 60) \ No newline at end of file From eee0682792a22e65fa028ffcb6ba483598ddc2c6 Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Thu, 13 Oct 2022 16:31:48 +0200 Subject: [PATCH 15/27] adds missing dependency --- paat/analysis.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/paat/analysis.py b/paat/analysis.py index bc81ecc..3ba6fd8 100644 --- a/paat/analysis.py +++ b/paat/analysis.py @@ -1,3 +1,5 @@ +import pandas as pd + from .estimates import calculate_pa_levels, create_activity_column from .sleep import detect_sleep_weitz2022, detect_sleep_triaxial_weitz2022 from .wear_time import detect_non_wear_time_syed2021 From 9d99921fbebae2a276770fa0100ca81368804016 Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Thu, 1 Feb 2024 10:14:11 +0100 Subject: [PATCH 16/27] fixes gh actions syntax --- .github/workflows/python-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index db4adf9..9798c2e 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -14,7 +14,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macOS-latest] - python-version: [3.8, 3.9] + python-version: ['3.8', '3.9'] poetry-version: [1.1.13] fail-fast: false runs-on: ${{ matrix.os }} From 8bb48e68ef2043492544f7424da7f7fc634fb8cb Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Thu, 1 Feb 2024 10:16:00 +0100 Subject: [PATCH 17/27] adds further python versions --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 116d1a8..dcad57c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ classifiers = ['Development Status :: 4 - Beta', 'Intended Audience :: Science/Research'] [tool.poetry.dependencies] -python = ">=3.8,<3.10" # Compatible python versions must be declared here +python = ">=3.8,<3.13" # Compatible python versions must be declared here pandas = "^1.2.4" scipy = "^1.6.2" psutil = "^5.8.0" From 1ae30bbe778fb9aef4a217494172f0b6b7f3e73b Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Thu, 1 Feb 2024 10:18:50 +0100 Subject: [PATCH 18/27] updates python versions --- .github/workflows/python-test.yml | 4 ++-- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 9798c2e..7476882 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -14,8 +14,8 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macOS-latest] - python-version: ['3.8', '3.9'] - poetry-version: [1.1.13] + python-version: ['3.9', '3.10', '3.11'] + poetry-version: [1.1.14] fail-fast: false runs-on: ${{ matrix.os }} steps: diff --git a/pyproject.toml b/pyproject.toml index dcad57c..dce6f2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ classifiers = ['Development Status :: 4 - Beta', 'Intended Audience :: Science/Research'] [tool.poetry.dependencies] -python = ">=3.8,<3.13" # Compatible python versions must be declared here +python = ">=3.9,<3.12" # Compatible python versions must be declared here pandas = "^1.2.4" scipy = "^1.6.2" psutil = "^5.8.0" From 6a791112b8489717766c6cd08e97a7fccce6a71f Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Thu, 1 Feb 2024 10:21:09 +0100 Subject: [PATCH 19/27] updates python versions because of pygt3x --- .github/workflows/python-test.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 7476882..04c373c 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -14,7 +14,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macOS-latest] - python-version: ['3.9', '3.10', '3.11'] + python-version: ['3.9', '3.10'] poetry-version: [1.1.14] fail-fast: false runs-on: ${{ matrix.os }} diff --git a/pyproject.toml b/pyproject.toml index dce6f2d..31bc1c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ classifiers = ['Development Status :: 4 - Beta', 'Intended Audience :: Science/Research'] [tool.poetry.dependencies] -python = ">=3.9,<3.12" # Compatible python versions must be declared here +python = ">=3.9,<3.11" # Compatible python versions must be declared here pandas = "^1.2.4" scipy = "^1.6.2" psutil = "^5.8.0" From dffdb24f6bf8a27495f50f22ddfa4abb9464ae74 Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Thu, 1 Feb 2024 10:50:14 +0100 Subject: [PATCH 20/27] pin tensorflow version to avoid windows errors --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 31bc1c2..e6f734b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ classifiers = ['Development Status :: 4 - Beta', 'Intended Audience :: Science/Research'] [tool.poetry.dependencies] -python = ">=3.9,<3.11" # Compatible python versions must be declared here +python = "3.9,3.10" # Compatible python versions must be declared here pandas = "^1.2.4" scipy = "^1.6.2" psutil = "^5.8.0" @@ -31,7 +31,7 @@ resampy = "^0.2.2" joblib = "^1.0.1" numpy = "^1.20.3" bitstring = "^3.1.7" -tensorflow = "^2.7.0" +tensorflow = "^2.7.0,<2.15" #Pin tensorflow below 2.15 for now as tensorflow-io-gcs-filesystem does not yet work on windows torch = "^1.10.1" agcounts = "^0.1.1" toml = "^0.10.2" From a3da460403c4c3008c4fcb9352292fe12415489a Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Thu, 1 Feb 2024 10:56:04 +0100 Subject: [PATCH 21/27] fixes pinning tensorflow version to avoid windows errors --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e6f734b..26e976e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ resampy = "^0.2.2" joblib = "^1.0.1" numpy = "^1.20.3" bitstring = "^3.1.7" -tensorflow = "^2.7.0,<2.15" #Pin tensorflow below 2.15 for now as tensorflow-io-gcs-filesystem does not yet work on windows +tensorflow = ">=2.7.0,<2.15" #Pin tensorflow below 2.15 for now as tensorflow-io-gcs-filesystem does not yet work on windows torch = "^1.10.1" agcounts = "^0.1.1" toml = "^0.10.2" From 6917692190ef661744bffb91e2f1ae3716754597 Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Thu, 1 Feb 2024 11:04:22 +0100 Subject: [PATCH 22/27] fixes pinning tensorflow version to avoid windows errors --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 26e976e..6efed32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ resampy = "^0.2.2" joblib = "^1.0.1" numpy = "^1.20.3" bitstring = "^3.1.7" -tensorflow = ">=2.7.0,<2.15" #Pin tensorflow below 2.15 for now as tensorflow-io-gcs-filesystem does not yet work on windows +tensorflow = ">2.7.0,<2.15" #Pin tensorflow below 2.15 for now as tensorflow-io-gcs-filesystem does not yet work on windows torch = "^1.10.1" agcounts = "^0.1.1" toml = "^0.10.2" From ecdcc747a63072689d63f90dbdbbe4723c619ac7 Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Thu, 1 Feb 2024 11:07:49 +0100 Subject: [PATCH 23/27] fixes pinning tensorflow version to avoid windows errors --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6efed32..06ca0c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ resampy = "^0.2.2" joblib = "^1.0.1" numpy = "^1.20.3" bitstring = "^3.1.7" -tensorflow = ">2.7.0,<2.15" #Pin tensorflow below 2.15 for now as tensorflow-io-gcs-filesystem does not yet work on windows +tensorflow = "2.14" #Pin tensorflow below 2.15 for now as tensorflow-io-gcs-filesystem does not yet work on windows torch = "^1.10.1" agcounts = "^0.1.1" toml = "^0.10.2" From 1406f7f7e8d7157a2121673b3bdd33a5a3a6089c Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Thu, 1 Feb 2024 11:11:03 +0100 Subject: [PATCH 24/27] fixes pinning tensorflow version to avoid windows errors --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 06ca0c8..d601b90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ resampy = "^0.2.2" joblib = "^1.0.1" numpy = "^1.20.3" bitstring = "^3.1.7" -tensorflow = "2.14" #Pin tensorflow below 2.15 for now as tensorflow-io-gcs-filesystem does not yet work on windows +tensorflow = ">2.7, <2.15" #Pin tensorflow below 2.15 for now as tensorflow-io-gcs-filesystem does not yet work on windows torch = "^1.10.1" agcounts = "^0.1.1" toml = "^0.10.2" From 5ac469c2060fef3f657ebe7023450aa6c58b2e4e Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Thu, 1 Feb 2024 11:13:03 +0100 Subject: [PATCH 25/27] fixes pinning tensorflow version to avoid windows errors --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d601b90..b0cf99c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ resampy = "^0.2.2" joblib = "^1.0.1" numpy = "^1.20.3" bitstring = "^3.1.7" -tensorflow = ">2.7, <2.15" #Pin tensorflow below 2.15 for now as tensorflow-io-gcs-filesystem does not yet work on windows +tensorflow = "^2.7" torch = "^1.10.1" agcounts = "^0.1.1" toml = "^0.10.2" From c0d7d6ff39901b5eb5092348f5be5d6a31d0714d Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Thu, 1 Feb 2024 11:14:39 +0100 Subject: [PATCH 26/27] fixes pinning tensorflow version to avoid windows errors --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b0cf99c..f8f2147 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ classifiers = ['Development Status :: 4 - Beta', 'Intended Audience :: Science/Research'] [tool.poetry.dependencies] -python = "3.9,3.10" # Compatible python versions must be declared here +python = ">=3.9,<3.11" # Compatible python versions must be declared here pandas = "^1.2.4" scipy = "^1.6.2" psutil = "^5.8.0" @@ -31,7 +31,7 @@ resampy = "^0.2.2" joblib = "^1.0.1" numpy = "^1.20.3" bitstring = "^3.1.7" -tensorflow = "^2.7" +tensorflow = ">=2.7, <2.15" #Pin tensorflow below 2.15 for now as tensorflow-io-gcs-filesystem does not yet work on windows torch = "^1.10.1" agcounts = "^0.1.1" toml = "^0.10.2" From c3e66a492ad2ce862ccb2e443a9c62211aafca8e Mon Sep 17 00:00:00 2001 From: Marc Weitz Date: Tue, 27 Aug 2024 10:21:26 +0200 Subject: [PATCH 27/27] fixes dependencies --- pyproject.toml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 28fd5b0..09ff68e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "paat" -version = "1.0.0-beta.2" +version = "1.0.0b5" description = "A comprehensive toolbox to analyse and model raw physical activity data" license = "MIT" @@ -42,7 +42,7 @@ agcounts = "^0.1.1" toml = "^0.10.2" tables = "^3.7.0" matplotlib = "^3.6.1" -seaborn = "^0.12.0" +seaborn = "^0.13.2" [tool.poetry.extras] docs = ["sphinx", "sphinx_rtd_theme", "numpydoc", "easydev", "nbsphinx", "docutils"] @@ -56,7 +56,6 @@ pytest-cov = "^4.1.0" pycodestyle = "^2.11.1" flake8 = "^7.0.0" notebook = "^7.0.7" -seaborn = "^0.13.2" numpydoc = "^1.6.0" easydev = "^0.12.1" pylint = "^3.0.3"