Example2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example 2\n",
    "\n",
    "Example 2 shows how to use the tilt correction on long-time data which was collected over days. The difference to a short time data set is that over days the data might be influenced in different ways/directions. Therefore the data is fed into the tilt-correction in hourly chunks whereas it is overlapped by 30 minutes to create a pseudo-running mean in the calculation. Then the overlapping 30 minutes is discarded. Additionally a simple but careful outlier removal over the entire dataset is applied with this example.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Created on Thu Apr 12 13:50:35 2021\n",
    "\n",
    "@author: Benjamin Schumacher\n",
    "\n",
    "Example file for correction of sonic anemometer data\n",
    "\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "from sonic_func import *\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "# create relative path variables\n",
    "wd_path = os.getcwd()\n",
    "data_path = os.path.join(wd_path, \"data\")\n",
    "example_file = os.path.join(data_path, \"example.csv\")\n",
    "\n",
    "# read the data from the file\n",
    "irg2 = pd.read_csv(example_file, header=[0,1], na_values='NAN')\n",
    "\n",
    "recording_freq = 20\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Outlier removal\n",
    "timestamp = irg2[\"TIMESTAMP\"].values\n",
    "u = irg2[\"Ux\"].values\n",
    "v = irg2[\"Uy\"].values\n",
    "w = irg2[\"Uz\"].values\n",
    "\n",
    "u_int = wrapper_outl_interpolation(u)\n",
    "v_int = wrapper_outl_interpolation(v)\n",
    "w_int = wrapper_outl_interpolation(w)\n",
    "\n",
    "\n",
    "irg2_clean = pd.DataFrame({'TIMESTAMP': timestamp.flatten(), 'Ux': u_int, 'Uy': v_int, 'Uz': w_int})\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shortening array to even number by 8 measurements!\n",
      "Shortening array to even number by 8 measurements!\n",
      "Shortening array to even number by 8 measurements!\n"
     ]
    }
   ],
   "source": [
    "\n",
    "hour = recording_freq*60*60\n",
    "half_hour = int(hour/2)\n",
    "\n",
    "for i in range(0,len(irg2),half_hour):\n",
    "    \n",
    "    #print(i, end = \" \")\n",
    "   \n",
    "    \n",
    "    if i == 0:\n",
    "        end_data = i+hour\n",
    "        begin_data = 0\n",
    "    else:\n",
    "        begin_data = i\n",
    "        end_data = begin_data + hour\n",
    "    \n",
    "    if begin_data+hour > len(irg2):\n",
    "        end_data = len(irg2)\n",
    "    \n",
    "\n",
    "    #print(begin_data)\n",
    "    #print(end_data)\n",
    "    \n",
    "    timestamp = irg2_clean[\"TIMESTAMP\"].values[begin_data:end_data]\n",
    "    u = irg2_clean[\"Ux\"].values[begin_data:end_data]\n",
    "    v = irg2_clean[\"Uy\"].values[begin_data:end_data]\n",
    "    w = irg2_clean[\"Uz\"].values[begin_data:end_data]\n",
    "    \n",
    "    timestamp_pf, u1_pf, v1_pf, w1_pf, Ts_pf, CO2_pf, H2O_pf = planar_fit(u, v, w, sub_size = 10, timestamp = timestamp)\n",
    "    \n",
    "    if i == 0:\n",
    "        pf = pd.DataFrame({'TIMESTAMP': timestamp_pf.flatten(), 'Ux': u1_pf, 'Uy': v1_pf, 'Uz': w1_pf})\n",
    "        pf_final = copy.copy(pf)\n",
    "    else:\n",
    "        pf = pd.DataFrame({'TIMESTAMP': timestamp_pf.flatten()[half_hour:], 'Ux': u1_pf[half_hour:], 'Uy': v1_pf[half_hour:], 'Uz': w1_pf[half_hour:]})\n",
    "        pf_final = pf_final.append(pf) \n",
    "    \n",
    "    # not beautiful but currently my only idea how to break the for loop and keep it from the last iteration\n",
    "    if end_data == len(irg2):\n",
    "        break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pf_final.to_csv(data_path+\"/example_result_planarfit.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}