diff --git a/docs/source/examples/QartodTestExample_Glider.ipynb b/docs/source/examples/QartodTestExample_Glider.ipynb new file mode 100644 index 0000000..ceea7c7 --- /dev/null +++ b/docs/source/examples/QartodTestExample_Glider.ipynb @@ -0,0 +1,218 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup\n", + "\n", + "Import ioos qc libraries, as well as erddapy for data fetching and Bokeh for plotting" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import ioos_qc\n", + "from ioos_qc.config import Config\n", + "from ioos_qc.qartod import aggregate\n", + "from ioos_qc.streams import XarrayStream\n", + "from ioos_qc.results import collect_results, CollectedResult\n", + "import xarray as xr\n", + "import numpy as np\n", + "import pandas as pd\n", + "import pooch\n", + "from bokeh.plotting import output_notebook\n", + "\n", + "output_notebook()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Load the dataset\n", + "\n", + "Here we use a glider mission from the Baltic as a test dataset.\n", + "\n", + "https://observations.voiceoftheocean.org/SEA067/M37" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"https://github.com/ioos/ioos_qc/releases/download\"\n", + "version = \"2.1.0\"\n", + "fname = \"nrt_SEA067_M37.nc\"\n", + "\n", + "download = pooch.retrieve(\n", + " url=f\"{url}/{version}/{fname}\",\n", + " known_hash=\"sha256:06e8a79cc17a2d55bb32dbfdc85f9922c1a1cc14726df004ae971125f91b27ac\",\n", + ")\n", + "ds = xr.open_dataset(download)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Generate test configs \n", + "Make dictionaries of test configurations for salinity. To generate salinity flags, we test against salinity, conductivity and temperature" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "config = {\n", + " \"temperature\": {\n", + " \"qartod\": {\n", + " \"gross_range_test\": {\"suspect_span\": [0, 30], \"fail_span\": [-2.5, 40]},\n", + " \"spike_test\": {\"suspect_threshold\": 2.0, \"fail_threshold\": 6.0}\n", + " }\n", + " },\n", + " \"conductivity\": {\n", + " \"qartod\": {\n", + " \"gross_range_test\": {\"suspect_span\": [6, 42], \"fail_span\": [3, 45]}\n", + " }\n", + " },\n", + " \"salinity\": {\n", + " \"qartod\": {\n", + " \"gross_range_test\": {\"suspect_span\": [5, 38], \"fail_span\": [2, 41]},\n", + " \"spike_test\": {\"suspect_threshold\": 0.3, \"fail_threshold\": 0.9},\n", + " \"location_test\": {\"bbox\": [10, 50, 25, 60]},\n", + " }\n", + " }\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Run the QC\n", + "Create the config stream and run it on the salinity data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "c = Config(config)\n", + "qc = XarrayStream(ds, lon=\"longitude\", lat=\"latitude\")\n", + "runner = list(qc.run(c))\n", + "results = collect_results(runner, how='list')\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Aggregate results\n", + "\n", + "This makes the plotting a bit simpler, as we roll up the flags into one array" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "agg = CollectedResult(\n", + " stream_id='',\n", + " package='qartod',\n", + " test='qc_rollup',\n", + " function=aggregate,\n", + " results=aggregate(results),\n", + " tinp=qc.time(),\n", + " data=ds\n", + ")\n", + "flag_vals = agg.results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Plot results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "time = ds.time\n", + "meaning = np.empty(len(time), dtype=object)\n", + "meaning[:] = \"UNKNOWN\"\n", + "meaning[flag_vals == 1] = \"GOOD\"\n", + "meaning[flag_vals == 9] = \"MISSING\"\n", + "meaning[flag_vals == 3] = \"SUSPECT\"\n", + "meaning[flag_vals == 4] = \"FAIL\"\n", + "df = pd.DataFrame({\"time\": time, \"salinity\": ds[\"salinity\"], \"flag\": flag_vals, \"depth\": ds.depth, \"quality control\": meaning})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from bokeh.plotting import figure, show\n", + "from bokeh.sampledata.penguins import data\n", + "from bokeh.transform import factor_cmap, factor_mark\n", + "\n", + "flag_vals = [\"GOOD\",\"UNKNOWN\",\"MISSING\",\"SUSPECT\",\"FAIL\"]\n", + "markers = ['hex', 'circle_x', 'circle', 'triangle', 'square']\n", + "\n", + "p = figure(title = \"Salinity flags\", background_fill_color=\"#fafafa\", x_axis_type='datetime')\n", + "p.yaxis.axis_label = 'salinity (PSU)'\n", + "\n", + "p.scatter(\"time\", \"salinity\", source=df,\n", + " legend_group=\"quality control\", fill_alpha=0.4, size=12,\n", + " marker=factor_mark('quality control', markers, flag_vals),\n", + " color=factor_cmap('quality control', 'Category10_5', flag_vals))\n", + "\n", + "p.legend.location = \"top_left\"\n", + "p.legend.title = \"IOOS flags\"\n", + "\n", + "show(p)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.4" + }, + "nbsphinx": { + "orphan": true + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}