refactor ATN GTS metrics (#105)

* refactor ATN GTS metrics

* Update website_create_and_deploy.yml

adding bs4

.github/workflows/metrics.yml

@@ -8,7 +8,6 @@ on:
      paths:
        - '.github/workflows/metrics.yml'
        - 'btn_metrics.py'
-       - 'gts_atn_metrics.py'
   schedule:
     - cron: "0 12 5 * *"
 
@@ -42,7 +41,6 @@ jobs:
         shell: bash -l {0}
         run: >
           python btn_metrics.py
-          && python gts_atn_metrics.py
 
       - name: Get current date
         run: echo "NOW=$(date -u)" >> ${GITHUB_ENV}

.github/workflows/website_create_and_deploy.yml

@@ -30,6 +30,7 @@ jobs:
         pip install datetime
         pip install fiscalyear
         pip install erddapy
+        pip install beautifulsoup4
     - name: create HTML file
       run: |
         cd website

website/create_gts_atn_landing_page.py

@@ -1,17 +1,15 @@
 # This script run once creates a catalog landing page based on the *_config.json file
 # reference when called. E.g., python create_gts_regional_landing_page.py EcoSys_config.json
-import base64
-from io import BytesIO
 
+from bs4 import BeautifulSoup
 from jinja2 import Environment, FileSystemLoader
 import json
 import os
 import pandas as pd
-import plotly.express as px
 import plotly.graph_objects as go
 from plotly.subplots import make_subplots
 import plotly
-
+import requests
 
 def write_html_index(template, configs, org_config):
     root = os.path.dirname(os.path.abspath(__file__))
@@ -35,22 +33,22 @@ def write_templates(configs, org_config):
 
 
 def timeseries_plot(output):
-    table = output.copy()
 
-    output["date"] = pd.to_datetime(output["date"])
+    output["date"] = pd.to_datetime(output.index.strftime("%Y-%m"))
+
+    table = output[["date","total"]].copy()
+    table["date"] = table.index.strftime("%Y-%m")
 
     figure = go.Figure(
-        # data=[go.Bar(x=[1, 2, 3], y=[1, 3, 2])],
         layout=go.Layout(height=600, width=1500)
     )
 
     fig = make_subplots(
         rows=1,
         cols=3,
-        #        vertical_spacing=0.03,
         specs=[
             [{"type": "table"}, {"colspan": 2, "type": "bar"}, None]
-        ],  # {"type": "bar"},{'colspan': 1}]],
+            ],
         figure=figure,
     )
 
@@ -73,48 +71,65 @@ def timeseries_plot(output):
         row=1,
         col=2,
     )
-    #     title="ATN Messages sent to the GTS via NDBC",
-    #     width=800,
-    #     height=600,
-    #     )
-    # )
-    #
-    # fig.update_xaxes(
-    #     title_text="Date",
-    #     dtick="M3",
-    #     tickformat="%b\n%Y",
-    #     rangeslider_visible=True,
-    #     rangeselector={
-    #         "buttons": [
-    #             dict(count=3, label="3m", step="month", stepmode="backward"),
-    #             dict(count=6, label="6m", step="month", stepmode="backward"),
-    #             dict(count=9, label="9m", step="month", stepmode="backward"),
-    #             dict(count=1, label="1y", step="year", stepmode="backward"),
-    #             dict(step="all"),
-    #         ]
-    #     },
-    # )
-    #
-    # fig.update_yaxes(title_text="Messages Delivered to the GTS")
 
     fig = plotly.io.to_html(fig, full_html=False)
 
     return fig
 
 
-def main(org_config):
-    configs = dict()
+def get_atn_gts_metrics():
 
-    file = "GTS_ATN_monthly_totals.csv"
+    # recursively search the https index for bufr messages
+    url = "https://stage-ndbc-bufr.srv.axds.co/platforms/atn/smru/profiles/"
 
-    filename = os.path.join(org_config["location_of_metrics"], file)
-    output = pd.read_csv(filename)
-    f_out = filename.replace(".csv", ".html").replace(
-        org_config["location_of_metrics"], "deploy"
-    )
+    html = requests.get(url).text
+    soup = BeautifulSoup(html, "html.parser")
+
+    df_out = pd.DataFrame()
+
+    for deployment in soup.find_all("a"):
+        depl_url = url + deployment.text
+        depl_html = requests.get(depl_url).text
+
+        depl_soup = BeautifulSoup(depl_html, "html.parser")
+
+        # some content is not in an html node, so we have to parse line by line
+        files = depl_soup.get_text().split("\r\n")[1:-1]
+
+        for file in files:
+            content = file.split()
+
+            if ".bufr" in content[0]:
+                # save the index file information to DF.
+                fname = deployment.text + content[0]
+                df_file = pd.DataFrame(
+                    {
+                        "fname": [fname],
+                        "date": pd.to_datetime([content[1] + "T" + content[2]]),
+                        "size": [content[3]],
+                    }
+                )
+
+                df_out = pd.concat([df_out, df_file])
+
+    # mask for FY Quarter
+    df_out = df_out.set_index("date").sort_index()
+
+    # groupby month and save data
+    group = df_out.groupby(pd.Grouper(freq="ME"))
+
+    s = group["fname"].count()
+
+    s.index = s.index.to_period("M")
+
+    s = s.rename("total")
+
+    return pd.DataFrame(s)
+
+
+def main(org_config):
 
-    print(f_out)
-    # key = "{} {}".format(f_out.split("\\")[-1].split("_")[0], f_out.split("_")[1])
+    output = get_atn_gts_metrics()
 
     table = output.to_html(
         index=False,
@@ -129,7 +144,6 @@ def main(org_config):
     fig = timeseries_plot(output)
 
     configs = {
-        "name": f_out,
         "data": f,
         "table": table,
         "figure": fig,

website/templates/gts_atn_landing_page.html

@@ -10,8 +10,6 @@
 
     gtag('config',"{{org_config.google_analytics_code}}" );
 
-
-
     </script>
     <title>{{org_config.main_title}}</title>
     <link rel="icon" type="image/png"
@@ -39,16 +37,11 @@ <h1 class="header_title">NOAA Integrated Ocean Observing System Metrics</h1>
         <button class="main_button" onclick="location.href='https://github.com/ioos/ioos_metrics/issues/new?labels=website';" value="Feedback">
             Feedback
         </button>
-        {#<!--<button class="header_button" onclick="location.href='{{org_config.githubrepo}}';" value="Explore Resources">
-            Explore Resources
-        </button>
-        <button class="header_button" onclick="location.href='{{org_config.githubrepo}}';" value="Contact IOOS">Contact
-            IOOS
-        </button>-->#}
     </div>
     <div class="divider"></div>
     <div class="mywrap">
-        <h2 class="title">{{org_config.general_title}} <a href={{org_config.githubrepo}}blob/main/gts/GTS_ATN_monthly_totals.csv>here</a>
+        <h2 class="title">
+            {{org_config.general_title}}
         </h2>
     </div>
 
@@ -68,7 +61,6 @@ <h2 class="title">{{org_config.general_title}} <a href={{org_config.githubrepo}}
             <a class="mylink" href="https://www.ioos.noaa.gov/">NOAA IOOS</a>
         </div>
     </footer>
-    {#<!--<div class ="attribution">Icons by icons8 https://icons8.com/</div>--> #}
 </main>
 <script>
         var sidenav = document.getElementsByClassName("sidenav");
@@ -152,14 +144,6 @@ <h2 class="title">{{org_config.general_title}} <a href={{org_config.githubrepo}}
            }
           // boxcontainer
 
-
-
-
-
-
-
-
-
 </script>
 </body>
 </html>