Merge pull request #33 from wurstbroteater/dev

Release HomeTemp v3.1

api/fetcher.py

@@ -11,6 +11,37 @@
 
 log = logging.getLogger("api.fetcher")
 
+class UlmDeFetcher:
+
+    @staticmethod
+    def get_data():
+        """
+        Fetches the temperature data from ulm.de
+        """
+        try:
+            # set connect and read timeout to 5 seconds
+            response = requests.get('https://www.ulm.de/', timeout=(5,5))
+        except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e:
+            log.error(f"Ulm.de connection problem: " + str(e))
+            return None
+
+        if response.status_code == 200:
+            soup = bs(response.text, 'html.parser')
+            temperature_element = soup.find('p', class_='temp')
+            if temperature_element:
+                try:
+                    temperature = int(temperature_element.text.replace("°", "").replace("C", "").strip())
+                except ValueError as e:
+                    log.error(f"Ulm.de could not parse value {str(e)}")
+                    return None
+                return temperature
+            else:
+                log.error("Ulm.de: Temperature element not found on the page.")
+                return None
+        else:
+            log.error("Ulm.de: Failed to retrieve weather data.")
+            return None
+
 
 class WetterComFetcher:
     """
@@ -27,23 +58,27 @@ def get_data_static(url):
         Fetches the static temperature data from Wetter.com link for a city/region
         """
         try:
-            response = requests.get(url)
-        except requests.exceptions.ConnectionError as e:
-            log.error("Wetter.com connection problem: " + str(e))
+            # set connect and read timeout to 5 seconds
+            response = requests.get(url, timeout=(5,5))
+        except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e:
+            log.error(f"Wetter.com connection problem: " + str(e))
             return None
 
         if response.status_code == 200:
             soup = bs(response.text, 'html.parser')
             temperature_element = soup.find('div', class_='delta rtw_temp')
             if temperature_element:
-                temperature = int(temperature_element.text.strip().replace("°", "").replace("C", ""))
-                return temperature
+                try:
+                    temperature = int(temperature_element.text.strip().replace("°", "").replace("C", ""))
+                    return temperature
+                except ValueError as e:
+                    log.error(f"Wetter.com could not parse value {str(e)}")
             else:
                 log.error("Temperature element not found on the page.")
-                return None
         else:
             log.error("Failed to retrieve weather data.")
-            return None
+
+        return None
 
     @staticmethod
     def get_data_dynamic(url):
@@ -57,6 +92,9 @@ def get_data_dynamic(url):
         options.add_argument('--disable-blink-features=AutomationControlled')
         service = webdriver.ChromeService(executable_path='/usr/lib/chromium-browser/chromedriver')
         driver = webdriver.Chrome(service=service, options=options)
+        timeout_s = 30
+        driver.set_page_load_timeout(timeout_s) 
+        driver.implicitly_wait(timeout_s)
         try:
             driver.get(url)
             found_temp = driver.find_element(By.XPATH, '//div[@class="delta rtw_temp"]')
@@ -66,6 +104,7 @@ def get_data_dynamic(url):
             log.error(f"An error occurred while dynamically fetching temperature data: {str(e)}")
             return None
         finally:
+            display.stop()
             driver.quit()
 
 
@@ -87,15 +126,16 @@ def get_weather_data(location: str):
             session.headers["User-Agent"] = user_agent
             session.headers["Accept-Language"] = language
             session.headers["Content-Language"] = language
-            html = session.get(url)
+            # set connect and read timeout to 5 seconds
+            html = session.get(url, timeout=(5,5))
             soup = bs(html.text, "html.parser")
 
             return {"region": soup.find("div", attrs={"id": "wob_loc"}).text,
                     "temp_now": float(soup.find("span", attrs={"id": "wob_tm"}).text),
                     "precipitation": float(soup.find("span", attrs={"id": "wob_pp"}).text.replace("%", "")),
                     "humidity": float(soup.find("span", attrs={"id": "wob_hm"}).text.replace("%", "")),
                     "wind": float(soup.find("span", attrs={"id": "wob_ws"}).text.replace(" km/h", ""))}
-        except requests.exceptions.ConnectionError as e:
+        except (AttributeError, requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e:
             log.error("Google Weather connection problem: " + str(e))
             return None
 
@@ -111,6 +151,7 @@ def __init__(self, endpoint: str, params: list = None):
         self.endpoint = endpoint
         self.params = None if not params else urllib.parse.urlencode(params)
         self.api_link = self._create_api_link()
+        self.timeouts = (30,5) # connect timeout 30s and read timeout 5s
 
     def __str__(self):
         return f"Fetcher[{self.api_link}]"
@@ -120,12 +161,12 @@ def _create_api_link(self):
 
     def _fetch_data(self):
         try:
-            response = requests.get(self.api_link)
+            response = requests.get(self.api_link, timeout=self.timeouts)
             if (code := response.status_code) == 200:
                 return self._handle_ok_status_code(response)
             else:
                 return self._handle_bad_status_code(code)
-        except requests.exceptions.ConnectionError as e:
+        except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e:
             log.error("Creating the connection failed with error: {e}")
             return None
 
@@ -200,16 +241,16 @@ def get_dwd_data(self, trigger_new_fetch=True):
 
         if len(temp_std) != len(temp_values):
             log.error(f"Error: Unable to validate DWD temperature data because temp values and std differ!")
-            return current_time, float("nan"), float("nan")
+            return current_time, None, None
 
         current_temp_forecast_index = self._get_index()
         if len(temp_values) < current_temp_forecast_index:
             log.error(
                 f"Error: Forecast index out of range, size: {len(temp_values)}, index: {current_temp_forecast_index}")
-            return current_time, float("nan"), float("nan")
+            return current_time, None, None
         elif temp_std[current_temp_forecast_index] == 0:
             log.error(f"Error: 0 tempStd for found temperature {temp_values[current_temp_forecast_index]}")
-            return current_time, float("nan"), float("nan")
+            return current_time, None, None
         else:
             temp = float(temp_values[current_temp_forecast_index]) / 10.0
             dev = self.data["temperatureStd"][current_temp_forecast_index]

changelog.md

@@ -1,5 +1,12 @@
 # Project: HomeTemp
 
+## 0.4
+
+## 0.3.1
+
+- Added data fetcher, database handler and visualization for **Ulm.de**
+- Fixed deadlocks causing whole Pi to be unresponsive (database recovery mode was triggered by to many pending connections)
+
 ## 0.3
 
 - Refactored code to modules `api`, `distribute`, `persist` and `visualize`

crunch_numbers.ipynb

@@ -14,12 +14,12 @@
     "import pandas as pd\n",
     "import seaborn as sns\n",
     "from datetime import datetime, timedelta\n",
-    "from api.fetcher import DWDFetcher\n",
-    "from persist.database import DwDDataHandler, SensorDataHandler, GoogleDataHandler, WetterComHandler\n",
+    "from api.fetcher import DWDFetcher, UlmDeFetcher\n",
+    "from persist.database import DwDDataHandler, SensorDataHandler, GoogleDataHandler, UlmDeHandler, WetterComHandler\n",
     "from util.manager import DockerManager, PostgresDockerManager\n",
     "from visualize.plots import draw_plots\n",
     "\n",
-    "auth = config[\"db\"]"
+    "auth = config[\"db\"]\n"
    ]
   },
   {
@@ -28,11 +28,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "wettercom_handler = WetterComHandler(auth['db_port'], auth['db_host'], auth['db_user'], auth['db_pw'], 'wettercom_data')\n",
-    "wettercom_handler.init_db_connection()\n",
-    "wettercom_df = wettercom_handler.read_data_into_dataframe()\n",
-    "wettercom_df['timestamp'] = wettercom_df['timestamp'].map(lambda x: datetime.strptime(str(x).strip().replace('+00:00', ''), '%Y-%m-%d %H:%M:%S'))\n",
-    "wettercom_df"
+    "ulmde_handler = UlmDeHandler(auth['db_port'], auth['db_host'], auth['db_user'], auth['db_pw'], 'ulmde_data')\n",
+    "ulmde_handler.init_db_connection()\n",
+    "ulmde_df = ulmde_handler.read_data_into_dataframe()\n",
+    "ulmde_df['timestamp'] = ulmde_df['timestamp'].map(lambda x: datetime.strptime(str(x).strip().replace('+00:00', ''), '%Y-%m-%d %H:%M:%S'))\n",
+    "ulmde_df"
    ]
   },
   {
@@ -50,6 +50,19 @@
     "dwd_df.drop(['id', 'timestamp'],axis=1).describe()"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wettercom_handler = WetterComHandler(auth['db_port'], auth['db_host'], auth['db_user'], auth['db_pw'],'wettercom_data')\n",
+    "wettercom_handler.init_db_connection()\n",
+    "wettercom_df = wettercom_handler.read_data_into_dataframe()\n",
+    "wettercom_df['timestamp'] = wettercom_df['timestamp'].map(lambda x: datetime.strptime(str(x).strip().replace('+00:00', ''), '%Y-%m-%d %H:%M:%S'))\n",
+    "wettercom_df"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -128,7 +141,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "draw_plots(df,dwd_df=dwd_df,google_df=google_df,wettercom_df=wettercom_df, with_save=False)"
+    "draw_plots(df,dwd_df=dwd_df,google_df=google_df,wettercom_df=wettercom_df,ulmde_df=ulmde_df, with_save=False)"
    ]
   },
   {

default_hometemp.ini

@@ -1,5 +1,5 @@
 [hometemp]
-version = 0.3
+version = 0.3.1
 
 [db]
 container_name = 

fetch_forecasts.py

@@ -1,8 +1,8 @@
 import logging
 import configparser, schedule, time
 from datetime import datetime, timedelta
-from api.fetcher import DWDFetcher, GoogleFetcher, WetterComFetcher
-from persist.database import DwDDataHandler, GoogleDataHandler, WetterComHandler
+from api.fetcher import DWDFetcher, GoogleFetcher, UlmDeFetcher, WetterComFetcher
+from persist.database import DwDDataHandler, GoogleDataHandler, UlmDeHandler, WetterComHandler
 from hometemp import run_threaded
 
 for handler in logging.root.handlers[:]:
@@ -49,7 +49,9 @@ def dwd_fetch_and_save():
                         old_temp = handler.get_temp_for_timestamp(timestamp_to_update.strftime("%Y-%m-%d %H:%M:%S"))
                         log.info(timestamp_to_update.strftime(
                             "%Y-%m-%d %H:%M:%S") + f" old/new: {old_temp}/{new_temp} {new_dev}")
-                        if old_temp != new_temp:
+                        if old_temp is None:
+                            handler.insert_dwd_data(timestamp_to_update.strftime("%Y-%m-%d %H:%M:%S"), new_temp, new_dev)
+                        elif old_temp != new_temp:
                             handler.update_temp_by_timestamp(timestamp_to_update.strftime("%Y-%m-%d %H:%M:%S"),
                                                              new_temp,
                                                              new_dev)
@@ -83,31 +85,49 @@ def google_fetch_and_save():
 
 def wettercom_fetch_and_save():
     # dyn is allowed to be null in database
-    wettercom_temp_dyn = WetterComFetcher.get_data_dynamic(config["wettercom"]["url"][1:-1])
     wettercom_temp_static = WetterComFetcher().get_data_static(config["wettercom"]["url"][1:-1])
-    c_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
     if wettercom_temp_static is None:
-        log.error("[Wetter.com] Error while retrieving temperature")
+        log.error("[Wetter.com] Failed to fetch static temp data. Skipping!")
+        return
+    wettercom_temp_dyn = WetterComFetcher.get_data_dynamic(config["wettercom"]["url"][1:-1])
+    c_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    if wettercom_temp_dyn is None:
+        log.warning("[Wetter.com] Failed to fetch dynamic temp data.")
+    log.info(
+        f"[Wetter.com] Static vs Dynamic Temperature at {c_time} is {wettercom_temp_static}°C vs {wettercom_temp_dyn}°C")
+    auth = config["db"]
+    handler = WetterComHandler(auth['db_port'], auth['db_host'], auth['db_user'], auth['db_pw'], 'wettercom_data')
+    handler.init_db_connection()
+    handler.insert_wettercom_data(timestamp=c_time, temp_stat=wettercom_temp_static, temp_dyn=wettercom_temp_dyn)
+
+
+def ulmde_fetch_and_save():
+    auth = config["db"]
+    ulm_temp = UlmDeFetcher.get_data()
+    if ulm_temp is None:
+        log.error("[Ulm] Could not receive google data")
     else:
-        log.info(
-            f"[Wetter.com] Static vs Dynamic Temperature at {c_time} is {wettercom_temp_static}°C vs {wettercom_temp_dyn}°C")
-        auth = config["db"]
-        handler = WetterComHandler(auth['db_port'], auth['db_host'], auth['db_user'], auth['db_pw'], 'wettercom_data')
+        c_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        msg = f"[Ulm] Forecast is: {c_time} temp={ulm_temp}°C"
+        log.info(msg)
+        handler = UlmDeHandler(auth['db_port'], auth['db_host'], auth['db_user'], auth['db_pw'], 'ulmde_data')
         handler.init_db_connection()
-        handler.insert_wettercom_data(timestamp=c_time, temp_stat=wettercom_temp_static, temp_dyn=wettercom_temp_dyn)
+        handler.insert_ulmde_data(timestamp=c_time, temp=ulm_temp)
 
 
 def main():
     # Todo: integrate into hometemp for final release
     log.info(f"------------------- Fetch DWD Measurements v{config['hometemp']['version']} -------------------")
+    schedule.every(10).minutes.do(ulmde_fetch_and_save)
     schedule.every(10).minutes.do(dwd_fetch_and_save)
     schedule.every(10).minutes.do(google_fetch_and_save)
-    schedule.every(10).minutes.do(run_threaded, wettercom_fetch_and_save)
+    schedule.every(10).minutes.do(wettercom_fetch_and_save)
 
     log.info("finished initialization")
+    ulmde_fetch_and_save()
     dwd_fetch_and_save()
     google_fetch_and_save()
-    run_threaded(wettercom_fetch_and_save)
+    wettercom_fetch_and_save()
 
     while True:
         schedule.run_pending()

hometemp.py

@@ -2,7 +2,7 @@
 from datetime import datetime, timedelta
 from gpiozero import CPUTemperature
 from distribute.email import EmailDistributor
-from persist.database import DwDDataHandler, GoogleDataHandler, PostgresHandler, SensorDataHandler, WetterComHandler
+from persist.database import DwDDataHandler, GoogleDataHandler, UlmDeHandler, SensorDataHandler, WetterComHandler
 from util.manager import PostgresDockerManager
 from visualize.plots import draw_plots
 
@@ -74,8 +74,13 @@ def create_and_backup_visualization():
     wettercom_df = wettercom_handler.read_data_into_dataframe()
     wettercom_df['timestamp'] = wettercom_df['timestamp'].map(
         lambda x: datetime.strptime(str(x).strip().replace('+00:00', ''), '%Y-%m-%d %H:%M:%S'))
+    # Ulm.de data
+    ulmde_handler = UlmDeHandler(auth['db_port'], auth['db_host'], auth['db_user'], auth['db_pw'], 'ulmde_data')
+    ulmde_handler.init_db_connection()
+    ulmde_df = ulmde_handler.read_data_into_dataframe()
+    ulmde_df['timestamp'] = ulmde_df['timestamp'].map(lambda x: datetime.strptime(str(x).strip().replace('+00:00', ''), '%Y-%m-%d %H:%M:%S'))
 
-    draw_plots(df, google_df=google_df, dwd_df=dwd_df, wettercom_df=wettercom_df)
+    draw_plots(df, google_df=google_df, dwd_df=dwd_df, wettercom_df=wettercom_df, ulmde_df=ulmde_df)
     log.info("Done")
     EmailDistributor.send_visualization_email(df, google_df=google_df, dwd_df=dwd_df, wettercom_df=wettercom_df)
 

persist/database.py

@@ -49,7 +49,14 @@ def init_db_connection(self, check_table=True):
     def _init_db(self):
         db_url = f"postgresql://{self.user}:{self.password}@{self.host}:{self.port}"
         try:
-            return create_engine(db_url)
+            return create_engine(db_url,
+                    pool_pre_ping=True,
+                    connect_args={
+                        "keepalives": 1,
+                        "keepalives_idle": 30,
+                        "keepalives_interval": 10,
+                        "keepalives_count": 5,
+                    })
         except exc.SQLAlchemyError as e:
             log.error("Problems while initialising database access: " + str(e))
             return None
@@ -327,3 +334,32 @@ def insert_wettercom_data(self, timestamp, temp_stat, temp_dyn):
 
         if was_successful:
             log.info("Wetter.com data inserted successfully.")
+
+
+class UlmDeHandler(PostgresHandler):
+    """
+    Implementation of PostgresHandler with table schema for ulmde_data.
+    In addition, it provides a method for inserting measurement data into the table.
+    """
+
+    def _create_table(self):
+        metadata = MetaData()
+        table_schema = Table(self.table, metadata,
+                             Column('id', Integer, primary_key=True, autoincrement=True),
+                             Column('timestamp', TIMESTAMP(timezone=True), nullable=False),
+                             Column('temp', DECIMAL, nullable=True))
+        try:
+            metadata.create_all(self.connection)
+            log.info(f"Table '{self.table}' created successfully.")
+
+        except exc.SQLAlchemyError as e:
+            log.error("Problem with database " + str(e))
+
+    def insert_ulmde_data(self, timestamp, temp):
+        insert_successful = self._insert_in_table({
+            'timestamp': timestamp,
+            'temp': temp
+        })
+
+        if insert_successful:
+            log.info("Ulm.de data inserted successfully.")