update fetcher and changelog

api/fetcher.py

@@ -68,14 +68,17 @@ def get_data_static(url):
             soup = bs(response.text, 'html.parser')
             temperature_element = soup.find('div', class_='delta rtw_temp')
             if temperature_element:
-                temperature = int(temperature_element.text.strip().replace("°", "").replace("C", ""))
-                return temperature
+                try:
+                    temperature = int(temperature_element.text.strip().replace("°", "").replace("C", ""))
+                    return temperature
+                except ValueError as e:
+                    log.error(f"Wetter.com could not parse value {str(e)}")
             else:
                 log.error("Temperature element not found on the page.")
-                return None
         else:
             log.error("Failed to retrieve weather data.")
-            return None
+
+        return None
 
     @staticmethod
     def get_data_dynamic(url):
@@ -89,7 +92,9 @@ def get_data_dynamic(url):
         options.add_argument('--disable-blink-features=AutomationControlled')
         service = webdriver.ChromeService(executable_path='/usr/lib/chromium-browser/chromedriver')
         driver = webdriver.Chrome(service=service, options=options)
-        driver.set_page_load_timeout(30) # 30 seconds timeout
+        timeout_s = 30
+        driver.set_page_load_timeout(timeout_s) 
+        driver.implicitly_wait(timeout_s)
         try:
             driver.get(url)
             found_temp = driver.find_element(By.XPATH, '//div[@class="delta rtw_temp"]')
@@ -99,6 +104,7 @@ def get_data_dynamic(url):
             log.error(f"An error occurred while dynamically fetching temperature data: {str(e)}")
             return None
         finally:
+            display.stop()
             driver.quit()
 
 

changelog.md

@@ -5,7 +5,7 @@
 ## 0.3.1
 
 - Added data fetcher, database handler and visualization for **Ulm.de**
-- Fixed deadlocks causing whole Pi to be unresponsive (database recovery mode was triggered)
+- Fixed deadlocks causing whole Pi to be unresponsive (database recovery mode was triggered by to many pending connections)
 
 ## 0.3
 

fetch_forecasts.py

@@ -85,18 +85,20 @@ def google_fetch_and_save():
 
 def wettercom_fetch_and_save():
     # dyn is allowed to be null in database
-    wettercom_temp_dyn = WetterComFetcher.get_data_dynamic(config["wettercom"]["url"][1:-1])
     wettercom_temp_static = WetterComFetcher().get_data_static(config["wettercom"]["url"][1:-1])
-    c_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
     if wettercom_temp_static is None:
-        log.error("[Wetter.com] Error while retrieving temperature")
-    else:
-        log.info(
-            f"[Wetter.com] Static vs Dynamic Temperature at {c_time} is {wettercom_temp_static}°C vs {wettercom_temp_dyn}°C")
-        auth = config["db"]
-        handler = WetterComHandler(auth['db_port'], auth['db_host'], auth['db_user'], auth['db_pw'], 'wettercom_data')
-        handler.init_db_connection()
-        handler.insert_wettercom_data(timestamp=c_time, temp_stat=wettercom_temp_static, temp_dyn=wettercom_temp_dyn)
+        log.error("[Wetter.com] Failed to fetch static temp data. Skipping!")
+        return
+    wettercom_temp_dyn = WetterComFetcher.get_data_dynamic(config["wettercom"]["url"][1:-1])
+    c_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    if wettercom_temp_dyn is None:
+        log.warning("[Wetter.com] Failed to fetch dynamic temp data.")
+    log.info(
+        f"[Wetter.com] Static vs Dynamic Temperature at {c_time} is {wettercom_temp_static}°C vs {wettercom_temp_dyn}°C")
+    auth = config["db"]
+    handler = WetterComHandler(auth['db_port'], auth['db_host'], auth['db_user'], auth['db_pw'], 'wettercom_data')
+    handler.init_db_connection()
+    handler.insert_wettercom_data(timestamp=c_time, temp_stat=wettercom_temp_static, temp_dyn=wettercom_temp_dyn)
 
 
 def ulmde_fetch_and_save():
@@ -119,13 +121,13 @@ def main():
     schedule.every(10).minutes.do(ulmde_fetch_and_save)
     schedule.every(10).minutes.do(dwd_fetch_and_save)
     schedule.every(10).minutes.do(google_fetch_and_save)
-    schedule.every(10).minutes.do(run_threaded, wettercom_fetch_and_save)
+    schedule.every(10).minutes.do(wettercom_fetch_and_save)
 
     log.info("finished initialization")
     ulmde_fetch_and_save()
     dwd_fetch_and_save()
     google_fetch_and_save()
-    run_threaded(wettercom_fetch_and_save)
+    wettercom_fetch_and_save()
 
     while True:
         schedule.run_pending()