Merge branch 'dev' into introducing-log-aggregators-support

CHANGELOG.md

@@ -1,28 +1,76 @@
 # Changelog
 
-## [1.9.2a3](https://github.com/NeonGeckoCom/neon-utils/tree/1.9.2a3) (2024-04-04)
+## [1.10.2a9](https://github.com/NeonGeckoCom/neon-utils/tree/1.10.2a9) (2024-06-25)
 
-[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.9.2a2...1.9.2a3)
+[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.10.2a8...1.10.2a9)
 
 **Merged pull requests:**
 
-- Add ovos-core 0.0.8 compat. for CommonQuery skills [\#508](https://github.com/NeonGeckoCom/neon-utils/pull/508) ([NeonDaniel](https://github.com/NeonDaniel))
+- Fix bug causing old token to be used with requests after refreshing [\#530](https://github.com/NeonGeckoCom/neon-utils/pull/530) ([NeonDaniel](https://github.com/NeonDaniel))
 
-## [1.9.2a2](https://github.com/NeonGeckoCom/neon-utils/tree/1.9.2a2) (2024-04-02)
+## [1.10.2a8](https://github.com/NeonGeckoCom/neon-utils/tree/1.10.2a8) (2024-06-17)
 
-[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.9.2a1...1.9.2a2)
+[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.10.2a7...1.10.2a8)
 
 **Merged pull requests:**
 
-- Loosen ovos-utils dependency to allow 0.1 [\#507](https://github.com/NeonGeckoCom/neon-utils/pull/507) ([NeonDaniel](https://github.com/NeonDaniel))
+- Mitigate issues with expired HANA tokens [\#529](https://github.com/NeonGeckoCom/neon-utils/pull/529) ([NeonDaniel](https://github.com/NeonDaniel))
 
-## [1.9.2a1](https://github.com/NeonGeckoCom/neon-utils/tree/1.9.2a1) (2024-03-07)
+## [1.10.2a7](https://github.com/NeonGeckoCom/neon-utils/tree/1.10.2a7) (2024-05-31)
 
-[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.9.1...1.9.2a1)
+[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.10.2a6...1.10.2a7)
 
 **Merged pull requests:**
 
-- Mark `neon_must_respond` method as deprecated [\#505](https://github.com/NeonGeckoCom/neon-utils/pull/505) ([NeonDaniel](https://github.com/NeonDaniel))
+- Refactor to use pytz instead of pendulum [\#527](https://github.com/NeonGeckoCom/neon-utils/pull/527) ([NeonDaniel](https://github.com/NeonDaniel))
+
+## [1.10.2a6](https://github.com/NeonGeckoCom/neon-utils/tree/1.10.2a6) (2024-05-31)
+
+[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.10.2a5...1.10.2a6)
+
+**Merged pull requests:**
+
+- Resolve Test Failures [\#528](https://github.com/NeonGeckoCom/neon-utils/pull/528) ([NeonDaniel](https://github.com/NeonDaniel))
+
+## [1.10.2a5](https://github.com/NeonGeckoCom/neon-utils/tree/1.10.2a5) (2024-05-21)
+
+[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.10.2a4...1.10.2a5)
+
+**Merged pull requests:**
+
+- Refactor to use HANA to send email instead of MQ [\#526](https://github.com/NeonGeckoCom/neon-utils/pull/526) ([NeonDaniel](https://github.com/NeonDaniel))
+
+## [1.10.2a4](https://github.com/NeonGeckoCom/neon-utils/tree/1.10.2a4) (2024-05-11)
+
+[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.10.2a3...1.10.2a4)
+
+**Merged pull requests:**
+
+- Update deprecated import to supported path [\#524](https://github.com/NeonGeckoCom/neon-utils/pull/524) ([NeonDaniel](https://github.com/NeonDaniel))
+
+## [1.10.2a3](https://github.com/NeonGeckoCom/neon-utils/tree/1.10.2a3) (2024-05-10)
+
+[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.10.2a2...1.10.2a3)
+
+**Merged pull requests:**
+
+- Fix ovos-core 0.0.8 Common Query compat. [\#523](https://github.com/NeonGeckoCom/neon-utils/pull/523) ([NeonDaniel](https://github.com/NeonDaniel))
+
+## [1.10.2a2](https://github.com/NeonGeckoCom/neon-utils/tree/1.10.2a2) (2024-05-08)
+
+[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.10.2a1...1.10.2a2)
+
+**Merged pull requests:**
+
+- Remove old patch [\#522](https://github.com/NeonGeckoCom/neon-utils/pull/522) ([NeonDaniel](https://github.com/NeonDaniel))
+
+## [1.10.2a1](https://github.com/NeonGeckoCom/neon-utils/tree/1.10.2a1) (2024-05-07)
+
+[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.10.1...1.10.2a1)
+
+**Merged pull requests:**
+
+- Enable `hana` backend URL to be configured [\#521](https://github.com/NeonGeckoCom/neon-utils/pull/521) ([NeonDaniel](https://github.com/NeonDaniel))
 
 
 

neon_utils/hana_utils.py

@@ -29,19 +29,39 @@
 import requests
 import json
 
+from typing import Optional
 from os import makedirs
 from os.path import join, isfile, isdir, dirname
 from time import time
 from ovos_utils.log import LOG
 from ovos_utils.xdg_utils import xdg_cache_home
 
-_DEFAULT_BACKEND_URL = "https://hana.neonaiservices.com"
+_DEFAULT_BACKEND_URL = None
 _client_config = {}
 _headers = {}
 
 
-def _get_client_config_path(url: str = _DEFAULT_BACKEND_URL):
-    url_key = hash(url)
+def set_default_backend_url(url: Optional[str] = None):
+    """
+    Set the default backend URL
+    @param url: HANA backend url to use, else read from configuration
+    """
+    global _DEFAULT_BACKEND_URL
+    if not url:
+        from ovos_config.config import Configuration
+        url = Configuration().get('hana', {}).get('url') or \
+            "https://hana.neonaiservices.com"
+    if url and url != _DEFAULT_BACKEND_URL:
+        LOG.info(f"Updating HANA backend URL to {url}")
+        _DEFAULT_BACKEND_URL = url
+        global _client_config
+        global _headers
+        _client_config = {}
+        _headers = {}
+
+
+def _get_client_config_path(url: str):
+    url_key = url.split('/')[2]
     return join(xdg_cache_home(), "neon", f"hana_token_{url_key}.json")
 
 
@@ -112,6 +132,11 @@ def _refresh_token(backend_address: str):
         raise ServerException(f"Error updating token from {backend_address}. "
                               f"{update.status_code}: {update.text}")
     _client_config = update.json()
+
+    # Update request headers with new token
+    global _headers
+    _headers['Authorization'] = f"Bearer {_client_config['access_token']}"
+
     client_config_path = _get_client_config_path(backend_address)
     with open(client_config_path, "w+") as f:
         json.dump(_client_config, f, indent=2)
@@ -126,16 +151,45 @@ def request_backend(endpoint: str, request_data: dict,
     @param server_url: Base URL of Hana server to query
     @returns: dict response
     """
+    global _client_config
+    global _headers
+    if not server_url:
+        set_default_backend_url()
+        server_url = _DEFAULT_BACKEND_URL
+    if server_url != _DEFAULT_BACKEND_URL and _client_config:
+        LOG.info(f"Using new remote: {server_url}")
+        _client_config = {}
+        _headers = {}
     _init_client(server_url)
-    if time() >= _client_config.get("expiration", 0):
+    if _client_config.get("expiration", 0) - time() < 30:
         try:
             _refresh_token(server_url)
         except ServerException as e:
             LOG.error(e)
             _get_token(server_url)
-    resp = requests.post(f"{server_url}/{endpoint.lstrip('/')}",
-                         json=request_data, headers=_headers)
+    request_kwargs = {"url": f"{server_url}/{endpoint.lstrip('/')}",
+                      "json": request_data, "headers": _headers}
+    resp = requests.post(**request_kwargs)
+    if resp.status_code == 502:
+        # This is raised occasionally on valid requests. Need to resolve in HANA
+        resp = requests.post(**request_kwargs)
     if resp.ok:
         return resp.json()
+
     else:
+        try:
+            error = resp.json()["detail"]
+            # Token is actually expired, refresh and retry
+            if error == "Invalid or expired token.":
+                LOG.warning(f"Token is expired. time={time()}|"
+                            f"expiration={_client_config.get('expiration')}")
+                _refresh_token(server_url)
+                resp = requests.post(**request_kwargs)
+                if resp.ok:
+                    return resp.json()
+        except Exception as e:
+            LOG.error(e)
+            # Clear cached config to force re-evaluation on next request
+            _client_config = {}
+            _headers = {}
         raise ServerException(f"Error response {resp.status_code}: {resp.text}")

neon_utils/location_utils.py

@@ -25,18 +25,19 @@
 # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 # SOFTWARE,  EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+from time import time
 
-import pendulum
+import pytz
 
 from datetime import datetime
 from typing import Optional, Union
+
 from dateutil.tz import tzlocal
-from geopy.exc import GeocoderServiceError
-from geopy.geocoders import Nominatim
 from timezonefinder import TimezoneFinder
 from re import sub
 from ovos_utils.log import LOG
 
+from neon_utils.hana_utils import request_backend
 
 # geocode.maps.co nominatim.openstreetmap.org
 _NOMINATIM_DOMAIN = "nominatim.openstreetmap.org"
@@ -63,22 +64,30 @@ def get_full_location(address: Union[str, tuple],
         None if service is not available
     """
     try:
-        nominatim = Nominatim(user_agent="neon-ai", domain=_NOMINATIM_DOMAIN,
-                              timeout=10)
         if isinstance(address, str):
-            location = nominatim.geocode(address, addressdetails=True,
-                                         language=lang)
+            response = request_backend("proxy/geolocation/geocode",
+                                       {"address": address})
+            coords = (response.get('lat'), response.get('lon'))
         else:
-            location = nominatim.reverse(address, addressdetails=True,
-                                         language=lang)
+            coords = address
 
-        dict_location = location.raw
+        dict_location = request_backend("proxy/geolocation/reverse",
+                                        {"lat": coords[0], "lon": coords[1]})
         dict_location['address']['country'] = sub(f'[0-9]', '',
                                                   dict_location['address'].
                                                   get('country'))
+        if lang:
+            try:
+                # TODO: make this optional with a deprecation notice
+                from geopy.geocoders import Nominatim
+                resp = Nominatim(user_agent="neon-ai", domain=_NOMINATIM_DOMAIN,
+                                 timeout=10).reverse(coords, language=lang)
+                return resp.raw
+            except ImportError:
+                LOG.error("geopy not installed")
+            except Exception as e:
+                LOG.exception(e)
         return dict_location
-    except GeocoderServiceError as e:
-        LOG.error(e)
     except Exception as e:
         LOG.exception(e)
     return None
@@ -90,14 +99,14 @@ def get_coordinates(gps_loc: dict) -> (float, float):
     :param gps_loc: dict of "city", "state", "country"
     :return: lat, lng float values
     """
-    coordinates = Nominatim(user_agent="neon-ai", domain=_NOMINATIM_DOMAIN,
-                            timeout=10)
     try:
-        location = coordinates.geocode(gps_loc)
+        request_str = ', '.join((x for x in [gps_loc.get('city'),
+                                             gps_loc.get('state'),
+                                             gps_loc.get('country')] if x))
+        location = request_backend("proxy/geolocation/geocode",
+                                   {"address": request_str})
         LOG.debug(f"{location}")
-        return location.latitude, location.longitude
-    except GeocoderServiceError as e:
-        LOG.error(e)
+        return float(location.get('lat')), float(location.get('lon'))
     except Exception as x:
         LOG.exception(x)
     return -1, -1
@@ -112,24 +121,21 @@ def get_location(lat, lng) -> (str, str, str, str):
     :return: city, county, state, country
     """
     try:
-        address = Nominatim(user_agent="neon-ai", domain=_NOMINATIM_DOMAIN,
-                            timeout=10)
-    except GeocoderServiceError as e:
-        LOG.error(e)
-        return None
+        location = request_backend("proxy/geolocation/reverse",
+                                   {"lat": lat, "lon": lng})
+
+        dict_location = location.get('address')
     except Exception as x:
         LOG.exception(x)
         return None
-    location = address.reverse([lat, lng], language="en-US")
     LOG.debug(f"{location}")
-    LOG.debug(f"{location.raw}")
-    LOG.debug(f"{location.raw.get('address')}")
-    city = location.raw.get('address').get('city') or \
-        location.raw.get('address').get('town') or \
-        location.raw.get('address').get('village')
-    county = location.raw.get('address').get('county')
-    state = location.raw.get('address').get('state')
-    country = location.raw.get('address').get('country')
+    city = dict_location.get('city') or \
+        dict_location.get('town') or \
+        dict_location.get('village') or \
+        dict_location.get('hamlet')
+    county = dict_location.get('county')
+    state = dict_location.get('state')
+    country = dict_location.get('country')
     return city, county, state, country
 
 
@@ -139,10 +145,11 @@ def get_timezone(lat, lng) -> (str, float):
     Note that some coordinates do not have a city, but may have a county.
     :param lat: latitude
     :param lng: longitude
-    :return: timezone name, offset from GMT
+    :return: timezone name, offset in hours from UTC
     """
     timezone = TimezoneFinder().timezone_at(lng=float(lng), lat=float(lat))
-    offset = pendulum.from_timestamp(0, timezone).offset_hours
+    offset = pytz.timezone(timezone).utcoffset(
+        datetime.now()).total_seconds() / 3600
     return timezone, offset
 
 

neon_utils/metrics_utils.py

@@ -67,7 +67,11 @@ def start(self):
         self.start_time = time()
 
     def stop(self):
-        self.time = time() - self.start_time
+        try:
+            self.time = time() - self.start_time
+        except TypeError:
+            LOG.error("stop called before start!")
+            self.time = None
         return self.time
 
     def report(self):

neon_utils/mq_utils.py

@@ -29,8 +29,10 @@
 import logging
 import uuid
 
-from ovos_utils.log import deprecated
+from ovos_utils.log import deprecated, log_deprecation
 
+log_deprecation("This module has moved to neon_mq_connector.utils.client_utils",
+                "2.0.0")
 try:
     from threading import Event
     from pika.channel import Channel
@@ -47,7 +49,7 @@
 logging.getLogger("pika").setLevel(logging.CRITICAL)
 
 _default_mq_config = {
-    "server": "api.neon.ai",
+    "server": "mq.neonaiservices.com",
     "port": 5672,
     "users": {
         "mq_handler": {
@@ -60,6 +62,8 @@
 
 class NeonMQHandler(MQConnector):
     def __init__(self, config: dict, service_name: str, vhost: str):
+        log_deprecation("Import from neon_mq_connector.utils.client_utils",
+                        "2.0.0")
         super().__init__(config, service_name)
         self.vhost = vhost
         import pika
@@ -70,13 +74,11 @@ def __init__(self, config: dict, service_name: str, vhost: str):
 @deprecated("Use `neon_mq_connector.client.send_mq_request`", "2.0.0")
 def get_mq_response(vhost: str, request_data: dict, target_queue: str,
                     response_queue: str = None, timeout: int = 30) -> dict:
-    # TODO: Remove in v1.0.0 DM
     return send_mq_request(vhost, request_data, target_queue, response_queue,
                            timeout, True)
 
 
-# TODO: Mark deprecation after stable neon_mq_connector release
-# @deprecated("Use `neon_mq_connector.client.send_mq_request`", "2.0.0")
+@deprecated("Import from neon_mq_connector.utils.client_utils", "2.0.0")
 def send_mq_request(vhost: str, request_data: dict, target_queue: str,
                     response_queue: str = None, timeout: int = 30,
                     expect_response: bool = True) -> dict:

neon_utils/skills/common_play_skill.py

@@ -48,7 +48,7 @@
 
 from neon_utils.skills.neon_skill import NeonSkill
 
-from ovos_utils.skills.audioservice import AudioServiceInterface as AudioService
+from ovos_bus_client.apis.ocp import ClassicAudioServiceInterface as AudioService
 
 
 class CPSMatchLevel(Enum):