From 9188215a176dc2ae2012264b35cd7efacb00759e Mon Sep 17 00:00:00 2001 From: deralden Date: Mon, 19 Aug 2024 17:55:24 +0300 Subject: [PATCH 1/3] Opensearch introduction --- service_status/service_status.py | 38 ++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/service_status/service_status.py b/service_status/service_status.py index 7e99a343f..49df4ca5d 100644 --- a/service_status/service_status.py +++ b/service_status/service_status.py @@ -2,7 +2,7 @@ import re from datetime import datetime as dt from datetime import timedelta - +from opensearchpy import OpenSearch import grpc from grpc_health.v1 import health_pb2 as heartb_pb2 from grpc_health.v1 import health_pb2_grpc as heartb_pb2_grpc @@ -11,7 +11,7 @@ from common.logger import get_logger from common.utils import Utils from resources.certificates.root_certificate import certificate -from service_status.config import REGION_NAME, NOTIFICATION_ARN, SLACK_HOOK, NETWORKS, NETWORK_ID, \ +from service_status.config import REGION_NAME, NOTIFICATION_ARN, SLACK_HOOK, NETWORKS, NETWORK_ID, HOST, AUTH, \ MAXIMUM_INTERVAL_IN_HOUR, MINIMUM_INTERVAL_IN_HOUR, NETWORK_NAME, BASE_URL_TO_RESET_SERVICE_HEALTH from service_status.constant import SRVC_STATUS_GRPC_TIMEOUT, LIMIT @@ -78,6 +78,39 @@ def _update_service_failed_status_count(self, failed_status_count, row_id): response = self.repo.execute(update_query, [failed_status_count, row_id]) return response + def _send_logs_to_opensearch(self, service_id, debug_error_string): + client = OpenSearch( + http_compress = True, + hosts = [{'host': HOST, 'port': 443}], + http_auth = AUTH, + use_ssl = True, + verify_certs = True, + ssl_assert_hostname = False, + ssl_show_warn = False, + ) + timestamp = dt.datetime.now(dt.timezone.utc) + index_name = f"services-logs-{NETWORKS[NETWORK_ID]['name']}-{timestamp.strftime('%Y.%m.%d')}" + if not client.indices.exists(index_name): + index_body = { + 'settings': { + 'index': { + 'number_of_shards': 1 + } + } + } + response = client.indices.create(index_name, body=index_body) + document = { + '@timestamp': timestamp, + 'Log': debug_error_string, + 'Service': service_id + } + response = client.index( + index = index_name, + body = document, + refresh = True + ) + return response + def _update_service_status_stats(self, org_id, service_id, old_status, status): previous_state = "UP" if (old_status == 1) else "DOWN" current_state = "UP" if (status == 1) else "DOWN" @@ -117,6 +150,7 @@ def update_service_status(self): org_id = record["org_id"] service_id = record["service_id"] recipients = self._get_service_provider_email(org_id=org_id, service_id=service_id) + self._send_logs_to_opensearch(service_id=service_id, debug_error_string=debug_error_string) if failed_status_count <= 10: self._send_notification(org_id=org_id, service_id=service_id, recipients=recipients, endpoint=record["endpoint"], error_details=error_details, From 39c616217cb891c306ec55b03a91acc6767582ef Mon Sep 17 00:00:00 2001 From: Deralden Date: Mon, 19 Aug 2024 18:05:44 +0300 Subject: [PATCH 2/3] Add endpoint information to logs --- service_status/service_status.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/service_status/service_status.py b/service_status/service_status.py index 49df4ca5d..32d4b55b1 100644 --- a/service_status/service_status.py +++ b/service_status/service_status.py @@ -78,7 +78,7 @@ def _update_service_failed_status_count(self, failed_status_count, row_id): response = self.repo.execute(update_query, [failed_status_count, row_id]) return response - def _send_logs_to_opensearch(self, service_id, debug_error_string): + def _send_logs_to_opensearch(self, service_id, debug_error_string, endpoint): client = OpenSearch( http_compress = True, hosts = [{'host': HOST, 'port': 443}], @@ -102,7 +102,8 @@ def _send_logs_to_opensearch(self, service_id, debug_error_string): document = { '@timestamp': timestamp, 'Log': debug_error_string, - 'Service': service_id + 'Service': service_id, + 'Endpoint': endpoint } response = client.index( index = index_name, @@ -150,7 +151,7 @@ def update_service_status(self): org_id = record["org_id"] service_id = record["service_id"] recipients = self._get_service_provider_email(org_id=org_id, service_id=service_id) - self._send_logs_to_opensearch(service_id=service_id, debug_error_string=debug_error_string) + self._send_logs_to_opensearch(service_id=service_id, debug_error_string=debug_error_string, endpoint=record["endpoint"]) if failed_status_count <= 10: self._send_notification(org_id=org_id, service_id=service_id, recipients=recipients, endpoint=record["endpoint"], error_details=error_details, From a891270b094e8372c99bd576a4d1af867d5e2872 Mon Sep 17 00:00:00 2001 From: Deralden Date: Mon, 19 Aug 2024 18:20:18 +0300 Subject: [PATCH 3/3] Opensearch package added --- service_status/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/service_status/requirements.txt b/service_status/requirements.txt index 07dba1372..aa4e9149e 100644 --- a/service_status/requirements.txt +++ b/service_status/requirements.txt @@ -9,3 +9,4 @@ pymysql==0.10.1 aws-xray-sdk==2.4.2 grpcio-tools==1.32.0 grpcio-health-checking==1.32.0 +opensearch-py