Merge pull request #1340 from Jaseci-Labs/logger

[LOGGER]: Initial Jac-Cloud logger handler

jac-cloud/.gitignore

@@ -51,3 +51,7 @@ out.txt
 # Mypy files #
 .mypy_cache*
 .jac_mypy_cache
+
+
+# Jac-Cloud #
+logs/

jac-cloud/README.md

@@ -11,4 +11,5 @@ pre-commit install
 ```
 # **REFERENCES**
 ### [**jac-cloud**](./docs/Jaseci.md)
-### [**Environment Variables**](./docs/Environment-Variables.md)
+### [**Environment Variables**](./docs/Environment-Variables.md)
+### [**Logger**](./docs/Logger.md)

jac-cloud/docs/Environment-Variables.md

@@ -25,6 +25,13 @@
 | RESET_CODE_TIMEOUT | Password reset code expiration in hours | 24 |
 | SENDGRID_HOST | Sendgrid host used for hyperlinking verification/reset code | http://localhost:8000 |
 | SENDGRID_API_KEY | Sendgrid api key | null    |
+| LOGGER_NAME   | Specified logger name | app   |
+| LOGGER_LEVEL  | Control log level     | debug |
+| LOGGER_FILE_PATH | Log directory and name | /tmp/jac_cloud_logs/jac-cloud.log |
+| LOGGER_ROLLOVER_INTERVAL | M = every minute, H = hourly, D = daily, W = weekly | D |
+| LOGGER_MAX_BACKUP | Maximum number of backup files before it will deletes old file. Non positive value will not have maximum | -1 |
+| LOGGER_ROLLOVER_MAX_FILE_SIZE | Maximum file size in bytes before it will rollover to new file | 10000000 |
+| LOGGER_USE_UTC | If logger will use UTC | false |
 
 # **SSO Supported Enviroment Variable**
 ## Supported Platform

jac-cloud/docs/Logger.md

@@ -0,0 +1,57 @@
+# Logging
+* jac-cloud server logs incoming requests and outgoing responses by default to log files stored in `/tmp/` directory
+* The log files are on a daily rotation. Meaning there will be a new log file created every day to prevent log files gets too large.
+* For production usage, we recommend connect your jac-cloud logs to an Elastic instance.
+# Quick Start: Integration with Elasitc
+* Assuming you have a running Elastic instance, you just need to use filebeat to ingest the log files into elastic.
+* We provide a template filebeat config file to get started at `scripts/filebeat-template.yaml`. If you want to adopt the default configuration, simply change the `hosts` and `api_key` field.
+  * Change the hosts field to point to your elastic instance. 
+
+> :warning: It seems that filebeat automatically append a 9200 port to the host URL if no port is specified. If your elastic instance is behind a load balancer and simply has a URL without a custom port, you will need to add either :80 or :443 to the hosts config. For example, `hosts: ["https://my_elastic_instance.me.com:443/]`
+
+* Then simply run `filebeat -e -c scripts/filebeat-template.yaml`.
+
+## More Details on Configuring and Starting Filebeat
+- [Download](https://www.elastic.co/downloads/beats/filebeat) and install filebeat.
+- setup yml based on your setup
+```yml
+filebeat.inputs:
+- type: log
+  enabled: true
+  paths:
+    - /tmp/jac_cloud_logs/*-jac-cloud-*.log
+    - /tmp/jac_cloud_logs/jac-cloud.log
+  json:
+    keys_under_root: true
+    overwrite_keys: true
+    add_error_key: true
+    expand_keys: true
+
+output.elasticsearch:
+  hosts: ["localhost:9200"]
+  protocol: https
+  api_key: "id:api_key"
+  index: "filebeat-testing"
+
+setup.template.name: "filebeat"
+setup.template.pattern: "filebeat-*"
+```
+- to run without restriction, run filebeat as root
+    - `sudo cp filebeat.yml /etc/filebeat/filebeat.yml`
+    - `sudo filebeat -e`
+- normal run
+    - `filebeat -e -c filebeat.yml`
+- for complete documentation
+    - https://www.elastic.co/guide/en/cloud/current/ec-getting-started-search-use-cases-python-logs.html
+    - https://www.elastic.co/guide/en/beats/filebeat/current/configuring-howto-filebeat.html
+
+## Additional Env Vars to customize logger
+| **NAME**  | **DESCRIPTION**   | **DEFAULT**   |
+|-----------|-------------------|---------------|
+| LOGGER_NAME   | Specified logger name | app   |
+| LOGGER_LEVEL  | Control log level     | debug |
+| LOGGER_FILE_PATH | Log directory and name | /tmp/jac_cloud_logs/jac-cloud.log |
+| LOGGER_ROLLOVER_INTERVAL | M = every minute, H = hourly, D = daily, W = weekly | D |
+| LOGGER_MAX_BACKUP | Maximum number of backup files before it will deletes old file. Non positive value will not have maximum | -1 |
+| LOGGER_ROLLOVER_MAX_FILE_SIZE | Maximum file size in bytes before it will rollover to new file | 10000000 |
+| LOGGER_USE_UTC | If logger will use UTC | false |

jac-cloud/jac_cloud/core/context.py

@@ -8,7 +8,6 @@
 from bson import ObjectId
 
 from fastapi import Request
-from fastapi.responses import ORJSONResponse
 
 from jaclang.runtimelib.context import ExecutionContext
 
@@ -140,7 +139,7 @@ def get_root() -> Root:  # type: ignore[override]
         """Get current root."""
         return cast(Root, JaseciContext.get().root.architype)
 
-    def response(self, returns: list[Any]) -> ORJSONResponse:
+    def response(self, returns: list[Any]) -> dict[str, Any]:
         """Return serialized version of reports."""
         resp = ContextResponse[Any](status=self.status)
 
@@ -155,7 +154,7 @@ def response(self, returns: list[Any]) -> ORJSONResponse:
         if SHOW_ENDPOINT_RETURNS:
             resp.returns = returns
 
-        return ORJSONResponse(resp.__serialize__(), status_code=self.status)
+        return resp.__serialize__()
 
     def clean_response(
         self, key: str | int, val: Any, obj: list | dict  # noqa: ANN401

jac-cloud/jac_cloud/jaseci/__init__.py

@@ -2,13 +2,15 @@
 
 from contextlib import asynccontextmanager
 from os import getenv
+from traceback import format_exception
 from typing import Any, AsyncGenerator
 
-from fastapi import FastAPI as _FaststAPI
+from fastapi import FastAPI as _FaststAPI, Request
+from fastapi.responses import ORJSONResponse
 
 from uvicorn import run as _run
 
-from .utils import Emailer
+from .utils import Emailer, logger
 
 
 class FastAPI:
@@ -63,6 +65,22 @@ async def lifespan(app: _FaststAPI) -> AsyncGenerator[None, _FaststAPI]:
             for router in [healthz_router, sso_router, user_router, walker_router]:
                 cls.__app__.include_router(router)
 
+            @cls.__app__.exception_handler(Exception)
+            async def uncatched_exception_handler(
+                request: Request, exc: Exception
+            ) -> ORJSONResponse:
+                """Catched uncatched exceptions."""
+                response = {"errors": format_exception(exc)}
+
+                log: dict[str, Any] = {"request_url": str(request.url)}
+                log["extra_fields"] = list(log.keys())
+                logger.error(
+                    f"Call from to {log["request_url"]} returns unexpected errors: {response["errors"]}",
+                    extra=log,
+                )
+
+                return ORJSONResponse(response, status_code=500)
+
         return cls.__app__
 
     @classmethod
@@ -71,7 +89,7 @@ def start(
         host: str | None = None,
         port: int | None = None,
         emailer: type[Emailer] | None = None,
-        **kwargs: Any  # noqa ANN401
+        **kwargs: Any,  # noqa ANN401
     ) -> None:
         """Run FastAPI Handler via Uvicorn."""
         if emailer:
@@ -81,5 +99,5 @@ def start(
             cls.get(),
             host=host or getenv("HOST") or "0.0.0.0",
             port=port or int(getenv("PORT", "8000")),
-            **kwargs
+            **kwargs,
         )

jac-cloud/jac_cloud/jaseci/models/user.py

@@ -29,6 +29,12 @@ def obfuscate(self) -> dict:
             data["password"] = pbkdf2_sha512.hash(self.password).encode()
         return data
 
+    def printable(self) -> dict:
+        """Return BaseModel.model_dump where the password is hashed."""
+        data = self.model_dump(exclude={"password"})
+        data["password"] = "****"
+        return data
+
 
 @dataclass(kw_only=True)
 class User:

jac-cloud/jac_cloud/jaseci/routers/user.py

@@ -25,7 +25,7 @@
     invalidate_token,
     verify_code,
 )
-from ..utils import Emailer, logger
+from ..utils import Emailer, log_entry, log_exit, logger
 from ...core.architype import BulkWrite, NodeAnchor, Root
 
 
@@ -38,6 +38,8 @@
 @router.post("/register", status_code=status.HTTP_200_OK)
 def register(req: User.register_type()) -> ORJSONResponse:  # type: ignore
     """Register user API."""
+    log = log_entry("register", req.email, req.printable())
+
     with User.Collection.get_session() as session, session.start_transaction():
         root = Root().__jac__
 
@@ -56,7 +58,9 @@ def register(req: User.register_type()) -> ORJSONResponse:  # type: ignore
                     BulkWrite.commit(session)
                     if not is_activated:
                         User.send_verification_code(create_code(id), req.email)
-                    return ORJSONResponse({"message": "Successfully Registered!"}, 201)
+                    resp = {"message": "Successfully Registered!"}
+                    log_exit(resp, log)
+                    return ORJSONResponse(resp, 201)
             except (ConnectionFailure, OperationFailure) as ex:
                 if ex.has_error_label("TransientTransactionError"):
                     retry += 1
@@ -72,7 +76,10 @@ def register(req: User.register_type()) -> ORJSONResponse:  # type: ignore
                 logger.exception("Error executing bulk write!")
                 session.abort_transaction()
                 break
-    return ORJSONResponse({"message": "Registration Failed!"}, 409)
+
+    resp = {"message": "Registration Failed!"}
+    log_exit(resp, log)
+    return ORJSONResponse(resp, 409)
 
 
 @router.post(

jac-cloud/jac_cloud/jaseci/utils/__init__.py

@@ -1,10 +1,10 @@
 """Jaseci Utilities."""
 
-import logging
 from datetime import datetime, timedelta, timezone
 from random import choice
 from string import ascii_letters, digits
 
+from .logger import log_dumps, log_entry, log_exit, logger
 from .mail import Emailer, SendGridEmailer
 
 
@@ -23,14 +23,14 @@ def utc_timestamp(**addons: int) -> int:
     return int(utc_datetime(**addons).timestamp())
 
 
-logger = logging.getLogger(__name__)
-# logger.addHandler(logging.StreamHandler(sys.stdout))
-
 __all__ = [
     "Emailer",
     "SendGridEmailer",
     "random_string",
     "utc_datetime",
     "utc_timestamp",
+    "log_dumps",
+    "log_entry",
+    "log_exit",
     "logger",
 ]