From bdbe551b56cac7a1e79ef65590583fa21ed291a6 Mon Sep 17 00:00:00 2001 From: Markus Kuhn Date: Sat, 28 Sep 2024 11:55:04 +0200 Subject: [PATCH] made logging externally customizable --- ToDo | 2 + aquaPi/__init__.py | 104 +++++++++++++++++++++++++----- aquaPi/api.py | 6 +- aquaPi/driver/DriverADC.py | 7 +- aquaPi/driver/DriverGPIO.py | 6 +- aquaPi/driver/DriverOneWire.py | 6 +- aquaPi/driver/DriverPWM.py | 7 +- aquaPi/driver/DriverTC420.py | 7 +- aquaPi/driver/__init__.py | 8 +-- aquaPi/driver/base.py | 6 +- aquaPi/machineroom/__init__.py | 6 +- aquaPi/machineroom/alert_nodes.py | 6 +- aquaPi/machineroom/aux_nodes.py | 6 +- aquaPi/machineroom/ctrl_nodes.py | 6 +- aquaPi/machineroom/hist_nodes.py | 10 +-- aquaPi/machineroom/in_nodes.py | 6 +- aquaPi/machineroom/msg_bus.py | 6 +- aquaPi/machineroom/msg_types.py | 3 +- aquaPi/machineroom/out_nodes.py | 6 +- aquaPi/pages/about.py | 7 +- aquaPi/pages/config.py | 6 +- aquaPi/pages/home.py | 7 +- aquaPi/pages/settings.py | 6 +- aquaPi/pages/spa.py | 6 +- aquaPi/pages/sse_util.py | 5 ++ 25 files changed, 126 insertions(+), 125 deletions(-) diff --git a/ToDo b/ToDo index 692dbbe..d82762c 100644 --- a/ToDo +++ b/ToDo @@ -2,8 +2,10 @@ aquaPi ToDo list ================ - Known defects: + # logging: resolve the abuse of logging.WARNING as loglevel.BRIEF; log.verbose() could be a functools.partialmethod(log.log, loglevel.INFO-1,...) # REAL_CONFIG: GPIO13 must be permanently on (Filter!) - might need a new ConstInput(100) # startup behaviour of bus isn't good - let everybody post its data as response to HELLO? + # on Raspi driver TC420 finds a device although there's none - Navigation: / Home, a configurable dashboard diff --git a/aquaPi/__init__.py b/aquaPi/__init__.py index b247233..57018af 100644 --- a/aquaPi/__init__.py +++ b/aquaPi/__init__.py @@ -4,11 +4,15 @@ from os import path import sys from flask import Flask -import logging + +import json +import logging.config +import logging.handlers # from logging.handlers import SMTPHandler -log = logging.getLogger('werkzeug') +log = logging.getLogger('aquaPi') + log.brief = log.warning # alias, warning used as brief info, info is verbose logging.addLevelName(logging.WARN, 'LOG') # this makes log.warn kind of useless @@ -21,27 +25,97 @@ # logging.addLevelName(logging.VERBOSE, 'LOG') # log.verbose = log.log( ... need to implant a methods into logging for this -log.setLevel(logging.WARNING) -# log.setLevel(logging.INFO) -# log.setLevel(logging.DEBUG) +# this is a json string to make it a template for log_config.json +log_default = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "simple": { + "format": "%(asctime)s %(levelname).3s %(name)s: %(message)s", + "datefmt": "%H:%M:%S" + } + }, + "handlers": { + "stdout": { + "class": "logging.StreamHandler", + "level": "INFO", + "formatter": "simple", + "stream": "ext://sys.stdout" + }, + "file": { + "class": "logging.handlers.RotatingFileHandler", + "level": "WARNING", + "formatter": "simple", + "filename": "logs/aquaPi.log", + "maxBytes": 1000000, + "backupCount": 3 + } + }, + "loggers": { + "root": { + "level": "WARNING", + "handlers": ["stdout","file"] + }, + + "aquaPi": {"level": "NOTSET"}, + #"aquaPi.api": {"level": "NOTSET"}, + + "machineroom": {"level": "NOTSET"}, + #"machineroom.alert_nodes": {"level": "NOTSET"}, + #"machineroom.aux_nodes": {"level": "NOTSET"}, + #"machineroom.ctrl_nodes": {"level": "NOTSET"}, + #"machineroom.hist_nodes": {"level": "NOTSET"}, + #"machineroom.in_nodes": {"level": "NOTSET"}, + #"machineroom.msg_bus": {"level": "NOTSET"}, + #"machineroom.msg_types": {"level": "NOTSET"}, + #"machineroom.out_nodes": {"level": "NOTSET"}, + + "driver": {"level": "NOTSET"}, + #"driver.base": {"level": "NOTSET"}, + #"driver.DriverADC": {"level": "NOTSET"}, + #"driver.DriverAlert": {"level": "NOTSET"}, + #"driver.DriverGPIO": {"level": "NOTSET"}, + #"driver.DriverOneWire": {"level": "NOTSET"}, + #"driver.DriverPWM": {"level": "NOTSET"}, + #"driver.DriverTC420": {"level": "NOTSET"}, + + "pages": {"level": "NOTSET"}, + #"pages.about": {"level": "NOTSET"}, + #"pages.config": {"level": "NOTSET"}, + #"pages.home": {"level": "NOTSET"}, + #"pages.settings": {"level": "NOTSET"}, + #"pages.spa": {"level": "NOTSET"}, + #"pages.sse_util": {"level": "NOTSET"}, + + "werkzeug": { + "comment": "werkzeug is noisy, reduce to >=WARNING, INFO shows all https requests", + "level": "WARNING", + "propagate": False + } + } +} + -#TODO remove sqlite completely -#TODO remove test_config & config.py, use ArgumentParser instead, see https://stackoverflow.com/questions/48346025/how-to-pass-an-arbitrary-argument-to-flask-through-app-run -# args: -c "config"[.pickle] def create_app(): - logging.basicConfig(format='%(asctime)s %(levelname).3s %(name)s: %(message)s' - , datefmt='%H:%M:%S', stream=sys.stdout, level=logging.WARNING) + # TODO wrap in try/catch, but how should exceptions be handled? + app = Flask(__name__, instance_relative_config=True) -# TODO wrap in try/catch, but how should exceptions be handled? + config_file = path.join(app.instance_path, "log_config.json") + if path.exists(config_file): + with open(config_file) as f_in: + log_config = json.load(f_in) + else: + log_config = log_default + logging.config.dictConfig(log_config) - app = Flask(__name__, instance_relative_config=True) + logging.warning("Press CTRL+C to quit") + log.brief("... und los geht's") - # no luck with comand line parsing: + # no luck with command line parsing: # 1. Flask uses "click", which conflicts with the simple argparse - # 2. click is complex, I simply didn't succeed to add options to Flask's commnd groups + # 2. click is complex, I simply didn't succeed to add options to Flask's command groups # for now use env. vars instead - try: cfg_file = os.environ['AQUAPI_CFG'] except KeyError: diff --git a/aquaPi/api.py b/aquaPi/api.py index c0f00d7..3ab2036 100644 --- a/aquaPi/api.py +++ b/aquaPi/api.py @@ -16,13 +16,9 @@ from .pages.sse_util import send_sse_events -log = logging.getLogger('API') +log = logging.getLogger('aquaPi.api') log.brief = log.warning # alias, warning used as brief info, info is verbose -log.setLevel(logging.WARNING) -# log.setLevel(logging.INFO) -# log.setLevel(logging.DEBUG) - bp = Blueprint('api', __name__) diff --git a/aquaPi/driver/DriverADC.py b/aquaPi/driver/DriverADC.py index 85916c3..ed68c49 100644 --- a/aquaPi/driver/DriverADC.py +++ b/aquaPi/driver/DriverADC.py @@ -24,12 +24,9 @@ class ADS(Enum): from .base import (AInDriver, IoPort, PortFunc) -log = logging.getLogger('DriverADS111x') -log.brief = log.warning # alias, warning used as brief info, info is verbose -log.setLevel(logging.WARNING) -# log.setLevel(logging.INFO) -# log.setLevel(logging.DEBUG) +log = logging.getLogger('driver.DriverADC') +log.brief = log.warning # alias, warning used as brief info, info is verbose # ========== ADC inputs ========== diff --git a/aquaPi/driver/DriverGPIO.py b/aquaPi/driver/DriverGPIO.py index 65eb692..6aab3a0 100644 --- a/aquaPi/driver/DriverGPIO.py +++ b/aquaPi/driver/DriverGPIO.py @@ -47,13 +47,9 @@ def output(pin, value): from .base import (InDriver, OutDriver, IoPort, PortFunc, is_raspi, DriverWriteError) -log = logging.getLogger('DriverGPIO') +log = logging.getLogger('driver.DriverGPIO') log.brief = log.warning # alias, warning is used as brief info, level info is verbose -log.setLevel(logging.WARNING) -# log.setLevel(logging.INFO) -# log.setLevel(logging.DEBUG) - if is_raspi(): GPIO.setmode(GPIO.BCM) diff --git a/aquaPi/driver/DriverOneWire.py b/aquaPi/driver/DriverOneWire.py index 60d5207..74c45b8 100644 --- a/aquaPi/driver/DriverOneWire.py +++ b/aquaPi/driver/DriverOneWire.py @@ -7,13 +7,9 @@ from .base import (AInDriver, IoPort, PortFunc, is_raspi, DriverInvalidAddrError, DriverReadError) -log = logging.getLogger('DriverOneWire') +log = logging.getLogger('driver.DriverOneWire') log.brief = log.warning # alias, warning is used as brief info, level info is verbose -log.setLevel(logging.WARNING) -# log.setLevel(logging.INFO) -# log.setLevel(logging.DEBUG) - # ========== 1-wire ========== diff --git a/aquaPi/driver/DriverPWM.py b/aquaPi/driver/DriverPWM.py index 8c2f516..7cd4070 100644 --- a/aquaPi/driver/DriverPWM.py +++ b/aquaPi/driver/DriverPWM.py @@ -18,12 +18,9 @@ def gpio_function(pin): from .base import (OutDriver, IoPort, PortFunc, PinFunc, is_raspi) -log = logging.getLogger('DriverPWM') -log.brief = log.warning # alias, warning is used as brief info, level info is verbose -log.setLevel(logging.WARNING) -# log.setLevel(logging.INFO) -# log.setLevel(logging.DEBUG) +log = logging.getLogger('driver.DriverPWM') +log.brief = log.warning # alias, warning is used as brief info, level info is verbose # ========== PWM ========== diff --git a/aquaPi/driver/DriverTC420.py b/aquaPi/driver/DriverTC420.py index 7347818..575bf17 100644 --- a/aquaPi/driver/DriverTC420.py +++ b/aquaPi/driver/DriverTC420.py @@ -12,12 +12,9 @@ from .base import (OutDriver, IoPort, PortFunc) -log = logging.getLogger('DriverTC420') -log.brief = log.warning # alias, warning is used as brief info, level info is verbose -log.setLevel(logging.WARNING) -# log.setLevel(logging.INFO) -# log.setLevel(logging.DEBUG) +log = logging.getLogger('driver.DriverTC420') +log.brief = log.warning # alias, warning is used as brief info, level info is verbose # ========== PWM ========== diff --git a/aquaPi/driver/__init__.py b/aquaPi/driver/__init__.py index d9238b1..28004d7 100644 --- a/aquaPi/driver/__init__.py +++ b/aquaPi/driver/__init__.py @@ -9,13 +9,9 @@ from .base import Driver, DriverParamError, DriverPortInuseError from .base import * -log = logging.getLogger('Driver Base') +log = logging.getLogger('driver') log.brief = log.warning # alias, warning is used as brief info, level info is verbose -log.setLevel(logging.WARNING) -# log.setLevel(logging.INFO) -# log.setLevel(logging.DEBUG) - # ========== IO registry ========== @@ -158,7 +154,7 @@ def driver_destruct(self, port, driver): DRIVER_FILE_PREFIX = 'Driver' CUSTOM_DRIVERS = 'CustomDrivers' -# import all files named Driver*,py into our package, icluding a subfolder CustomDrivers +# import all files named Driver*.py into our package, including a subfolder CustomDrivers __path__.append(path.join(__path__[0], CUSTOM_DRIVERS)) for drv_path in __path__: diff --git a/aquaPi/driver/base.py b/aquaPi/driver/base.py index ed188e6..b5912a4 100644 --- a/aquaPi/driver/base.py +++ b/aquaPi/driver/base.py @@ -7,13 +7,9 @@ import math import random -log = logging.getLogger('Driver Base') +log = logging.getLogger('driver.base') log.brief = log.warning # alias, warning used as brief info, info is verbose -log.setLevel(logging.WARNING) -# log.setLevel(logging.INFO) -# log.setLevel(logging.DEBUG) - # ========== common functions ========== diff --git a/aquaPi/machineroom/__init__.py b/aquaPi/machineroom/__init__.py index 8799ece..3e040a9 100644 --- a/aquaPi/machineroom/__init__.py +++ b/aquaPi/machineroom/__init__.py @@ -14,13 +14,9 @@ from .alert_nodes import Alert, AlertAbove, AlertBelow -log = logging.getLogger('MachineRoom') +log = logging.getLogger('machineroom') log.brief = log.warning # alias, warning used as brief info, info is verbose -log.setLevel(logging.WARNING) -# log.setLevel(logging.INFO) -# log.setLevel(logging.DEBUG) - mr = None diff --git a/aquaPi/machineroom/alert_nodes.py b/aquaPi/machineroom/alert_nodes.py index 00eb356..b78a048 100644 --- a/aquaPi/machineroom/alert_nodes.py +++ b/aquaPi/machineroom/alert_nodes.py @@ -8,13 +8,9 @@ from ..driver import (PortFunc, io_registry, DriverReadError) -log = logging.getLogger('AlertNodes') +log = logging.getLogger('machineroom.alert_nodes') log.brief = log.warning # alias, warning is used as brief info, level info is verbose -log.setLevel(logging.WARNING) -log.setLevel(logging.INFO) -# log.setLevel(logging.DEBUG) - # ========== alert conditions ========== diff --git a/aquaPi/machineroom/aux_nodes.py b/aquaPi/machineroom/aux_nodes.py index 364b20c..35d7feb 100644 --- a/aquaPi/machineroom/aux_nodes.py +++ b/aquaPi/machineroom/aux_nodes.py @@ -5,13 +5,9 @@ from .msg_bus import (BusListener, BusRole, DataRange, MsgData) -log = logging.getLogger('AuxNodes') +log = logging.getLogger('machineroom.aux_nodes') log.brief = log.warning # alias, warning used as brief info, info is verbose -log.setLevel(logging.WARNING) -# log.setLevel(logging.INFO) -# log.setLevel(logging.DEBUG) - # ========== auxiliary ========== diff --git a/aquaPi/machineroom/ctrl_nodes.py b/aquaPi/machineroom/ctrl_nodes.py index cfbca66..43ed1da 100644 --- a/aquaPi/machineroom/ctrl_nodes.py +++ b/aquaPi/machineroom/ctrl_nodes.py @@ -10,13 +10,9 @@ from .msg_bus import (BusListener, BusRole, DataRange, MsgData) -log = logging.getLogger('CtrlNodes') +log = logging.getLogger('machineroom.ctrl_nodes') log.brief = log.warning # alias, warning used as brief info, info is verbose -log.setLevel(logging.WARNING) -# log.setLevel(logging.INFO) -# log.setLevel(logging.DEBUG) - def get_unit_limits(unit): if ('°C') in unit: diff --git a/aquaPi/machineroom/hist_nodes.py b/aquaPi/machineroom/hist_nodes.py index 7236232..c352012 100644 --- a/aquaPi/machineroom/hist_nodes.py +++ b/aquaPi/machineroom/hist_nodes.py @@ -21,13 +21,9 @@ # from ..driver import (PortFunc, io_registry, DriverReadError) -log = logging.getLogger('HistoryNodes') +log = logging.getLogger('machineroom.hist_nodes') log.brief = log.warning # alias, warning is used as brief info, level info is verbose -log.setLevel(logging.WARNING) -# log.setLevel(logging.INFO) -# log.setLevel(logging.DEBUG) - # ========== interface to time series DB ========== @@ -350,12 +346,12 @@ def __init__(self, name, inputs, duration=24, _cont=False): if QUEST_DB: try: self.db = TimeDbQuest() - log.brief('Recording history in QuestDB') + log.brief('Recording history %s in QuestDB', name) except (NotImplementedError, ModuleNotFoundError, ImportError): pass if not self.db: self.db = TimeDbMemory(duration) - log.brief('Recording history in main memory with limited depth of %dh!', duration) + log.brief('Recording history %s in main memory with limited depth of %dh!', name, duration) for snd in self._inputs.sender: self.db.add_field(snd) diff --git a/aquaPi/machineroom/in_nodes.py b/aquaPi/machineroom/in_nodes.py index 5976ee6..1f17f10 100644 --- a/aquaPi/machineroom/in_nodes.py +++ b/aquaPi/machineroom/in_nodes.py @@ -10,13 +10,9 @@ from ..driver import (io_registry, DriverReadError) -log = logging.getLogger('InNodes') +log = logging.getLogger('machineroom.in_nodes') log.brief = log.warning # alias, warning is used as brief info, level info is verbose -log.setLevel(logging.WARNING) -# log.setLevel(logging.INFO) -# log.setLevel(logging.DEBUG) - # ========== inputs AKA sensors ========== diff --git a/aquaPi/machineroom/msg_bus.py b/aquaPi/machineroom/msg_bus.py index b34502c..0bc6cb2 100644 --- a/aquaPi/machineroom/msg_bus.py +++ b/aquaPi/machineroom/msg_bus.py @@ -9,13 +9,9 @@ from .msg_types import (MsgInfra, MsgBorn, MsgBye, MsgReply, MsgReplyHello, MsgData, MsgFilter) -log = logging.getLogger('MsgBus') +log = logging.getLogger('machineroom.msg_bus') log.brief = log.warning # alias, warning is used as brief info, level info is verbose -log.setLevel(logging.WARNING) -# log.setLevel(logging.INFO) -# log.setLevel(logging.DEBUG) - class BusRole(Flag): IN_ENDP = auto() # data sources: sensor, switch, schedule diff --git a/aquaPi/machineroom/msg_types.py b/aquaPi/machineroom/msg_types.py index 872b85b..741af8e 100644 --- a/aquaPi/machineroom/msg_types.py +++ b/aquaPi/machineroom/msg_types.py @@ -3,8 +3,7 @@ import logging -log = logging.getLogger('MsgBusMsgs') -log.setLevel(logging.WARNING) # INFO) +log = logging.getLogger('machineroom.msg_types') class Msg: diff --git a/aquaPi/machineroom/out_nodes.py b/aquaPi/machineroom/out_nodes.py index 3ce3a0e..cfc7312 100644 --- a/aquaPi/machineroom/out_nodes.py +++ b/aquaPi/machineroom/out_nodes.py @@ -6,13 +6,9 @@ from ..driver import (io_registry) -log = logging.getLogger('OutNodes') +log = logging.getLogger('machineroom.out_nodes') log.brief = log.warning # alias, warning is used as brief info, level info is verbose -log.setLevel(logging.WARNING) -# log.setLevel(logging.INFO) -# log.setLevel(logging.DEBUG) - # ========== outputs AKA Device ========== diff --git a/aquaPi/pages/about.py b/aquaPi/pages/about.py index efdd31f..e89020c 100644 --- a/aquaPi/pages/about.py +++ b/aquaPi/pages/about.py @@ -6,12 +6,9 @@ from .sse_util import render_sse_template -log = logging.getLogger('/about') -log.brief = log.warning # alias, warning is used as brief info, level info is verbose -log.setLevel(logging.WARNING) -# log.setLevel(logging.INFO) -# log.setLevel(logging.DEBUG) +log = logging.getLogger('pages.about') +log.brief = log.warning # alias, warning is used as brief info, level info is verbose bp = Blueprint('about', __name__) diff --git a/aquaPi/pages/config.py b/aquaPi/pages/config.py index 02241f9..5381949 100644 --- a/aquaPi/pages/config.py +++ b/aquaPi/pages/config.py @@ -7,13 +7,9 @@ # from ..machineroom import msg_bus -log = logging.getLogger('/config') +log = logging.getLogger('pages.config') log.brief = log.warning # alias, warning is used as brief info, level info is verbose -log.setLevel(logging.WARNING) -# log.setLevel(logging.INFO) -# log.setLevel(logging.DEBUG) - bp = Blueprint('config', __name__) diff --git a/aquaPi/pages/home.py b/aquaPi/pages/home.py index cd86c2e..268bdb8 100644 --- a/aquaPi/pages/home.py +++ b/aquaPi/pages/home.py @@ -6,12 +6,9 @@ from ..machineroom import BusRole from .sse_util import render_sse_template -log = logging.getLogger('/home') -log.brief = log.warning # alias, warning used as brief info, info is verbose -log.setLevel(logging.WARNING) -# log.setLevel(logging.INFO) -# log.setLevel(logging.DEBUG) +log = logging.getLogger('pages.home') +log.brief = log.warning # alias, warning used as brief info, info is verbose bp = Blueprint('home', __name__) diff --git a/aquaPi/pages/settings.py b/aquaPi/pages/settings.py index a1bd0c7..994e111 100644 --- a/aquaPi/pages/settings.py +++ b/aquaPi/pages/settings.py @@ -6,13 +6,9 @@ # from ..machineroom import msg_bus -log = logging.getLogger('/settings') +log = logging.getLogger('pages.settings') log.brief = log.warning # alias, warning is used as brief info, level info is verbose -log.setLevel(logging.WARNING) -# log.setLevel(logging.INFO) -# log.setLevel(logging.DEBUG) - bp = Blueprint('settings', __name__) diff --git a/aquaPi/pages/spa.py b/aquaPi/pages/spa.py index 24b6498..ad528f0 100644 --- a/aquaPi/pages/spa.py +++ b/aquaPi/pages/spa.py @@ -4,13 +4,9 @@ from flask import (Blueprint, render_template) -log = logging.getLogger('/spa') +log = logging.getLogger('pages.spa') log.brief = log.warning # alias, warning is used as brief info, level info is verbose -log.setLevel(logging.WARNING) -# log.setLevel(logging.INFO) -# log.setLevel(logging.DEBUG) - bp = Blueprint('spa', __name__) diff --git a/aquaPi/pages/sse_util.py b/aquaPi/pages/sse_util.py index f9db5bf..7d3c7e0 100644 --- a/aquaPi/pages/sse_util.py +++ b/aquaPi/pages/sse_util.py @@ -1,9 +1,14 @@ #!/usr/bin/env python3 +import logging import time from flask import Response, request, render_template +log = logging.getLogger('pages.sse_util') +log.brief = log.warning # alias, warning is used as brief info, level info is verbose + + def format_msg(data: str, event=None) -> str: """ Formats a string and an event name in order to follow the event stream convention. for event!=None you'll need a custom event listener