made logging externally customizable

ToDo

@@ -2,8 +2,10 @@ aquaPi ToDo list
 ================
 
 - Known defects:
+  # logging: resolve the abuse of logging.WARNING as loglevel.BRIEF; log.verbose() could be a functools.partialmethod(log.log, loglevel.INFO-1,...)
   # REAL_CONFIG: GPIO13 must be permanently on (Filter!) - might need a new ConstInput(100)
   # startup behaviour of bus isn't good - let everybody post its data as response to HELLO?
+  # on Raspi driver TC420 finds a device although there's none
 
 - Navigation:
   /          Home, a configurable dashboard

aquaPi/__init__.py

@@ -4,11 +4,15 @@
 from os import path
 import sys
 from flask import Flask
-import logging
+
+import json
+import logging.config
+import logging.handlers
 # from logging.handlers import SMTPHandler
 
 
-log = logging.getLogger('werkzeug')
+log = logging.getLogger('aquaPi')
+
 
 log.brief = log.warning  # alias, warning used as brief info, info is verbose
 logging.addLevelName(logging.WARN, 'LOG')  # this makes log.warn kind of useless
@@ -21,27 +25,97 @@
 #  logging.addLevelName(logging.VERBOSE, 'LOG')
 #  log.verbose = log.log( ...  need to implant a methods into logging for this
 
-log.setLevel(logging.WARNING)
-# log.setLevel(logging.INFO)
-# log.setLevel(logging.DEBUG)
+# this is a json string to make it a template for log_config.json
+log_default = {
+  "version": 1,
+  "disable_existing_loggers": False,
+  "formatters": {
+    "simple": {
+      "format": "%(asctime)s %(levelname).3s %(name)s: %(message)s",
+      "datefmt": "%H:%M:%S"
+    }
+  },
+  "handlers": {
+    "stdout": {
+      "class": "logging.StreamHandler",
+      "level": "INFO",
+      "formatter": "simple",
+      "stream": "ext://sys.stdout"
+    },
+    "file": {
+      "class": "logging.handlers.RotatingFileHandler",
+      "level": "WARNING",
+      "formatter": "simple",
+      "filename": "logs/aquaPi.log",
+      "maxBytes": 1000000,
+      "backupCount": 3
+    }
+  },
+  "loggers": {
+    "root": {
+      "level": "WARNING",
+      "handlers": ["stdout","file"]
+    },
+
+    "aquaPi":     {"level": "NOTSET"},
+    #"aquaPi.api": {"level": "NOTSET"},
+
+    "machineroom":             {"level": "NOTSET"},
+    #"machineroom.alert_nodes": {"level": "NOTSET"},
+    #"machineroom.aux_nodes":   {"level": "NOTSET"},
+    #"machineroom.ctrl_nodes":  {"level": "NOTSET"},
+    #"machineroom.hist_nodes":  {"level": "NOTSET"},
+    #"machineroom.in_nodes":    {"level": "NOTSET"},
+    #"machineroom.msg_bus":     {"level": "NOTSET"},
+    #"machineroom.msg_types":   {"level": "NOTSET"},
+    #"machineroom.out_nodes":   {"level": "NOTSET"},
+
+    "driver":               {"level": "NOTSET"},
+    #"driver.base":          {"level": "NOTSET"},
+    #"driver.DriverADC":     {"level": "NOTSET"},
+    #"driver.DriverAlert":   {"level": "NOTSET"},
+    #"driver.DriverGPIO":    {"level": "NOTSET"},
+    #"driver.DriverOneWire": {"level": "NOTSET"},
+    #"driver.DriverPWM":     {"level": "NOTSET"},
+    #"driver.DriverTC420":   {"level": "NOTSET"},
+
+    "pages":          {"level": "NOTSET"},
+    #"pages.about":    {"level": "NOTSET"},
+    #"pages.config":   {"level": "NOTSET"},
+    #"pages.home":     {"level": "NOTSET"},
+    #"pages.settings": {"level": "NOTSET"},
+    #"pages.spa":      {"level": "NOTSET"},
+    #"pages.sse_util": {"level": "NOTSET"},
+
+    "werkzeug": {
+      "comment": "werkzeug is noisy, reduce to >=WARNING, INFO shows all https requests",
+      "level": "WARNING",
+      "propagate": False
+    }
+  }
+}
+
 
-#TODO remove sqlite completely
-#TODO remove test_config & config.py, use ArgumentParser instead, see https://stackoverflow.com/questions/48346025/how-to-pass-an-arbitrary-argument-to-flask-through-app-run
-# args:  -c "config"[.pickle]
 
 def create_app():
-    logging.basicConfig(format='%(asctime)s %(levelname).3s %(name)s: %(message)s'
-                       , datefmt='%H:%M:%S', stream=sys.stdout, level=logging.WARNING)
+    # TODO wrap in try/catch, but how should exceptions be handled?
+    app = Flask(__name__, instance_relative_config=True)
 
-# TODO wrap in try/catch, but how should exceptions be handled?
+    config_file = path.join(app.instance_path, "log_config.json")
+    if path.exists(config_file):
+        with open(config_file) as f_in:
+            log_config = json.load(f_in)
+    else:
+        log_config = log_default
+    logging.config.dictConfig(log_config)
 
-    app = Flask(__name__, instance_relative_config=True)
+    logging.warning("Press CTRL+C to quit")
+    log.brief("... und los geht's")
 
-    # no luck with comand line parsing:
+    # no luck with command line parsing:
     # 1. Flask uses "click", which conflicts with the simple argparse
-    # 2. click is complex, I simply didn't succeed to add options to Flask's commnd groups
+    # 2. click is complex, I simply didn't succeed to add options to Flask's command groups
     # for now use env. vars instead
-
     try:
         cfg_file = os.environ['AQUAPI_CFG']
     except KeyError:

aquaPi/api.py

@@ -16,13 +16,9 @@
 from .pages.sse_util import send_sse_events
 
 
-log = logging.getLogger('API')
+log = logging.getLogger('aquaPi.api')
 log.brief = log.warning  # alias, warning used as brief info, info is verbose
 
-log.setLevel(logging.WARNING)
-# log.setLevel(logging.INFO)
-# log.setLevel(logging.DEBUG)
-
 
 bp = Blueprint('api', __name__)
 

aquaPi/driver/DriverADC.py

@@ -24,12 +24,9 @@ class ADS(Enum):
 
 from .base import (AInDriver, IoPort, PortFunc)
 
-log = logging.getLogger('DriverADS111x')
-log.brief = log.warning  # alias, warning used as brief info, info is verbose
 
-log.setLevel(logging.WARNING)
-# log.setLevel(logging.INFO)
-# log.setLevel(logging.DEBUG)
+log = logging.getLogger('driver.DriverADC')
+log.brief = log.warning  # alias, warning used as brief info, info is verbose
 
 
 # ========== ADC inputs ==========

aquaPi/driver/DriverGPIO.py

@@ -47,13 +47,9 @@ def output(pin, value):
 
 from .base import (InDriver, OutDriver, IoPort, PortFunc, is_raspi, DriverWriteError)
 
-log = logging.getLogger('DriverGPIO')
+log = logging.getLogger('driver.DriverGPIO')
 log.brief = log.warning  # alias, warning is used as brief info, level info is verbose
 
-log.setLevel(logging.WARNING)
-# log.setLevel(logging.INFO)
-# log.setLevel(logging.DEBUG)
-
 
 if is_raspi():
     GPIO.setmode(GPIO.BCM)

aquaPi/driver/DriverOneWire.py

@@ -7,13 +7,9 @@
 from .base import (AInDriver, IoPort, PortFunc, is_raspi,
                    DriverInvalidAddrError, DriverReadError)
 
-log = logging.getLogger('DriverOneWire')
+log = logging.getLogger('driver.DriverOneWire')
 log.brief = log.warning  # alias, warning is used as brief info, level info is verbose
 
-log.setLevel(logging.WARNING)
-# log.setLevel(logging.INFO)
-# log.setLevel(logging.DEBUG)
-
 
 # ========== 1-wire ==========
 

aquaPi/driver/DriverPWM.py

@@ -18,12 +18,9 @@ def gpio_function(pin):
 
 from .base import (OutDriver, IoPort, PortFunc, PinFunc, is_raspi)
 
-log = logging.getLogger('DriverPWM')
-log.brief = log.warning  # alias, warning is used as brief info, level info is verbose
 
-log.setLevel(logging.WARNING)
-# log.setLevel(logging.INFO)
-# log.setLevel(logging.DEBUG)
+log = logging.getLogger('driver.DriverPWM')
+log.brief = log.warning  # alias, warning is used as brief info, level info is verbose
 
 
 # ========== PWM ==========

aquaPi/driver/DriverTC420.py

@@ -12,12 +12,9 @@
 
 from .base import (OutDriver, IoPort, PortFunc)
 
-log = logging.getLogger('DriverTC420')
-log.brief = log.warning  # alias, warning is used as brief info, level info is verbose
 
-log.setLevel(logging.WARNING)
-# log.setLevel(logging.INFO)
-# log.setLevel(logging.DEBUG)
+log = logging.getLogger('driver.DriverTC420')
+log.brief = log.warning  # alias, warning is used as brief info, level info is verbose
 
 
 # ========== PWM ==========

aquaPi/driver/__init__.py

@@ -9,13 +9,9 @@
 from .base import Driver, DriverParamError, DriverPortInuseError
 from .base import *
 
-log = logging.getLogger('Driver Base')
+log = logging.getLogger('driver')
 log.brief = log.warning  # alias, warning is used as brief info, level info is verbose
 
-log.setLevel(logging.WARNING)
-# log.setLevel(logging.INFO)
-# log.setLevel(logging.DEBUG)
-
 
 # ========== IO registry ==========
 
@@ -158,7 +154,7 @@ def driver_destruct(self, port, driver):
 DRIVER_FILE_PREFIX = 'Driver'
 CUSTOM_DRIVERS = 'CustomDrivers'
 
-# import all files named Driver*,py into our package, icluding a subfolder CustomDrivers
+# import all files named Driver*.py into our package, including a subfolder CustomDrivers
 __path__.append(path.join(__path__[0], CUSTOM_DRIVERS))
 
 for drv_path in __path__:

aquaPi/driver/base.py

@@ -7,13 +7,9 @@
 import math
 import random
 
-log = logging.getLogger('Driver Base')
+log = logging.getLogger('driver.base')
 log.brief = log.warning  # alias, warning used as brief info, info is verbose
 
-log.setLevel(logging.WARNING)
-# log.setLevel(logging.INFO)
-# log.setLevel(logging.DEBUG)
-
 
 # ========== common functions ==========
 

aquaPi/machineroom/__init__.py

@@ -14,13 +14,9 @@
 from .alert_nodes import Alert, AlertAbove, AlertBelow
 
 
-log = logging.getLogger('MachineRoom')
+log = logging.getLogger('machineroom')
 log.brief = log.warning  # alias, warning used as brief info, info is verbose
 
-log.setLevel(logging.WARNING)
-# log.setLevel(logging.INFO)
-# log.setLevel(logging.DEBUG)
-
 
 mr = None
 

aquaPi/machineroom/alert_nodes.py

@@ -8,13 +8,9 @@
 from ..driver import (PortFunc, io_registry, DriverReadError)
 
 
-log = logging.getLogger('AlertNodes')
+log = logging.getLogger('machineroom.alert_nodes')
 log.brief = log.warning  # alias, warning is used as brief info, level info is verbose
 
-log.setLevel(logging.WARNING)
-log.setLevel(logging.INFO)
-# log.setLevel(logging.DEBUG)
-
 
 # ========== alert conditions ==========
 

aquaPi/machineroom/aux_nodes.py

@@ -5,13 +5,9 @@
 from .msg_bus import (BusListener, BusRole, DataRange, MsgData)
 
 
-log = logging.getLogger('AuxNodes')
+log = logging.getLogger('machineroom.aux_nodes')
 log.brief = log.warning  # alias, warning used as brief info, info is verbose
 
-log.setLevel(logging.WARNING)
-# log.setLevel(logging.INFO)
-# log.setLevel(logging.DEBUG)
-
 
 # ========== auxiliary ==========
 

aquaPi/machineroom/ctrl_nodes.py

@@ -10,13 +10,9 @@
 from .msg_bus import (BusListener, BusRole, DataRange, MsgData)
 
 
-log = logging.getLogger('CtrlNodes')
+log = logging.getLogger('machineroom.ctrl_nodes')
 log.brief = log.warning  # alias, warning used as brief info, info is verbose
 
-log.setLevel(logging.WARNING)
-# log.setLevel(logging.INFO)
-# log.setLevel(logging.DEBUG)
-
 
 def get_unit_limits(unit):
     if ('°C') in unit:

aquaPi/machineroom/hist_nodes.py

@@ -21,13 +21,9 @@
 # from ..driver import (PortFunc, io_registry, DriverReadError)
 
 
-log = logging.getLogger('HistoryNodes')
+log = logging.getLogger('machineroom.hist_nodes')
 log.brief = log.warning  # alias, warning is used as brief info, level info is verbose
 
-log.setLevel(logging.WARNING)
-# log.setLevel(logging.INFO)
-# log.setLevel(logging.DEBUG)
-
 
 # ========== interface to time series DB ==========
 
@@ -350,12 +346,12 @@ def __init__(self, name, inputs, duration=24, _cont=False):
         if QUEST_DB:
             try:
                 self.db = TimeDbQuest()
-                log.brief('Recording history in QuestDB')
+                log.brief('Recording history %s in QuestDB', name)
             except (NotImplementedError, ModuleNotFoundError, ImportError):
                 pass
         if not self.db:
             self.db = TimeDbMemory(duration)
-            log.brief('Recording history in main memory with limited depth of %dh!', duration)
+            log.brief('Recording history %s in main memory with limited depth of %dh!', name, duration)
         for snd in self._inputs.sender:
             self.db.add_field(snd)
 

aquaPi/machineroom/in_nodes.py

@@ -10,13 +10,9 @@
 from ..driver import (io_registry, DriverReadError)
 
 
-log = logging.getLogger('InNodes')
+log = logging.getLogger('machineroom.in_nodes')
 log.brief = log.warning  # alias, warning is used as brief info, level info is verbose
 
-log.setLevel(logging.WARNING)
-# log.setLevel(logging.INFO)
-# log.setLevel(logging.DEBUG)
-
 
 # ========== inputs AKA sensors ==========
 

aquaPi/machineroom/msg_bus.py

@@ -9,13 +9,9 @@
 from .msg_types import (MsgInfra, MsgBorn, MsgBye, MsgReply, MsgReplyHello, MsgData, MsgFilter)
 
 
-log = logging.getLogger('MsgBus')
+log = logging.getLogger('machineroom.msg_bus')
 log.brief = log.warning  # alias, warning is used as brief info, level info is verbose
 
-log.setLevel(logging.WARNING)
-# log.setLevel(logging.INFO)
-# log.setLevel(logging.DEBUG)
-
 
 class BusRole(Flag):
     IN_ENDP = auto()  # data sources: sensor, switch, schedule

aquaPi/machineroom/msg_types.py

@@ -3,8 +3,7 @@
 import logging
 
 
-log = logging.getLogger('MsgBusMsgs')
-log.setLevel(logging.WARNING)  # INFO)
+log = logging.getLogger('machineroom.msg_types')
 
 
 class Msg: