From a126a87158eb1198bc3481629e25d89d9cde5714 Mon Sep 17 00:00:00 2001 From: Michael Schuster Date: Tue, 28 Jan 2025 17:11:06 +0100 Subject: [PATCH] Allow custom log formats (#3288) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add log format env variable * Docs * Add style hint * Remove auto-propagation * Update docs/book/reference/environment-variables.md Co-authored-by: Barış Can Durak <36421093+bcdurak@users.noreply.github.com> * Fix dashboard opening logic * Add missing imports in docs * Quote env vars in dockerfile --------- Co-authored-by: Barış Can Durak <36421093+bcdurak@users.noreply.github.com> --- .../disable-colorful-logging.md | 3 ++ .../control-logging/disable-rich-traceback.md | 3 ++ .../enable-or-disable-logs-storing.md | 3 ++ .../control-logging/set-logging-format.md | 40 +++++++++++++++++++ .../control-logging/set-logging-verbosity.md | 3 ++ docs/book/reference/environment-variables.md | 8 ++++ docs/book/toc.md | 1 + src/zenml/constants.py | 2 + src/zenml/logger.py | 17 +++++++- src/zenml/utils/dashboard_utils.py | 27 +++++++++++-- .../utils/pipeline_docker_image_builder.py | 2 +- 11 files changed, 103 insertions(+), 6 deletions(-) create mode 100644 docs/book/how-to/control-logging/set-logging-format.md diff --git a/docs/book/how-to/control-logging/disable-colorful-logging.md b/docs/book/how-to/control-logging/disable-colorful-logging.md index 20adaabe1f2..6709d7859b4 100644 --- a/docs/book/how-to/control-logging/disable-colorful-logging.md +++ b/docs/book/how-to/control-logging/disable-colorful-logging.md @@ -13,6 +13,9 @@ ZENML_LOGGING_COLORS_DISABLED=true Note that setting this on the [client environment](../pipeline-development/configure-python-environments/README.md#client-environment-or-the-runner-environment) (e.g. your local machine which runs the pipeline) will automatically disable colorful logging on remote pipeline runs. If you wish to only disable it locally, but turn on for remote pipeline runs, you can set the `ZENML_LOGGING_COLORS_DISABLED` environment variable in your pipeline runs environment as follows: ```python +from zenml import pipeline +from zenml.config import DockerSettings + docker_settings = DockerSettings(environment={"ZENML_LOGGING_COLORS_DISABLED": "false"}) # Either add it to the decorator diff --git a/docs/book/how-to/control-logging/disable-rich-traceback.md b/docs/book/how-to/control-logging/disable-rich-traceback.md index a47f37c388f..e73d674e7b5 100644 --- a/docs/book/how-to/control-logging/disable-rich-traceback.md +++ b/docs/book/how-to/control-logging/disable-rich-traceback.md @@ -17,6 +17,9 @@ Note that setting this on the [client environment](../pipeline-development/confi If you wish to disable it also for [remote pipeline runs](../../user-guide/production-guide/cloud-orchestration.md), you can set the `ZENML_ENABLE_RICH_TRACEBACK` environment variable in your pipeline runs environment as follows: ```python +from zenml import pipeline +from zenml.config import DockerSettings + docker_settings = DockerSettings(environment={"ZENML_ENABLE_RICH_TRACEBACK": "false"}) # Either add it to the decorator diff --git a/docs/book/how-to/control-logging/enable-or-disable-logs-storing.md b/docs/book/how-to/control-logging/enable-or-disable-logs-storing.md index 13965f93819..35737be85d6 100644 --- a/docs/book/how-to/control-logging/enable-or-disable-logs-storing.md +++ b/docs/book/how-to/control-logging/enable-or-disable-logs-storing.md @@ -40,6 +40,9 @@ If you do not want to store the logs in your artifact store, you can: 2. Disable it by using the environmental variable `ZENML_DISABLE_STEP_LOGS_STORAGE` and setting it to `true`. This environmental variable takes precedence over the parameters mentioned above. Note this environmental variable needs to be set on the [execution environment](../pipeline-development/configure-python-environments/README.md#execution-environments), i.e., on the orchestrator level: ```python +from zenml import pipeline +from zenml.config import DockerSettings + docker_settings = DockerSettings(environment={"ZENML_DISABLE_STEP_LOGS_STORAGE": "true"}) # Either add it to the decorator diff --git a/docs/book/how-to/control-logging/set-logging-format.md b/docs/book/how-to/control-logging/set-logging-format.md new file mode 100644 index 00000000000..fbbad34350e --- /dev/null +++ b/docs/book/how-to/control-logging/set-logging-format.md @@ -0,0 +1,40 @@ +--- +description: How to set the logging format in ZenML. +--- + +# Set logging format + +If you want to change the default ZenML logging format, you can do so with the following environment variable: + +```bash +export ZENML_LOGGING_FORMAT='%(asctime)s %(message)s' +``` + +The logging format must use the `%`-string formatting style. Check out [this page](https://docs.python.org/3/library/logging.html#logrecord-attributes) for all available attributes. + +Note that setting this on the [client environment](../pipeline-development/configure-python-environments/README.md#client-environment-or-the-runner-environment) (e.g. your local machine which runs the pipeline) will **not automatically change the log format on remote pipeline runs**. That means setting this variable locally with only effect pipelines that run locally. + +If you wish to configure it also for [remote pipeline runs](../../user-guide/production-guide/cloud-orchestration.md), you can set the `ZENML_LOGGING_FORMAT` environment variable in your pipeline runs environment as follows: + +```python +from zenml import pipeline +from zenml.config import DockerSettings + +docker_settings = DockerSettings(environment={"ZENML_LOGGING_FORMAT": "%(asctime)s %(message)s"}) + +# Either add it to the decorator +@pipeline(settings={"docker": docker_settings}) +def my_pipeline() -> None: + my_step() + +# Or configure the pipelines options +my_pipeline = my_pipeline.with_options( + settings={"docker": docker_settings} +) +``` + + + +
ZenML Scarf
+ + diff --git a/docs/book/how-to/control-logging/set-logging-verbosity.md b/docs/book/how-to/control-logging/set-logging-verbosity.md index fa21a318ade..15271bd29b2 100644 --- a/docs/book/how-to/control-logging/set-logging-verbosity.md +++ b/docs/book/how-to/control-logging/set-logging-verbosity.md @@ -18,6 +18,9 @@ Note that setting this on the [client environment](../pipeline-development/confi If you wish to control for [remote pipeline runs](../../user-guide/production-guide/cloud-orchestration.md), you can set the `ZENML_LOGGING_VERBOSITY` environment variable in your pipeline runs environment as follows: ```python +from zenml import pipeline +from zenml.config import DockerSettings + docker_settings = DockerSettings(environment={"ZENML_LOGGING_VERBOSITY": "DEBUG"}) # Either add it to the decorator diff --git a/docs/book/reference/environment-variables.md b/docs/book/reference/environment-variables.md index c6452c26e40..d8b81da7bf3 100644 --- a/docs/book/reference/environment-variables.md +++ b/docs/book/reference/environment-variables.md @@ -15,6 +15,14 @@ export ZENML_LOGGING_VERBOSITY=INFO Choose from `INFO`, `WARN`, `ERROR`, `CRITICAL`, `DEBUG`. +## Logging format + +```bash +export ZENML_LOGGING_FORMAT='%(asctime)s %(message)s' +``` + +See [this page](../how-to/control-logging/set-logging-format.md) for more information. + ## Disable step logs Usually, ZenML [stores step logs in the artifact store](../how-to/control-logging/enable-or-disable-logs-storing.md), but this can sometimes cause performance bottlenecks, especially if the code utilizes progress bars. diff --git a/docs/book/toc.md b/docs/book/toc.md index 4640c8eb5eb..5c6527a6368 100644 --- a/docs/book/toc.md +++ b/docs/book/toc.md @@ -215,6 +215,7 @@ * [View logs on the dashboard](how-to/control-logging/view-logs-on-the-dasbhoard.md) * [Enable or disable logs storage](how-to/control-logging/enable-or-disable-logs-storing.md) * [Set logging verbosity](how-to/control-logging/set-logging-verbosity.md) + * [Set logging format](how-to/control-logging/set-logging-format.md) * [Disable `rich` traceback output](how-to/control-logging/disable-rich-traceback.md) * [Disable colorful logging](how-to/control-logging/disable-colorful-logging.md) * [Popular integrations](how-to/popular-integrations/README.md) diff --git a/src/zenml/constants.py b/src/zenml/constants.py index d3b85a24ca4..369f1b57cec 100644 --- a/src/zenml/constants.py +++ b/src/zenml/constants.py @@ -138,6 +138,7 @@ def handle_int_env_var(var: str, default: int = 0) -> int: ENV_ZENML_CONFIG_PATH = "ZENML_CONFIG_PATH" ENV_ZENML_DEBUG = "ZENML_DEBUG" ENV_ZENML_LOGGING_VERBOSITY = "ZENML_LOGGING_VERBOSITY" +ENV_ZENML_LOGGING_FORMAT = "ZENML_LOGGING_FORMAT" ENV_ZENML_REPOSITORY_PATH = "ZENML_REPOSITORY_PATH" ENV_ZENML_PREVENT_PIPELINE_EXECUTION = "ZENML_PREVENT_PIPELINE_EXECUTION" ENV_ZENML_ENABLE_RICH_TRACEBACK = "ZENML_ENABLE_RICH_TRACEBACK" @@ -154,6 +155,7 @@ def handle_int_env_var(var: str, default: int = 0) -> int: ENV_ZENML_BACKUP_SECRETS_STORE_PREFIX = "ZENML_BACKUP_SECRETS_STORE_" ENV_ZENML_SKIP_PIPELINE_REGISTRATION = "ZENML_SKIP_PIPELINE_REGISTRATION" ENV_AUTO_OPEN_DASHBOARD = "AUTO_OPEN_DASHBOARD" +ENV_ZENML_AUTO_OPEN_DASHBOARD = "ZENML_AUTO_OPEN_DASHBOARD" ENV_ZENML_DISABLE_DATABASE_MIGRATION = "DISABLE_DATABASE_MIGRATION" ENV_ZENML_LOCAL_STORES_PATH = "ZENML_LOCAL_STORES_PATH" ENV_ZENML_CONTAINER = "ZENML_CONTAINER" diff --git a/src/zenml/logger.py b/src/zenml/logger.py index 685f416a0cb..615723048f0 100644 --- a/src/zenml/logger.py +++ b/src/zenml/logger.py @@ -24,6 +24,7 @@ from zenml.constants import ( ENABLE_RICH_TRACEBACK, ENV_ZENML_LOGGING_COLORS_DISABLED, + ENV_ZENML_LOGGING_FORMAT, ENV_ZENML_SUPPRESS_LOGS, ZENML_LOGGING_VERBOSITY, handle_bool_env_var, @@ -144,6 +145,18 @@ def set_root_verbosity() -> None: get_logger(__name__).debug("Logging NOTSET") +def get_formatter() -> logging.Formatter: + """Get a configured logging formatter. + + Returns: + The formatter. + """ + if log_format := os.environ.get(ENV_ZENML_LOGGING_FORMAT, None): + return logging.Formatter(fmt=log_format) + else: + return CustomFormatter() + + def get_console_handler() -> Any: """Get console handler for logging. @@ -151,7 +164,7 @@ def get_console_handler() -> Any: A console handler. """ console_handler = logging.StreamHandler(sys.stdout) - console_handler.setFormatter(CustomFormatter()) + console_handler.setFormatter(get_formatter()) return console_handler @@ -179,7 +192,7 @@ def init_logging() -> None: set_root_verbosity() console_handler = logging.StreamHandler(sys.stdout) - console_handler.setFormatter(CustomFormatter()) + console_handler.setFormatter(get_formatter()) logging.root.addHandler(console_handler) # Enable logs if environment variable SUPPRESS_ZENML_LOGS is not set to True diff --git a/src/zenml/utils/dashboard_utils.py b/src/zenml/utils/dashboard_utils.py index 12efa89b856..ab92293ee47 100644 --- a/src/zenml/utils/dashboard_utils.py +++ b/src/zenml/utils/dashboard_utils.py @@ -13,6 +13,7 @@ # permissions and limitations under the License. """Utility class to help with interacting with the dashboard.""" +import os from typing import Optional from uuid import UUID @@ -156,9 +157,27 @@ def show_dashboard(url: str) -> None: display(IFrame(src=url, width="100%", height=720)) elif environment in (EnvironmentType.NATIVE, EnvironmentType.WSL): - if constants.handle_bool_env_var( + open_dashboard = True + + if constants.ENV_AUTO_OPEN_DASHBOARD in os.environ: + logger.warning( + "The `%s` environment variable is deprecated, use the `%s` " + "environment variable instead.", + constants.ENV_AUTO_OPEN_DASHBOARD, + constants.ENV_ZENML_AUTO_OPEN_DASHBOARD, + ) + + if not constants.handle_bool_env_var( constants.ENV_AUTO_OPEN_DASHBOARD, default=True ): + open_dashboard = False + + if not constants.handle_bool_env_var( + constants.ENV_ZENML_AUTO_OPEN_DASHBOARD, default=True + ): + open_dashboard = False + + if open_dashboard: try: import webbrowser @@ -169,14 +188,16 @@ def show_dashboard(url: str) -> None: logger.info( "Automatically opening the dashboard in your " "browser. To disable this, set the env variable " - "AUTO_OPEN_DASHBOARD=false." + "`%s=false`.", + constants.ENV_ZENML_AUTO_OPEN_DASHBOARD, ) except Exception as e: logger.error(e) else: logger.info( "To open the dashboard in a browser automatically, " - "set the env variable AUTO_OPEN_DASHBOARD=true." + "set the env variable `%s=true`.", + constants.ENV_ZENML_AUTO_OPEN_DASHBOARD, ) else: diff --git a/src/zenml/utils/pipeline_docker_image_builder.py b/src/zenml/utils/pipeline_docker_image_builder.py index f90fa2403af..b82a260fe42 100644 --- a/src/zenml/utils/pipeline_docker_image_builder.py +++ b/src/zenml/utils/pipeline_docker_image_builder.py @@ -587,7 +587,7 @@ def _generate_zenml_pipeline_dockerfile( f"ENV {ENV_ZENML_LOGGING_COLORS_DISABLED}={str(handle_bool_env_var(ENV_ZENML_LOGGING_COLORS_DISABLED, False))}" ) for key, value in docker_settings.environment.items(): - lines.append(f"ENV {key.upper()}={value}") + lines.append(f"ENV {key.upper()}='{value}'") if apt_packages: apt_packages = " ".join(f"'{p}'" for p in apt_packages)