use structlog's get_logger if it is available (#591)

* use structlog's get_logger if it is available

If the user has set up structlog, this will add some additional context
to each log line, like level and timestamp. E.g.

    {"event": "Read traceparent header 00-42d5718b9ac8289e439a688afb17cbae-d6f9a1dad4c90dec-01"}

becomes

    {"event": "Read traceparent header 00-42d5718b9ac8289e439a688afb17cbae-d6f9a1dad4c90dec-01",
    "timestamp": "2019-09-17T15:37:00.049627Z", "logger": "elasticapm.traces", "level": "debug"}

This can be overridden by setting an environment variable
`ELASTIC_APM_DISABLE_STRUCTLOG` to "true".

* make use of flag necessary to switch to structlog, and add docs

docs/logging.asciidoc

@@ -64,6 +64,12 @@ log = logger.new()
 log.msg("some_event")
 ----
 
+===== Use structlog for agent-internal logging
+
+The Elastic APM Python agent uses logging to log internal events and issues.
+By default, it will use a `logging` logger.
+If your project uses structlog, you can tell the agent to use a structlog logger
+by setting the environment variable `ELASTIC_APM_USE_STRUCTLOG` to `true`.
 
 [[log-correlation]]
 === Log correlation in Elasticsearch

elasticapm/base.py

@@ -48,6 +48,7 @@
 from elasticapm.traces import Tracer, execution_context
 from elasticapm.utils import cgroup, compat, is_master_process, stacks, varmap
 from elasticapm.utils.encoding import enforce_label_format, keyword_field, shorten, transform
+from elasticapm.utils.logging import get_logger
 from elasticapm.utils.module_import import import_string
 from elasticapm.utils.threading import IntervalTimer
 
@@ -83,13 +84,13 @@ class Client(object):
     >>>     print ("Exception caught; reference is %%s" %% ident)
     """
 
-    logger = logging.getLogger("elasticapm")
+    logger = get_logger("elasticapm")
 
     def __init__(self, config=None, **inline):
         # configure loggers first
         cls = self.__class__
-        self.logger = logging.getLogger("%s.%s" % (cls.__module__, cls.__name__))
-        self.error_logger = logging.getLogger("elasticapm.errors")
+        self.logger = get_logger("%s.%s" % (cls.__module__, cls.__name__))
+        self.error_logger = get_logger("elasticapm.errors")
 
         self.tracer = None
         self.processors = []

elasticapm/conf/__init__.py

@@ -36,10 +36,11 @@
 import threading
 
 from elasticapm.utils import compat, starmatch_to_regex
+from elasticapm.utils.logging import get_logger
 
 __all__ = ("setup_logging", "Config")
 
-logger = logging.getLogger("elasticapm.conf")
+logger = get_logger("elasticapm.conf")
 
 
 class ConfigurationError(ValueError):

elasticapm/contrib/async_worker.py

@@ -29,15 +29,15 @@
 #  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 
 
-import logging
 import os
 import sys
 import time
 from threading import Lock, Thread
 
 from elasticapm.utils.compat import queue
+from elasticapm.utils.logging import get_logger
 
-logger = logging.getLogger("elasticapm")
+logger = get_logger("elasticapm")
 
 ELASTIC_APM_WAIT_SECONDS = 10
 

elasticapm/contrib/django/apps.py

@@ -28,7 +28,6 @@
 #  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 #  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-import logging
 from functools import partial
 
 from django.apps import AppConfig
@@ -37,8 +36,9 @@
 from elasticapm.conf import constants
 from elasticapm.contrib.django.client import get_client
 from elasticapm.utils.disttracing import TraceParent
+from elasticapm.utils.logging import get_logger
 
-logger = logging.getLogger("elasticapm.traces")
+logger = get_logger("elasticapm.traces")
 
 ERROR_DISPATCH_UID = "elasticapm-exceptions"
 REQUEST_START_DISPATCH_UID = "elasticapm-request-start"

elasticapm/contrib/django/client.py

@@ -31,8 +31,6 @@
 
 from __future__ import absolute_import
 
-import logging
-
 import django
 from django.conf import settings as django_settings
 from django.core.exceptions import DisallowedHost
@@ -43,6 +41,7 @@
 from elasticapm.conf import constants
 from elasticapm.contrib.django.utils import iterate_with_template_sources
 from elasticapm.utils import compat, encoding, get_url_dict
+from elasticapm.utils.logging import get_logger
 from elasticapm.utils.module_import import import_string
 from elasticapm.utils.wsgi import get_environ, get_headers
 
@@ -78,7 +77,7 @@ def get_client(client=None):
 
 
 class DjangoClient(Client):
-    logger = logging.getLogger("elasticapm.errors.client.django")
+    logger = get_logger("elasticapm.errors.client.django")
 
     def __init__(self, config=None, **inline):
         if config is None:

elasticapm/contrib/django/handlers.py

@@ -39,8 +39,9 @@
 from django.conf import settings as django_settings
 
 from elasticapm.handlers.logging import LoggingHandler as BaseLoggingHandler
+from elasticapm.utils.logging import get_logger
 
-logger = logging.getLogger("elasticapm.logging")
+logger = get_logger("elasticapm.logging")
 
 
 class LoggingHandler(BaseLoggingHandler):

elasticapm/contrib/flask/__init__.py

@@ -31,8 +31,6 @@
 
 from __future__ import absolute_import
 
-import logging
-
 import flask
 from flask import request, signals
 
@@ -45,8 +43,9 @@
 from elasticapm.traces import execution_context
 from elasticapm.utils import build_name_with_http_method_prefix
 from elasticapm.utils.disttracing import TraceParent
+from elasticapm.utils.logging import get_logger
 
-logger = logging.getLogger("elasticapm.errors.client")
+logger = get_logger("elasticapm.errors.client")
 
 
 def make_client(client_cls, app, **defaults):
@@ -128,7 +127,7 @@ def init_app(self, app, **defaults):
             self.client = make_client(self.client_cls, app, **defaults)
 
         # 0 is a valid log level (NOTSET), so we need to check explicitly for it
-        if self.logging or self.logging is 0:
+        if self.logging or self.logging is 0:  # noqa F632
             if self.logging is not True:
                 kwargs = {"level": self.logging}
             else:

elasticapm/contrib/opentracing/span.py

@@ -28,13 +28,12 @@
 #  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 #  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-import logging
-
 from opentracing.span import Span as OTSpanBase
 from opentracing.span import SpanContext as OTSpanContextBase
 
 from elasticapm import traces
 from elasticapm.utils import compat, get_url_dict
+from elasticapm.utils.logging import get_logger
 
 try:
     # opentracing-python 2.1+
@@ -47,7 +46,7 @@
     ot_logs = None
 
 
-logger = logging.getLogger("elasticapm.contrib.opentracing")
+logger = get_logger("elasticapm.contrib.opentracing")
 
 
 class OTSpan(OTSpanBase):

elasticapm/events.py

@@ -29,18 +29,18 @@
 #  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 
 
-import logging
 import random
 import sys
 
 from elasticapm.conf.constants import EXCEPTION_CHAIN_MAX_DEPTH
 from elasticapm.utils import compat, varmap
 from elasticapm.utils.encoding import keyword_field, shorten, to_unicode
+from elasticapm.utils.logging import get_logger
 from elasticapm.utils.stacks import get_culprit, get_stack_info, iter_traceback_frames
 
 __all__ = ("BaseEvent", "Exception", "Message")
 
-logger = logging.getLogger("elasticapm.events")
+logger = get_logger("elasticapm.events")
 
 
 class BaseEvent(object):

elasticapm/instrumentation/packages/base.py

@@ -29,13 +29,13 @@
 #  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import functools
-import logging
 import os
 
 from elasticapm.traces import execution_context
 from elasticapm.utils import wrapt
+from elasticapm.utils.logging import get_logger
 
-logger = logging.getLogger("elasticapm.instrument")
+logger = get_logger("elasticapm.instrument")
 
 
 class AbstractInstrumentedModule(object):

elasticapm/instrumentation/packages/elasticsearch.py

@@ -31,13 +31,13 @@
 from __future__ import absolute_import
 
 import json
-import logging
 
 import elasticapm
 from elasticapm.instrumentation.packages.base import AbstractInstrumentedModule
 from elasticapm.utils import compat
+from elasticapm.utils.logging import get_logger
 
-logger = logging.getLogger(__name__)
+logger = get_logger("elasticapm.instrument")
 
 
 API_METHOD_KEY_NAME = "__elastic_apm_api_method_name"

elasticapm/metrics/base_metrics.py

@@ -28,17 +28,17 @@
 #  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 #  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-import logging
 import threading
 import time
 from collections import defaultdict
 
 from elasticapm.conf import constants
 from elasticapm.utils import compat, is_master_process
+from elasticapm.utils.logging import get_logger
 from elasticapm.utils.module_import import import_string
 from elasticapm.utils.threading import IntervalTimer
 
-logger = logging.getLogger("elasticapm.metrics")
+logger = get_logger("elasticapm.metrics")
 
 DISTINCT_LABEL_LIMIT = 1000
 

elasticapm/traces.py

@@ -29,7 +29,6 @@
 #  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import functools
-import logging
 import random
 import re
 import threading
@@ -45,11 +44,12 @@
 from elasticapm.utils import compat, encoding, get_name_from_func
 from elasticapm.utils.deprecation import deprecated
 from elasticapm.utils.disttracing import TraceParent, TracingOptions
+from elasticapm.utils.logging import get_logger
 
 __all__ = ("capture_span", "tag", "label", "set_transaction_name", "set_custom_context", "set_user_context")
 
-error_logger = logging.getLogger("elasticapm.errors")
-logger = logging.getLogger("elasticapm.traces")
+error_logger = get_logger("elasticapm.errors")
+logger = get_logger("elasticapm.traces")
 
 _time_func = timeit.default_timer
 

elasticapm/transport/base.py

@@ -31,7 +31,6 @@
 #  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import gzip
-import logging
 import random
 import threading
 import time
@@ -40,8 +39,9 @@
 
 from elasticapm.contrib.async_worker import AsyncWorker
 from elasticapm.utils import compat, is_master_process, json_encoder
+from elasticapm.utils.logging import get_logger
 
-logger = logging.getLogger("elasticapm.transport")
+logger = get_logger("elasticapm.transport")
 
 
 class TransportException(Exception):

elasticapm/transport/http.py

@@ -31,7 +31,6 @@
 #  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import hashlib
-import logging
 import re
 import ssl
 
@@ -42,8 +41,9 @@
 from elasticapm.transport.base import TransportException
 from elasticapm.transport.http_base import AsyncHTTPTransportBase, HTTPTransportBase
 from elasticapm.utils import compat, json_encoder, read_pem_file
+from elasticapm.utils.logging import get_logger
 
-logger = logging.getLogger("elasticapm.transport.http")
+logger = get_logger("elasticapm.transport.http")
 
 
 class Transport(HTTPTransportBase):

elasticapm/utils/disttracing.py

@@ -29,9 +29,10 @@
 #  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import ctypes
-import logging
 
-logger = logging.getLogger("elasticapm.utils")
+from elasticapm.utils.logging import get_logger
+
+logger = get_logger("elasticapm.utils")
 
 
 class TraceParent(object):

elasticapm/utils/logging.py

@@ -0,0 +1,47 @@
+#  BSD 3-Clause License
+#
+#  Copyright (c) 2019, Elasticsearch BV
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+#
+#  * Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+#  * Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from
+#    this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+#  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+#  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+#  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+#  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+#  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import
+
+import logging
+import os
+
+get_logger = logging.getLogger
+
+if os.environ.get("ELASTIC_APM_USE_STRUCTLOG", "").lower() == "true":
+
+    try:
+        import structlog
+
+        get_logger = structlog.get_logger
+    except ImportError:
+        # use stdlib logger to log warning
+        logger = get_logger("elasticapm")
+        logger.warning("ELASTIC_APM_USE_STRUCTLOG is set, but structlog is not installed")