Denovo dev with monitoring

.gitignore

@@ -1,7 +1,10 @@
 .installed.cfg
 .mr.developer.cfg
+buildout.cfg
+.idea/
 bin/
 develop-eggs/
 eggs/
+etc/
 src/
 var/

buildout.cfg.example

@@ -2,6 +2,8 @@
 extends =
 	profiles/devel.cfg
 	profiles/docs.cfg
+#	profiles/statsd.cfg
+#	profiles/penstock_statsd.cfg
 
 [versions]
 cffi = 1.4.1

profiles/base.cfg

@@ -88,13 +88,15 @@ couchdb = True
 concord = True
 penstock =
 vaurien =
-statsd = True
+statsd = False
 webapp = True
 web_port = 6543
 web_host = 0.0.0.0
 endpoint = ipc://${buildout:directory}/var/circus_endpoint
 pubsub_endpoint = ipc://${buildout:directory}/var/circus_pubsub
 timezone = Europe/Kiev
+app_key = circus
+statsd_plugin = False
 
 [couchdb.ini]
 <= config-from-template

profiles/penstock_statsd.cfg

@@ -0,0 +1,11 @@
+[buildout]
+parts +=
+  penstock_handler.yaml
+
+[penstock.yaml]
+statsd = 1
+handler_config = ${buildout:directory}/etc/penstock_handler.yaml
+
+[penstock_handler.yaml]
+<= config-from-template
+app_key = penstock

profiles/statsd.cfg

@@ -0,0 +1,22 @@
+[buildout]
+package-name +=
+    pyramidtiming
+    statsdhandler
+parts +=
+    api_handler.yaml
+eggs += pyramidtiming [pyramid]
+
+[sources]
+pyramidtiming = git ${remotes:gh}openprocurement/pyramid-timing.git push_url=${remotes:gh_push}openprocurement/pyramid-timing.git
+statsdhandler = git ${remotes:gh}openprocurement/statsdhandler.git push_url=${remotes:gh_push}openprocurement/statsdhandler.git
+
+[openprocurement.api.ini]
+statsd = 1
+handler_config = ${buildout:directory}/etc/api_handler.yaml
+
+[api_handler.yaml]
+<= config-from-template
+app_key = api
+
+[circus.ini]
+statsd_plugin = True

profiles/versions.cfg

@@ -17,7 +17,6 @@ circus = 0.13.0
 circus-web = 1.0.0
 collective.recipe.sphinxbuilder = 0.8.2
 collective.recipe.template = 1.11
-coverage = 4.1
 jsonpointer = 1.9
 mr.developer = 1.31
 nose = 1.3.7
@@ -36,7 +35,7 @@ zope.deprecation = 4.1.1
 zope.interface = 4.1.1
 zope.proxy = 4.1.4
 ExtendedJournalHandler = 0.0.1
-mock = 1.0
+mock = 1.0.1
 
 # Required by:
 # zope.annotation==4.2.0

templates/api_handler.yaml

@@ -0,0 +1,13 @@
+main:
+  app_key: ${options['app_key']}
+  host: localhost
+  port: 8125
+publish_templates:
+  default:
+    - '%(attr)s;%(metric_name)s'
+counters:
+  MESSAGE_ID: {}
+  REQUEST_METHOD: {}
+  RESPONSE_CODE: {}
+histograms:
+  REQUEST_PROCESS_TIME: {}

templates/circus.ini

@@ -39,6 +39,16 @@ watcher = db
 max_mem = 1700M
 {% end %}
 
+{% if options['statsd_plugin'] %}
+[plugin:statsd]
+use = circus.plugins.statsd.FullStats
+host = localhost
+port = 8125
+sample_rate = 1.0
+application_name = ${options['app_key']}
+{% end %}
+
+
 [plugin:db_http]
 use = circus.plugins.http_observer.HttpObserver
 restart_on_error = db

templates/openprocurement.api.ini

@@ -16,6 +16,7 @@ pyramid.debug_routematch = false
 pyramid.debug_templates = ${options['debug']}
 pyramid.default_locale_name = en
 exclog.extra_info = true
+{% if 'pyramid_timing' in options %}pyramid_timing = ${options['pyramid_timing']}{% end %}
 {% if 'health_threshold' in options %}health_threshold = ${options['health_threshold']}{% end %}
 {% if 'update_after' in options %}update_after = ${options['update_after']}{% end %}
 {% if 'plugins' in options %}plugins = ${options['plugins']}{% end %}
@@ -71,14 +72,14 @@ backend = gevent
 keys = root,openprocurement${',exc_logger' if 'exc_logger' in options else ''}
 
 [handlers]
-keys = console${',loggly' if 'loggly' in options else ''}${',journal' if 'journal' in options else ''}
+keys = console${',loggly' if 'loggly' in options else ''}${',journal' if 'journal' in options else ''}${',statsd' if 'statsd' in options else ''}
 
 [formatters]
 keys = generic
 
 [logger_root]
-level = INFO
-handlers = console${',loggly' if 'loggly' in options else ''}
+level = DEBUG
+handlers = console${',loggly' if 'loggly' in options else ''}${',statsd' if 'statsd' in options else ''}
 
 [logger_openprocurement]
 level = DEBUG
@@ -90,6 +91,12 @@ class = StreamHandler
 args = (sys.stderr,)
 level = NOTSET
 formatter = generic
+{% if 'statsd' in options %}
+[handler_statsd]
+level = DEBUG
+class = statsdhandler.StatsdHandler
+args = ("${options['handler_config']}",)
+{% end %}
 {% if 'exc_logger' in options %}
 [logger_exc_logger]
 level = ERROR

templates/penstock.yaml

@@ -12,13 +12,18 @@ handlers:
     level: INFO
     formatter: simple
     stream: ext://sys.stdout
-
+{% if 'statsd' in options %}\
+  statsd:
+    level: DEBUG
+    class: statsdhandler.StatsdHandler
+    config_path: "${options['handler_config']}"
+{% end %}
 loggers:
   replichecker:
-    handlers: [console]
+    handlers: [console${',statsd' if 'statsd' in options else ''}]
     propagate: no
-    level: INFO
+    level: DEBUG
 
   "":
     handlers: [console]
-    level: INFO
+    level: DEBUG

templates/penstock_handler.yaml

@@ -0,0 +1,11 @@
+main:
+  app_key: ${options['app_key']}
+  host: localhost
+  port: 8125
+publish_templates:
+  default:
+    - '%(attr)s;%(metric_name)s'
+counters:
+  MESSAGE_ID: {}
+histograms:
+  REPLICATIONS_PROGRESS: {}