Merge pull request #319 from napalm-automation/develop

Release 0.9.0

.github/workflows/pythonpublish.yml

@@ -0,0 +1,27 @@
+name: Publish Python Package
+on:
+  push:
+    # Sequence of patterns matched against refs/tags
+    tags:
+      - '*'
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel twine
+    - name: Build and publish
+      env:
+        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+      run: |
+        python setup.py sdist bdist_wheel
+        twine upload dist/*

.travis.yml

@@ -3,6 +3,7 @@ python:
 - 2.7
 - 3.5
 - 3.6
+- 3.7
 install:
   - pip install tox-travis
   - pip install coveralls

docs/_templates/about.html

@@ -4,7 +4,7 @@
   </a>
 </p>
 <p>
-<iframe src="http://ghbtns.com/github-btn.html?user=napalm-automation&repo=napalm-logs&type=watch&count=true&size=large"
+<iframe src="https://ghbtns.com/github-btn.html?user=napalm-automation&repo=napalm-logs&type=watch&count=true&size=large"
   allowtransparency="true" frameborder="0" scrolling="0" width="200px" height="35px"></iframe>
 </p>
 

docs/_templates/links.html

@@ -3,7 +3,7 @@
 
 <h3>Useful Links</h3>
 <ul>
-  <li><a href="http://github.com/napalm-automation/napalm-logs">napalm-logs @ GitHub</a></li>
-  <li><a href="http://pypi.python.org/pypi/napalm-logs">napalm-logs @ PyPI</a></li>
-  <li><a href="http://github.com/napalm-automation/napalm-logs/issues">Issue Tracker</a></li>
+  <li><a href="https://github.com/napalm-automation/napalm-logs">napalm-logs @ GitHub</a></li>
+  <li><a href="https://pypi.python.org/pypi/napalm-logs">napalm-logs @ PyPI</a></li>
+  <li><a href="https://github.com/napalm-automation/napalm-logs/issues">Issue Tracker</a></li>
 </ul>

docs/publisher/alerta.rst

@@ -0,0 +1,114 @@
+.. _publisher-alerta:
+
+================
+Alerta Publisher
+================
+
+.. versionadded:: 0.9.0
+
+Publish napalm-logs events to an `Alerta <https://alerta.io/>`__ endpoint.
+
+.. note::
+
+    The :ref:`configuration-options-address` must have contain the ``http://``
+    or ``https://`` schema. The address can however be specified more explicitly
+    under the publisher configuration options, using the
+    :ref:`publisher-alerta-address` field.
+
+    Also, note that you need to provide the URL to the Alerta API, typically 
+    ending in ``/api``, but that may differ depending on your installation.
+
+.. image:: alerta_screenshot.png
+    :width: 100%
+    :alt: Alerta Screenshot
+
+Configuration examples:
+
+- From the command line
+
+.. code-block:: bash
+
+  $ napalm-logs --publisher alerta --address https://alerta.example.com/api
+
+- Basic YAML configuration
+
+.. code-block:: yaml
+
+  publisher: alerta
+
+- YAML configuration with more options
+
+.. code-block:: yaml
+
+  publisher:
+    - alerta:
+        address: https://alerta.example.com/api
+        environment: Production
+        key: HUGcQvd1_C-TKDrHVoZiNqaKS4jCcFYsGKuT0_W8
+        max_clients: 20
+
+Available options
+^^^^^^^^^^^^^^^^^
+
+The options are generally inherited from the :ref:`publisher-http` Publisher, 
+with the following notes:
+
+.. _publisher-alerta-address:
+
+``address``
+-----------
+
+Specifies the Alerta API address. The value must contain the ``http://`` or
+``https://`` schema.
+
+Example:
+
+.. code-block:: yaml
+
+  publisher:
+    - alerta:
+        address: 'https://alerta.example.com/api'
+
+.. _publisher-alerta-headers:
+
+``headers``
+-----------
+
+The headers to use with the HTTP requests.
+
+
+Some headers such as ``Content-type`` are added by default, while others
+such as ``Authorization`` are added depending on the
+:ref:`publisher-alerta-key` or :ref:`publisher-alerta-token` options.
+
+.. _publisher-alerta-key:
+
+``key``
+-------
+
+Optional value when executing the HTTP requests using an Alerta API key.
+
+Example:
+
+.. code-block:: yaml
+
+  publisher:
+    - alerta:
+        address: 'https://alerta.example.com/api'
+        key: HUGcQvd1_C-TKDrHVoZiNqaKS4jCcFYsGKuT0_W8
+
+.. _publisher-alerta-token:
+
+``token``
+---------
+
+Optional value when executing the HTTP requests using a bearer authentication.
+
+Example:
+
+.. code-block:: yaml
+
+  publisher:
+    - alerta:
+        address: 'https://alerta.example.com/api'
+        token: AbCdEf123456

docs/publisher/index.rst

@@ -68,6 +68,7 @@ Available publishers and their options
 .. toctree::
    :maxdepth: 1
 
+   alerta
    cli
    http
    kafka

docs/releases/0.9.0.rst

@@ -0,0 +1,47 @@
+.. _release-0.9.0:
+
+=================================
+Release 0.9.0 - Codename Hand Saw
+=================================
+
+New Publisher Interface: Alerta
+-------------------------------
+
+Publish napalm-logs events to an `Alerta <https://alerta.io/>`__ endpoint.
+The messages are published as alerts in the form ``<host>::<message ID>``, 
+e.g., ``router1::NTP_SERVER_UNREACHABLE``. See the following screenshot for
+a visual example:
+
+.. image:: alerta_screenshot.png
+    :width: 100%
+    :alt: Alerta Screenshot
+
+The alert is sent together with the complete napalm-logs event data, under the
+``attributes`` Alerta field, which you can potentially use for leveraging 
+various automatic actions or processing from these alerts. Using Alerta is 
+a great option to handle duplicate alerts.
+
+See :ref:`publisher-alerta` for more details.
+
+Update the key name for Fortinet
+--------------------------------
+
+The field ``source_ip``, and ``destination_ip`` have been renamed to 
+``source_address`` and ``destination_address`` respectively. See `PR #313 
+<https://github.com/napalm-automation/napalm-logs/pull/313>`_ for more 
+information.
+
+Fix Prometheus label displaying
+-------------------------------
+
+Avoid to display the 'b' prefix on the label with Python3. Thanks @Koaxiel 
+(Robin Douine) for `PR #317 
+<https://github.com/napalm-automation/napalm-logs/pull/317>`_.
+
+
+New Structured Messages
+-----------------------
+
+:ref:`NH_REGION_GRAB_FAILED` for Junos, a message from the FPC logs when failed
+to grab new region for next-hops. Thanks @luke-orden `PR #314 
+<https://github.com/napalm-automation/napalm-logs/pull/314>`_.

docs/releases/index.rst

@@ -8,11 +8,12 @@ Release Notes
 Latest Release
 ^^^^^^^^^^^^^^
 
-- :ref:`release-0.8.0`
+- :ref:`release-0.9.0`
 
 Previous Releases
 ^^^^^^^^^^^^^^^^^
 
+- :ref:`release-0.8.0`
 - :ref:`release-0.7.0`
 - :ref:`release-0.6.0`
 - :ref:`release-0.5.0`

napalm_logs/config/fortinet/FORWARD_TRAFFIC.yml

@@ -1,9 +1,9 @@
 # Logs files identified as type=traffic and subtype=forward (FortiOs 5.4).
 #
-# <189>date=2019-04-09 time=04:27:29 devname=fw01 devid=FG800D0123456789 logid=0000000013 type=traffic subtype=forward 
-# level=notice vd=root srcip=1.1.1.1 srcport=19982 srcintf="port1" dstip=10.10.10.10 dstport=179 dstintf="port3" 
-# poluuid=d4954a18-3b72-51e9-1163-77fc436ef3c9 sessionid=1768791677 proto=6 action=timeout policyid=34 policytype=policy 
-# dstcountry="Reserved" srccountry="Reserved" trandisp=noop service="BGP" duration=25 sentbyte=300 rcvdbyte=0 sentpkt=5 
+# <189>date=2019-04-09 time=04:27:29 devname=fw01 devid=FG800D0123456789 logid=0000000013 type=traffic subtype=forward
+# level=notice vd=root srcip=1.1.1.1 srcport=19982 srcintf="port1" dstip=10.10.10.10 dstport=179 dstintf="port3"
+# poluuid=d4954a18-3b72-51e9-1163-77fc436ef3c9 sessionid=1768791677 proto=6 action=timeout policyid=34 policytype=policy
+# dstcountry="Reserved" srccountry="Reserved" trandisp=noop service="BGP" duration=25 sentbyte=300 rcvdbyte=0 sentpkt=5
 # rcvdpkt=0 appcat="unscanned" crscore=5 craction=262144 crlevel=low
 #
 messages:
@@ -29,10 +29,10 @@ messages:
       variables:
         session//traffic//forward//level: level
         session//traffic//forward//vdom: vd
-        session//traffic//forward//source_ip: srcip
+        session//traffic//forward//source_address: srcip
         session//traffic//forward//source_port: srcport
         session//traffic//forward//source_interface: srcintf
-        session//traffic//forward//destination_ip: dstip
+        session//traffic//forward//destination_address: dstip
         session//traffic//forward//destination_port: dstport
         session//traffic//forward//destination_interface: dstintf
         session//traffic//forward//poluu_id: poluuId

napalm_logs/config/fortinet/LOCAL_TRAFFIC.yml

@@ -1,9 +1,9 @@
 # Logs files for traffic identified as type=traffic and subtype=local (FortiOs 5.4).
 #
-# <189>date=2019-04-04 time=09:19:21 devname=fw01 devid=FG800D0123456789 logid=0001000014 type=traffic 
-# subtype=local level=notice vd=root srcip=172.30.0.1 srcport=48770 srcintf="wan1" dstip=172.16.0.1 
-# dstport=443 dstintf="root" sessionid=3841389285 proto=6 action=close policyid=0 policytype=local-in-policy 
-# dstcountry="France" srccountry="France" trandisp=noop service="HTTPS" app="Web Management(HTTPS)" 
+# <189>date=2019-04-04 time=09:19:21 devname=fw01 devid=FG800D0123456789 logid=0001000014 type=traffic
+# subtype=local level=notice vd=root srcip=172.30.0.1 srcport=48770 srcintf="wan1" dstip=172.16.0.1
+# dstport=443 dstintf="root" sessionid=3841389285 proto=6 action=close policyid=0 policytype=local-in-policy
+# dstcountry="France" srccountry="France" trandisp=noop service="HTTPS" app="Web Management(HTTPS)"
 # duration=3 sentbyte=28187 rcvdbyte=334857 sentpkt=183 rcvdpkt=242 appcat="unscanned"
 #
 messages:
@@ -28,10 +28,10 @@ messages:
       variables:
         session//traffic//local//level: level
         session//traffic//local//vdom: vd
-        session//traffic//local//source_ip: srcip
+        session//traffic//local//source_address: srcip
         session//traffic//local//source_port: srcport
         session//traffic//local//source_interface: srcintf
-        session//traffic//local//destination_ip: dstip
+        session//traffic//local//destination_address: dstip
         session//traffic//local//destination_port: dstport
         session//traffic//local//destination_interface: dstintf
         session//traffic//local//session_id: sessiondId

napalm_logs/config/junos/BGP_PREFIX_THRESH_EXCEEDED.yml

@@ -18,8 +18,8 @@ messages:
     mapping:
       variables:
         bgp//neighbors//neighbor//{peer}//state//peer_as: asn
-        bgp//neighbors//neighbor//{peer}//afi_safis//afi_safi//{table}//state//prefixes//received: current
-        bgp//neighbors//neighbor//{peer}//afi_safis//afi_safi//{table}//ipv4_{type}//prefix_limit//state//max_prefixes: limit
+        bgp//neighbors//neighbor//{peer}//afi_safis//afi_safi//inet4//state//prefixes//received: current
+        bgp//neighbors//neighbor//{peer}//afi_safis//afi_safi//inet4//ipv4_{type}//prefix_limit//state//max_prefixes: limit
       static: {}
   - error: BGP_PREFIX_THRESH_EXCEEDED
     tag: BGP_PREFIX_THRESH_EXCEEDED

napalm_logs/config/junos/NH_REGION_GRAB_FAILED.yml

@@ -0,0 +1,15 @@
+messages:
+  # 'error' should be unique and vendor agnostic. Currently we are using the JUNOS syslog message name as the canonical name.
+  # This may change if we are able to find a more well defined naming system.
+  - error: NH_REGION_GRAB_FAILED
+    tag: JPRDS_NH
+    values:
+      id: (\d+)
+      nh: (\d+)
+    line: 'jprds_nh_alloc(),{id}: JNH[0] failed to grab new region for NH#{nh}'
+    model: NO_MODEL
+    mapping:
+      variables:
+        nh//region//failed//id: id
+        nh//region//failed//nh: nh
+      static: {}

napalm_logs/config/junos/init.yml

@@ -36,3 +36,16 @@ prefixes:
       # Some logs have data which can be inside brackets or parenthesis
       additionalData: (?:(?:\[|\()(.+)(?:\]|\)))?
     line: '{date} {time} {hostPrefix}{host} fpc{fpcId} {tag}{additionalData}:'
+  # The following matches dcpfe specific logs
+  - time_format: "%b %d %H:%M:%S"
+    values:
+      date: (\w+\s+\d+)
+      time: (\d\d:\d\d:\d\d)
+      hostPrefix: (re\d.)?
+      host: ([^ ]+)
+      fpcId: (\d+)
+      fpcId2: (\d+)
+      tag: (\w+)
+      # Some logs have data which can be inside brackets or parenthesis
+      additionalData: (?:(?:\[|\()(.+)(?:\]|\)))?
+    line: '{date} {time} {hostPrefix}{host} fpc{fpcId} fpc{fpcId2} dcpfe: {tag}{additionalData}:'

napalm_logs/server.py

@@ -296,21 +296,21 @@ def start(self):
                         message_key = base64.b64encode(message)
                         if self._buffer[message_key]:
                             log.info('"%s" seems to be already buffered, skipping', msg_dict['message'])
-                            napalm_logs_server_skipped_buffered_messages.labels(device_os=dev_os).inc()
+                            napalm_logs_server_skipped_buffered_messages.labels(device_os=dev_os.decode()).inc()
                             continue
                         log.debug('"%s" is not buffered yet, added', msg_dict['message'])
                         self._buffer[message_key] = 1
                     self.pub.send_multipart([dev_os,
                                              umsgpack.packb((msg_dict, address))])
                     # self.os_pipes[dev_os].send((msg_dict, address))
-                    napalm_logs_server_messages_with_identified_os.labels(device_os=dev_os).inc()
-                    napalm_logs_server_messages_device_queued.labels(device_os=dev_os).inc()
+                    napalm_logs_server_messages_with_identified_os.labels(device_os=dev_os.decode()).inc()
+                    napalm_logs_server_messages_device_queued.labels(device_os=dev_os.decode()).inc()
 
                 elif dev_os and dev_os not in self.started_os_proc:
                     # Identified the OS, but the corresponding process does not seem to be started.
                     log.info('Unable to queue the message to %s. Is the sub-process started?', dev_os)
-                    napalm_logs_server_messages_with_identified_os.labels(device_os=dev_os).inc()
-                    napalm_logs_server_messages_failed_device_queuing.labels(device_os=dev_os).inc()
+                    napalm_logs_server_messages_with_identified_os.labels(device_os=dev_os.decode()).inc()
+                    napalm_logs_server_messages_failed_device_queuing.labels(device_os=dev_os.decode()).inc()
 
                 elif not dev_os and self.opts['_server_send_unknown']:
                     # OS not identified, but the user requested to publish the message as-is

napalm_logs/transport/__init__.py

@@ -23,6 +23,8 @@
 from napalm_logs.transport.http import HAS_TORNADO
 from napalm_logs.transport.http import HAS_REQUESTS
 from napalm_logs.transport.http import HTTPTransport
+# ~~~Alerta~~~
+from napalm_logs.transport.alerta import AlertaTransport
 # from napalm_logs.transport.rabbitmq import RabbitMQTransport
 
 log = logging.getLogger(__file__)
@@ -46,6 +48,9 @@
 if HAS_REQUESTS or HAS_TORNADO:
     TRANSPORT_LOOKUP['http'] = HTTPTransport
 
+if HAS_REQUESTS or HAS_TORNADO:
+    TRANSPORT_LOOKUP['alerta'] = AlertaTransport
+
 
 def get_transport(name):
     '''