diff --git a/.gitignore b/.gitignore index 84c75c4d..dc2c9a9f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,17 @@ -Gemfile.lock -bin -node_modules -pkg -spec/tmp -src/github.com -vendor -version.txt -.Makefile.tags -.bundle -.log-courier -.vagrant -*.gem +/.Makefile.tags +/.bundle +/.log-courier +/.vagrant +/*.gem +/Gemfile.lock +/bin +/log-courier.gemspec +/logstash-input-log-courier.gemspec +/logstash-output-log-courier.gemspec +/node_modules +/pkg +/spec/tmp +/src/github.com +/src/lc-lib/core/version.go +/vendor +/version.txt diff --git a/Makefile b/Makefile index b931e32f..c27163de 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: prepare fix_version all log-courier gem gem_plugins push_gems test test_go test_rspec doc profile benchmark jrprofile jrbenchmark clean +.PHONY: prepare fix_version setup_root all log-courier gem gem_plugins push_gems test test_go test_rspec doc profile benchmark jrprofile jrbenchmark clean MAKEFILE := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST)) GOPATH := $(patsubst %/,%,$(dir $(abspath $(MAKEFILE)))) @@ -100,9 +100,12 @@ ifneq ($(implyclean),yes) endif fix_version: - build/fix_version + build/fix_version "${FIX_VERSION}" -prepare: | fix_version +setup_root: + build/setup_root + +prepare: | fix_version setup_root @go version >/dev/null || (echo "Go not found. You need to install Go version 1.2-1.4: http://golang.org/doc/install"; false) @go version | grep -q 'go version go1.[234]' || (echo "Go version 1.2-1.4, you have a version of Go that is not supported."; false) @echo "GOPATH: $${GOPATH}" diff --git a/README.md b/README.md index 6cdebc9c..19139e70 100644 --- a/README.md +++ b/README.md @@ -1,79 +1,134 @@ -# Log Courier [![Build Status](https://travis-ci.org/driskell/log-courier.svg?branch=develop)](https://travis-ci.org/driskell/log-courier) +# Log Courier -Log Courier is a tool created to ship log files speedily and securely to -remote [Logstash](http://logstash.net) instances for processing whilst using -small amounts of local resources. The project is an enhanced fork of +[![Build Status](https://img.shields.io/travis/driskell/log-courier/develop.svg)](https://travis-ci.org/driskell/log-courier) +[![Latest Release](https://img.shields.io/github/release/driskell/log-courier.svg)](https://github.com/driskell/log-courier/releases/latest) + +Log Courier is a lightweight tool created to ship log files speedily and +securely, with low resource usage, to remote [Logstash](http://logstash.net) +instances. The project is an enhanced fork of [Logstash Forwarder](https://github.com/elasticsearch/logstash-forwarder) 0.3.1 with many fixes and behavioural improvements. -**Table of Contents** *generated with [DocToc](http://doctoc.herokuapp.com/)* - -- [Features](#features) -- [Installation](#installation) - - [Requirements](#requirements) - - [Building](#building) - - [Logstash Integration](#logstash-integration) - - [Building with ZMQ support](#building-with-zmq-support) - - [Generating Certificates and Keys](#generating-certificates-and-keys) +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* + +- [Main Features](#main-features) +- [Differences to Logstash Forwarder](#differences-to-logstash-forwarder) +- [Public Repositories](#public-repositories) + - [RPM](#rpm) + - [DEB](#deb) +- [Building From Source](#building-from-source) +- [Logstash Integration](#logstash-integration) +- [Generating Certificates and Keys](#generating-certificates-and-keys) +- [ZeroMQ support](#zeromq-support) - [Documentation](#documentation) -## Features - -Log Courier implements the following features: +## Main Features -* Follow active log files -* Follow rotations -* Follow standard input stream -* Suspend tailing after periods of inactivity -* Set [extra fields](docs/Configuration.md#fields), supporting hashes and arrays -(`tags: ['one','two']`) +* Read events from a file or over a Unix pipeline +* Follow log file rotations and movements +* Close files after inactivity, reopening if they change +* Add [extra fields](docs/Configuration.md#fields) to events prior to shipping * [Reload configuration](docs/Configuration.md#reloading) without restarting -* Secure TLS shipping transport with server certificate verification -* TLS client certificate verification -* Secure CurveZMQ shipping transport to load balance across multiple Logstash -instances (optional, requires ZeroMQ 4+) -* Plaintext TCP shipping transport for configuration simplicity in local -networks -* Plaintext ZMQ shipping transport -* [Administration utility](docs/AdministrationUtility.md) to monitor the -shipping speed and status -* [Multiline](docs/codecs/Multiline.md) codec -* [Filter](docs/codecs/Filter.md) codec +* Ship events securely using TLS with server (and optionally client) certificate +verification +* Ship events securely to multiple Logstash instances using ZeroMQ with Curve +security (requires ZeroMQ 4+) +* Ship events in plaintext using TCP +* Ship events in plaintext using ZeroMQ (requires ZeroMQ 3+) +* Monitor shipping speed and status with the +[Administration utility](docs/AdministrationUtility.md) +* Pre-process events using codecs (e.g. [Multiline](docs/codecs/Multiline.md), +[Filter](docs/codecs/Filter.md)) * [Logstash Integration](docs/LogstashIntegration.md) with an input and output plugin +* Very low resource usage + +## Differences to Logstash Forwarder + +Log Courier is an enhanced fork of +[Logstash Forwarder](https://github.com/elasticsearch/logstash-forwarder) 0.3.1 +with many fixes and behavioural improvements. The primary changes are: + +* The publisher protocol is rewritten to avoid many causes of "i/o timeout" +which would result in duplicate events sent to Logstash +* The prospector and registrar are heavily revamped to handle log rotations and +movements far more reliably, and to report errors cleanly +* The harvester is improved to retry if an error occurred rather than stop +* The configuration can be reloaded without restarting +* An administration tool is available which can display the shipping speed and +status of all watched log files +* Fields configurations can contain arrays and dictionaries, not just strings +* Codec support is available which allows multiline processing at the sender +side +* A TCP transport is available which removes the requirement for SSL +certificates +* There is support for client SSL certificate verification +* Peer IP address and certificate DN can be added to received events in Logstash +to distinguish events send from different instances +* Windows: Log files are not locked allowing log rotation to occur +* Windows: Log rotation is detected correctly + +## Public Repositories + +### RPM -## Installation +The author maintains a **COPR** repository with RedHat/CentOS compatible RPMs +that may be installed using `yum`. This repository depends on the widely used +**EPEL** repository for dependencies. -### Requirements +The **EPEL** repository can be installed automatically on CentOS distributions +by running `yum install epel-release`. Otherwise, you may follow the +instructions on the [EPEL homepage](https://fedoraproject.org/wiki/EPEL). -1. \*nix, OS X or Windows +To install the Log Courier repository, download the corresponding `.repo` +configuration file below, and place it in `/etc/yum.repos.d`. Log Courier may +then be installed using `yum install log-courier`. + +* **CentOS/RedHat 6.x**: [driskell-log-courier-epel-6.repo](https://copr.fedoraproject.org/coprs/driskell/log-courier/repo/epel-6/driskell-log-courier-epel-6.repo) +* **CentOS/RedHat 7.x**: +[driskell-log-courier-epel-7.repo](https://copr.fedoraproject.org/coprs/driskell/log-courier/repo/epel-6/driskell-log-courier-epel-7.repo) + +***NOTE:*** *The RPM packages versions of Log Courier are built using ZeroMQ 3.2 +and therefore do not support the encrypted `zmq` transport. They do support the +unencrypted `plainzmq` transport.* + +### DEB + +A Debian/Ubuntu compatible **PPA** repository is under consideration. At the +moment, no such repository exists. + +## Building From Source + +Requirements: + +1. Linux, Unix, OS X or Windows 1. The [golang](http://golang.org/doc/install) compiler tools (1.2-1.4) 1. [git](http://git-scm.com) 1. GNU make -***\*nix:*** *Most requirements can usually be installed by your favourite package -manager.* +***Linux/Unix:*** *Most requirements can usually be installed by your favourite +package manager.* ***OS X:*** *Git and GNU make are provided automatically by XCode.* ***Windows:*** *GNU make for Windows can be found [here](http://gnuwin32.sourceforge.net/packages/make.htm).* -### Building - -To build, simply run `make` as follows. +To build the binaries, simply run `make` as follows. git clone https://github.com/driskell/log-courier cd log-courier make -The log-courier program can then be found in the 'bin' folder. +The log-courier program can then be found in the 'bin' folder. This can be +manually installed anywhere on your system. Startup scripts for various +platforms can be found in the [contrib/initscripts](contrib/initscripts) folder. *Note: If you receive errors whilst running `make`, try `gmake` instead.* -### Logstash Integration +## Logstash Integration Log Courier does not utilise the lumberjack Logstash plugin and instead uses its own custom plugin. This allows significant enhancements to the integration far @@ -87,14 +142,29 @@ Install using the Logstash 1.5+ Plugin manager. Detailed instructions, including integration with Logstash 1.4.x, can be found on the [Logstash Integration](docs/LogstashIntegration.md) page. -### Building with ZMQ support +## Generating Certificates and Keys -To use the 'plainzmq' and 'zmq' transports, you will need to install -[ZeroMQ](http://zeromq.org/intro:get-the-software) (>=3.2 for cleartext -'plainzmq', >=4.0 for encrypted 'zmq'). +Log Courier provides two commands to help generate SSL certificates and Curve +keys, `lc-tlscert` and `lc-curvekey` respectively. Both are bundled with the +packages provided by the public repositories. -***\*nix:*** *ZeroMQ >=3.2 is usually available via the package manager. ZeroMQ >=4.0 -may need to be built and installed manually.* +When building from source, running `make selfsigned` will automatically build +and run the `lc-tlscert` utility that can quickly and easily generate a +self-signed certificate for the TLS shipping transport. + +Likewise, running `make curvekey` will automatically build and run the +`lc-curvekey` utility that can quickly and easily generate CurveZMQ key pairs +for the CurveZMQ shipping transport. This tool is only available when Log +Courier is built with ZeroMQ >=4.0. + +## ZeroMQ support + +To use the 'plainzmq' or 'zmq' transports, you will need to install +[ZeroMQ](http://zeromq.org/intro:get-the-software) (>=3.2 for 'plainzmq', >=4.0 +for 'zmq' which supports encryption). + +***Linux\Unix:*** *ZeroMQ >=3.2 is usually available via the package manager. +ZeroMQ >=4.0 may need to be built and installed manually.* ***OS X:*** *ZeroMQ can be installed via [Homebrew](http://brew.sh).* ***Windows:*** *ZeroMQ will need to be built and installed manually.* @@ -113,19 +183,6 @@ the Log Courier hosts are of the same major version. A Log Courier host that has ZeroMQ 4.0.5 will not work with a Logstash host using ZeroMQ 3.2.4 (but will work with a Logstash host using ZeroMQ 4.0.4.)** -### Generating Certificates and Keys - -Running `make selfsigned` will automatically build and run the `lc-tlscert` -utility that can quickly and easily generate a self-signed certificate for the -TLS shipping transport. - -Likewise, running `make curvekey` will automatically build and run the -`lc-curvekey` utility that can quickly and easily generate CurveZMQ key pairs -for the CurveZMQ shipping transport. This tool is only available when Log -Courier is built with ZeroMQ >=4.0. - -Both tools also generate the required configuration file snippets. - ## Documentation * [Administration Utility](docs/AdministrationUtility.md) diff --git a/build/fix_version b/build/fix_version index 006ec5f5..7b27d0bd 100755 --- a/build/fix_version +++ b/build/fix_version @@ -1,32 +1,35 @@ #!/bin/bash -# If this is not a git repository, use the existing version -if [ ! -d '.git' ]; then - exit -fi - if [ -n "$1" ]; then + # When specified on the command line, it's always short, and means we're preparing a release VERSION="$1" + VERSION_SHORT="$VERSION" +elif [ ! -d '.git' ]; then + # Not a git repository, so use the existing version_short.txt + VERSION="$(cat version_short.txt)" + VERSION_SHORT="$VERSION" else # Describe version from Git, and ensure the only "-xxx" is the git revision # This ensures that gem builds only add one ".pre" tag automatically VERSION="$(git describe | sed 's/-\([0-9][0-9]*\)-\([0-9a-z][0-9a-z]*\)$/.\1.\2/g')" VERSION="${VERSION#v}" + VERSION_SHORT=$(git describe --abbrev=0) + VERSION_SHORT="${VERSION_SHORT#v}" fi # Patch version.go -sed "s/\\(const *Log_Courier_Version *string *= *\"\\)[^\"]*\\(\"\\)/\\1${VERSION}\\2/g" src/lc-lib/core/version.go > src/lc-lib/core/version.go.tmp -\mv -f src/lc-lib/core/version.go.tmp src/lc-lib/core/version.go +sed "s//${VERSION}/g" src/lc-lib/core/version.go.tmpl > src/lc-lib/core/version.go # Patch the gemspecs for GEM in log-courier logstash-input-log-courier logstash-output-log-courier; do - sed "s/\\(gem.version *= *'\\)[^']*\\('\\)/\\1${VERSION}\\2/g" ${GEM}.gemspec > ${GEM}.gemspec.tmp - \mv -f ${GEM}.gemspec.tmp ${GEM}.gemspec - [ ${GEM#logstash-} != $GEM ] && { - sed "s/\\(gem.add_runtime_dependency *'log-courier' *, *'= *\\)[^']*\\('\\)/\\1${VERSION}\\2/g" ${GEM}.gemspec > ${GEM}.gemspec.tmp - \mv -f ${GEM}.gemspec.tmp ${GEM}.gemspec - } + sed "s//${VERSION}/g" ${GEM}.gemspec.tmpl > ${GEM}.gemspec done +# Store the full version in version.txt for other scripts to use, such as push_gems echo "${VERSION}" > version.txt + +# Store the nearest tag in version_short.txt - this is the only file stored in the repo +# This file is used as the version if we download Log Courier as a non-git package +echo "${VERSION_SHORT}" > version_short.txt + echo "Set Log Courier Version ${VERSION}" diff --git a/build/push_gems b/build/push_gems index 5f199bea..365aa577 100755 --- a/build/push_gems +++ b/build/push_gems @@ -1,6 +1,6 @@ #!/bin/bash -for GEM in *-$(cat version.txt).gem; do +for GEM in *-$(cat version_short.txt).gem; do echo "- ${GEM}" gem push $GEM done diff --git a/build/setup_root b/build/setup_root new file mode 100755 index 00000000..79e6bedd --- /dev/null +++ b/build/setup_root @@ -0,0 +1,5 @@ +#!/bin/bash + +# Allow the source code to refer to github.com/driskell/log-courier paths +mkdir -p src/github.com/driskell +ln -nsf ../../.. src/github.com/driskell/log-courier diff --git a/contrib/rpm/log-courier.spec b/contrib/rpm/log-courier.spec index 0ead8b28..729217d7 100644 --- a/contrib/rpm/log-courier.spec +++ b/contrib/rpm/log-courier.spec @@ -4,8 +4,8 @@ Summary: Log Courier Name: log-courier -Version: 1.2 -Release: 4%{dist} +Version: 1.5 +Release: 1%{dist} License: GPL Group: System Environment/Libraries Packager: Jason Woods @@ -32,10 +32,8 @@ Requires: zeromq3 Requires: logrotate %description -Log Courier is a tool created to transmit log files speedily and securely to -remote Logstash instances for processing whilst using small amounts of local -resources. The project is an enhanced fork of Logstash Forwarder 0.3.1 with many -enhancements and behavioural improvements. +Log Courier is a lightweight tool created to ship log files speedily and +securely, with low resource usage, to remote Logstash instances. %prep %setup -q -n %{name}-%{version} @@ -71,6 +69,8 @@ mkdir -p %{buildroot}%{_sysconfdir}/init.d install -m 0755 contrib/initscripts/redhat-sysv.init %{buildroot}%{_sysconfdir}/init.d/log-courier touch %{buildroot}%{_var}/run/log-courier.pid %endif +mkdir -p %{buildroot}%{_sysconfdir}/sysconfig +install -m 0644 contrib/initscripts/log-courier.sysconfig %{buildroot}%{_sysconfdir}/sysconfig/log-courier # Make the state dir mkdir -p %{buildroot}%{_var}/lib/log-courier @@ -117,7 +117,9 @@ fi %endif %defattr(0644,root,root,0755) -%{_sysconfdir}/log-courier +%dir %{_sysconfdir}/log-courier +%{_sysconfdir}/log-courier/examples +%config(noreplace) %{_sysconfdir}/sysconfig/log-courier %if 0%{?rhel} < 7 %ghost %{_var}/run/log-courier.pid %endif @@ -127,6 +129,15 @@ fi %ghost %{_var}/lib/log-courier/.log-courier %changelog +* Sat Feb 28 2015 Jason Woods - 1.5-1 +- Upgrade to v1.5 + +* Mon Jan 5 2015 Jason Woods - 1.3-1 +- Upgrade to v1.3 + +* Wed Dec 3 2014 Jason Woods - 1.2-5 +- Upgrade to v1.2 final + * Sat Nov 8 2014 Jason Woods - 1.2-4 - Upgrade to v1.2 - Fix stop message on future upgrade diff --git a/docs/AdministrationUtility.md b/docs/AdministrationUtility.md index f822bf56..cc9e0861 100644 --- a/docs/AdministrationUtility.md +++ b/docs/AdministrationUtility.md @@ -2,7 +2,7 @@ -**Table of Contents** *generated with [DocToc](http://doctoc.herokuapp.com/)* +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - [Overview](#overview) - [Available Commands](#available-commands) diff --git a/docs/ChangeLog.md b/docs/ChangeLog.md index 1c57497a..da5704b4 100644 --- a/docs/ChangeLog.md +++ b/docs/ChangeLog.md @@ -2,9 +2,10 @@ -**Table of Contents** *generated with [DocToc](http://doctoc.herokuapp.com/)* +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* -- [?.?](#) +- [1.5](#15) +- [1.3](#13) - [1.2](#12) - [1.1](#11) - [1.0](#10) @@ -19,9 +20,70 @@ +## 1.5 + +*28th February 2015* + +***Breaking Changes*** + +* The way in which logs are read from stdin has been significantly changed. The +"-" path in the configuration is no longer special and no longer reads from +stdin. Instead, you must now start log-courier with the `-stdin` command line +argument, and configure the codec and additional fields in the new `stdin` +configuration file section. Log Courier will now also exit cleanly once all data +from stdin has been read and acknowledged by the server (previously it would +hang forever.) +* The output plugin will fail startup if more than a single `host` address is +provided. Previous versions would simply ignore additional hosts and cause +potential confusion. + +***Changes*** + +* Implement random selection of the initial server connection. This partly +reverts a change made in version 1.2. Subsequent connections due to connection +failures will still round robin. +* Allow use of certificate files containing intermediates within the Log Courier +configuration. (Thanks @mhughes - #88) +* A configuration reload will now reopen log files. (#91) +* Implement support for SRV record server entries (#85) +* Fix Log Courier output plugin (#96 #98) +* Fix Logstash input plugin with zmq transport failing when discarding a message +due to peer_recv_queue being exceeded (#92) +* Fix a TCP transport race condition that could deadlock publisher on a send() +error (#100) +* Fix "address already in use" startup error when admin is enabled on a unix +socket and the unix socket file already exists during startup (#101) +* Report the location in the configuration file of any syntax errors (#102) +* Fix an extremely rare race condition where a dead file may not be resumed if +it is updated at the exact moment it is marked as dead +* Remove use_bigdecimal JrJackson JSON decode option as Logstash does not +support it. Also, using this option enables it globally within Logstash due to +option leakage within the JrJackson gem (#103) +* Fix filter codec not saving offset correctly when dead time reached or stdin +EOF reached (reported in #108) +* Fix Logstash input plugin crash if the fields configuration for Log Courier +specifies a "tags" field that is not an Array, and the input configuration for +Logstash also specified tags (#118) +* Fix a registrar conflict bug that can occur if a followed log file becomes +inaccessible to Log Courier (#122) +* Fix inaccessible log files causing errors to be reported to the Log Courier +log target every 10 seconds. Only a single error should be reported (#119) +* Fix unknown plugin error in Logstash input plugin if a connection fails to +accept (#118) +* Fix Logstash input plugin crash with plainzmq and zmq transports when the +listen address is already in use (Thanks to @mheese - #112) +* Add support for SRV records in the servers configuration (#85) + +***Security*** + +* SSLv2 and SSLv3 are now explicitly disabled in Log Courier and the logstash +courier plugins to further enhance security when using the TLS transport. + ## 1.3 -*2nd January 2014* +*2nd January 2015* + +***Changes*** * Added support for Go 1.4 * Added new "host" option to override the "host" field in generated events @@ -43,6 +105,11 @@ events and add regression test correctly * Various other minor tweaks and fixes +***Known Issues*** + +* The Logstash courier output plugin triggers a NameError. This issue is fixed +in the following version. No workaround is available. + ## 1.2 *1st December 2014* diff --git a/docs/CommandLineArguments.md b/docs/CommandLineArguments.md index 3283e7da..a65575ba 100644 --- a/docs/CommandLineArguments.md +++ b/docs/CommandLineArguments.md @@ -2,14 +2,15 @@ -**Table of Contents** *generated with [DocToc](http://doctoc.herokuapp.com/)* +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - [Overview](#overview) -- [`-config=`](#-config=path) +- [`-config=`](#-configpath) - [`-config-test`](#-config-test) -- [`-cpuprofile=`](#-cpuprofile=path) +- [`-cpuprofile=`](#-cpuprofilepath) - [`-from-beginning`](#-from-beginning) - [`-list-supported`](#-list-supported) +- [`-stdin`](#-stdin) - [`-version`](#-version) @@ -60,6 +61,12 @@ discovered log files will start from the begining, regardless of this flag. Print a list of available transports and codecs provided by this build of Log Courier, then exit. +## `-stdin` + +Read log data from stdin and ignore files declaractions in the configuration +file. The fields and codec can be configured in the configuration file under +the `"stdin"` section. + ## `-version` Print the version of this build of Log Courier, then exit. diff --git a/docs/Configuration.md b/docs/Configuration.md index 000cf660..37fb2789 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -2,7 +2,7 @@ -**Table of Contents** *generated with [DocToc](http://doctoc.herokuapp.com/)* +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - [Overview](#overview) - [Reloading](#reloading) @@ -11,6 +11,10 @@ - [String, Number, Boolean, Array, Dictionary](#string-number-boolean-array-dictionary) - [Duration](#duration) - [Fileglob](#fileglob) +- [Stream Configuration](#stream-configuration) + - [`"codec"`](#codec) + - [`"dead time"`](#dead-time) + - [`"fields"`](#fields) - [`"general"`](#general) - [`"admin enabled"`](#admin-enabled) - [`"admin listen address"`](#admin-listen-address) @@ -32,6 +36,8 @@ - [`"curve secret key"`](#curve-secret-key) - [`"max pending payloads"`](#max-pending-payloads) - [`"reconnect"`](#reconnect) + - [`"rfc 2782 srv"`](#rfc-2782-srv) + - [`"rfc 2782 service"`](#rfc-2782-service) - [`"servers"`](#servers) - [`"ssl ca"`](#ssl-ca) - [`"ssl certificate"`](#ssl-certificate) @@ -39,11 +45,9 @@ - [`"timeout"`](#timeout) - [`"transport"`](#transport) - [`"files"`](#files) - - [`"codec"`](#codec) - - [`"dead time"`](#dead-time) - - [`"fields"`](#fields) - [`"paths"`](#paths) - [`"includes"`](#includes) +- [`"stdin"`](#stdin) @@ -80,11 +84,18 @@ command replacing 1234 with the Process ID of Log Courier. kill -HUP 1234 +Log Courier will reopen its own log file if one has been configured, allowing +native log rotation to take place. + Please note that files Log Courier has already started harvesting will continue -to harvest after the reload with their previous configuration. The reload -process will only affect new files and the network configuration. In the case of -a network configuration change, Log Courier will disconnect and reconnect at the -earliest opportunity. +to be harvested after the reload with their original configuration; the reload +process will only affect new files. Additionally, harvested log files will not +be reopened. Log rotations are detected automatically. To control when a +harvested log file is closed you can adjust the [`"dead time"`](#dead-time) +option. + +In the case of a network configuration change, Log Courier will disconnect and +reconnect at the earliest opportunity. *Configuration reload is not currently available on Windows builds of Log Courier.* @@ -148,6 +159,64 @@ character-range: * `"/var/log/httpd/access.log"` * `"/var/log/httpd/access.log.[0-9]"` +## Stream Configuration + +Stream Configuration parameters can be specified for file groups within +[`"files"`](#files) and also for [`"stdin"`](#stdin). They customise the log +entries produced by passing, for example, by passing them through a codec and +adding extra fields. + +### `"codec"` + +*Codec configuration. Optional. Default: `{ "name": "plain" }`* +*Configuration reload will only affect new or resumed files* + +*Depending on how log-courier was built, some codecs may not be available. Run +`log-courier -list-supported` to see the list of codecs available in a specific +build of log-courier.* + +The specified codec will receive the lines read from the log stream and perform +any decoding necessary to generate events. The plain codec does nothing and +simply ships the events unchanged. + +All configurations are a dictionary with at least a "name" key. Additional +options can be provided if the specified codec allows. + +{ "name": "codec-name" } +{ "name": "codec-name", "option1": "value", "option2": "42" } + +Aside from "plain", the following codecs are available at this time. + +* [Filter](codecs/Filter.md) +* [Multiline](codecs/Multiline.md) + +### `"dead time"` + +*Duration. Optional. Default: "24h"* +*Configuration reload will only affect new or resumed files* + +If a log file has not been modified in this time period, it will be closed and +Log Courier will simply watch it for modifications. If the file is modified it +will be reopened. + +If a log file that is being harvested is deleted, it will remain on disk until +Log Courier closes it. Therefore it is important to keep this value sensible to +ensure old log files are not kept open preventing deletion. + +### `"fields"` + +*Dictionary. Optional* +*Configuration reload will only affect new or resumed files* + +Extra fields to attach the event prior to shipping. These can be simple strings, +numbers or even arrays and dictionaries. + +Examples: + +* `{ "type": "syslog" }` +* `{ "type": "apache", "server_names": [ "example.com", "www.example.com" ] }` +* `{ "type": "program", "program": { "exec": "program.py", "args": [ "--run", "--daemon" ] } }` + ## `"general"` The general configuration affects the general behaviour of Log Courier, such @@ -201,7 +270,7 @@ instead of the system FQDN. Available values: "critical", "error", "warning", "notice", "info", "debug"* *Requires restart* -The maximum level of detail to produce in Log Courier's internal log. +The minimum level of detail to produce in Log Courier's internal log. ### `"log stdout"` @@ -351,13 +420,37 @@ this slows down the rate of reconnection attempts. When using the ZMQ transport, this is how long to wait before restarting the ZMQ stack when it was reset. +### `"rfc 2782 srv"` + +*Boolean. Optional. Default: true* + +When performing SRV DNS lookups for entries in the [`"servers"`](#servers) list, +use RFC 2782 style lookups of the form `_service._proto.example.com`. + +### `"rfc 2782 service"` + +*String. Optional. Default: "courier"* + +Specifies the service to request when using RFC 2782 style SRV lookups. Using +the default, "courier", an "@example.com" server entry would result in a lookup +for `_courier._tcp.example.com`. + ### `"servers"` *Array of Strings. Required* -Sets the list of servers to send logs to. DNS names are resolved into IP -addresses each time connections are made and all available IP addresses are -used. +Sets the list of servers to send logs to. Accepted formats for each server entry +are: + +* `ipaddress:port` +* `hostname:port` (A DNS lookup is performed) +* `@hostname` (A SRV DNS lookup is performed, with further DNS lookups if +required) + +The initial server is randomly selected. Subsequent connection attempts are made +to the next IP address available (if the server had multiple IP addresses) or to +the next server listed in the configuration file (if all addresses for the +previous server were exausted.) ### `"ssl ca"` @@ -418,9 +511,8 @@ option. ## `"files"` -The file configuration lists the file groups that contain the logs you wish to -ship. It is an array of file group configurations. A minimum of one file group -configuration must be specified. +The files configuration lists the file groups that contain the logs you wish to +ship. It is an array of file group configurations. ``` [ @@ -433,66 +525,27 @@ configuration must be specified. ] ``` -### `"codec"` - -*Codec configuration. Optional. Default: `{ "name": "plain" }`* -*Configuration reload will only affect new or resumed files* - -*Depending on how log-courier was built, some codecs may not be available. Run -`log-courier -list-supported` to see the list of codecs available in a specific -build of log-courier.* - -The specified codec will receive the lines read from the log stream and perform -any decoding necessary to generate events. The plain codec does nothing and -simply ships the events unchanged. - -All configurations are a dictionary with at least a "name" key. Additional -options can be provided if the specified codec allows. - - { "name": "codec-name" } - { "name": "codec-name", "option1": "value", "option2": "42" } - -Aside from "plain", the following codecs are available at this time. - -* [Filter](codecs/Filter.md) -* [Multiline](codecs/Multiline.md) - -### `"dead time"` - -*Duration. Optional. Default: "24h"* -*Configuration reload will only affect new or resumed files* - -If a log file has not been modified in this time period, it will be closed and -Log Courier will simply watch it for modifications. If the file is modified it -will be reopened. - -If a log file that is being harvested is deleted, it will remain on disk until -Log Courier closes it. Therefore it is important to keep this value sensible to -ensure old log files are not kept open preventing deletion. - -### `"fields"` - -*Dictionary. Optional* -*Configuration reload will only affect new or resumed files* - -Extra fields to attach the event prior to shipping. These can be simple strings, -numbers or even arrays and dictionaries. - -Examples: - -* `{ "type": "syslog" }` -* `{ "type": "apache", "server_names": [ "example.com", "www.example.com" ] }` -* `{ "type": "program", "program": { "exec": "program.py", "args": [ "--run", "--daemon" ] } }` +In addition to the configuration parameters specified below, each file group may +also have [Stream Configuration](#streamconfiguration) parameters specified. ### `"paths"` *Array of Fileglobs. Required* At least one Fileglob must be specified and all matching files for all provided -globs will be tailed. +globs will be monitored. + +If the log file is rotated, Log Courier will detect this and automatically start +harvesting the new file. It will also keep the old file open to catch any +delayed writes that a still-reloading application has not yet written. You can +configure the time period before this old log file is closed using the +[`"dead time"`](#dead-time) option. See above for a description of the Fileglob field type. +*To read from stdin, see the [`-stdin`](CommandLineArguments.md#stdin) command +line argument.* + Examples: * `[ "/var/log/*.log" ]` @@ -513,5 +566,12 @@ following. [ { "paths": [ "/var/log/httpd/access.log" ], - "fields": [ "type": "access_log" ] + "fields": { "type": "access_log" } } ] + +## `"stdin"` + +The stdin configuration contains the +[Stream Configuration](#streamconfiguration) parameters that should be used when +Log Courier is set to read log data from stdin using the +[`-stdin`](CommandLineArguments.md#stdin) command line entry. diff --git a/docs/LogstashIntegration.md b/docs/LogstashIntegration.md index f2df4e84..1e25e05a 100644 --- a/docs/LogstashIntegration.md +++ b/docs/LogstashIntegration.md @@ -5,7 +5,7 @@ Log Courier is built to work seamlessly with [Logstash](http://logstash.net) -**Table of Contents** *generated with [DocToc](http://doctoc.herokuapp.com/)* +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - [Installation](#installation) - [Logstash 1.5+ Plugin Manager](#logstash-15-plugin-manager) @@ -50,7 +50,7 @@ which will require an internet connection. cd /path/to/logstash export GEM_HOME=vendor/bundle/jruby/1.9 - java -jar vendor/jar/jruby-complete-1.7.11.jar -S gem install /path/to/log-courier-X.X.gem + java -jar vendor/jar/jruby-complete-1.7.11.jar -S gem install /path/to/the.gem The remaining step is to manually install the Logstash plugins. @@ -60,13 +60,13 @@ The remaining step is to manually install the Logstash plugins. ### Local-only Installation If you need to install the gem and plugins on a server without an internet -connection, you can download the latest ffi-rzmq-core and ffi-zmq gems from the -rubygems site, transfer them across, and install them yourself. Once they are -installed, follow the instructions for Manual Installation and the process can -be completed without an internet connection. +connection, you can download the gem dependencies from the rubygems site and +transfer them across. Follow the instructions for Manual Installation and +install the dependency gems before the Log Courier gem. * https://rubygems.org/gems/ffi-rzmq-core * https://rubygems.org/gems/ffi-rzmq +* https://rubygems.org/gems/multi_json ## Configuration diff --git a/docs/codecs/Filter.md b/docs/codecs/Filter.md index 111dac8b..b46b165a 100644 --- a/docs/codecs/Filter.md +++ b/docs/codecs/Filter.md @@ -4,7 +4,7 @@ The filter codec strips out unwanted events, shipping only those desired. -**Table of Contents** *generated with [DocToc](http://doctoc.herokuapp.com/)* +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - [Example](#example) - [Options](#options) diff --git a/docs/codecs/Multiline.md b/docs/codecs/Multiline.md index 93504deb..5a082dd0 100644 --- a/docs/codecs/Multiline.md +++ b/docs/codecs/Multiline.md @@ -8,7 +8,7 @@ option. -**Table of Contents** *generated with [DocToc](http://doctoc.herokuapp.com/)* +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - [Example](#example) - [Options](#options) diff --git a/docs/examples/example-stdin.conf b/docs/examples/example-stdin.conf index 6b23dc6a..e7832d3e 100644 --- a/docs/examples/example-stdin.conf +++ b/docs/examples/example-stdin.conf @@ -3,10 +3,7 @@ "servers": [ "localhost:5043" ], "ssl ca": "./logstash.cer" }, - "files": [ - { - "paths": [ "-" ], - "fields": { "type": "stdin" } - } - ] + "stdin": { + "fields": { "type": "stdin" } + } } diff --git a/lib/log-courier/client.rb b/lib/log-courier/client.rb index b77eb1a8..16740c88 100644 --- a/lib/log-courier/client.rb +++ b/lib/log-courier/client.rb @@ -25,6 +25,7 @@ class NativeException; end module LogCourier + class TimeoutError < StandardError; end class ShutdownSignal < StandardError; end class ProtocolError < StandardError; end @@ -92,15 +93,26 @@ class Client def initialize(options = {}) @options = { logger: nil, + transport: 'tls', spool_size: 1024, - idle_timeout: 5 + idle_timeout: 5, + port: nil, + addresses: [], }.merge!(options) @logger = @options[:logger] - @logger['plugin'] = 'output/courier' + @logger['plugin'] = 'output/courier' unless @logger.nil? - require 'log-courier/client_tls' - @client = ClientTls.new(@options) + case @options[:transport] + when 'tcp', 'tls' + require 'log-courier/client_tcp' + @client = ClientTcp.new(@options) + else + fail 'output/courier: \'transport\' must be tcp or tls' + end + + fail 'output/courier: \'addresses\' must contain at least one address' if @options[:addresses].empty? + fail 'output/courier: \'addresses\' only supports a single address at this time' if @options[:addresses].length > 1 @event_queue = EventQueue.new @options[:spool_size] @pending_payloads = {} @@ -115,6 +127,16 @@ def initialize(options = {}) run_spooler end + # TODO: Make these configurable? + @keepalive_timeout = 1800 + @network_timeout = 30 + + # TODO: Make pending payload max configurable? + @max_pending_payloads = 100 + + @retry_payload = nil + @received_payloads = Queue.new + @pending_ping = false # Start the IO thread @@ -130,11 +152,17 @@ def publish(event) return end - def shutdown - # Raise a shutdown signal in the spooler and wait for it - @spooler_thread.raise ShutdownSignal - @io_thread.raise ShutdownSignal - @spooler_thread.join + def shutdown(force=false) + if force + # Raise a shutdown signal in the spooler and wait for it + @spooler_thread.raise ShutdownSignal + @spooler_thread.join + @io_thread.raise ShutdownSignal + else + @event_queue.push nil + @spooler_thread.join + @io_control << ['!', nil] + end @io_thread.join return @pending_payloads.length == 0 end @@ -150,7 +178,13 @@ def run_spooler begin loop do event = @event_queue.pop next_flush - Time.now.to_i + + if event.nil? + raise ShutdownSignal + end + spooled.push(event) + break if spooled.length >= @options[:spool_size] end rescue TimeoutError @@ -158,12 +192,19 @@ def run_spooler next if spooled.length == 0 end + if spooled.length >= @options[:spool_size] + @logger.debug 'Flushing full spool', :events => spooled.length unless @logger.nil? + else + @logger.debug 'Flushing spool due to timeout', :events => spooled.length unless @logger.nil? + end + # Pass through to io_control but only if we're ready to send @send_mutex.synchronize do - @send_cond.wait(@send_mutex) unless @send_ready + @send_cond.wait(@send_mutex) until @send_ready @send_ready = false - @io_control << ['E', spooled] end + + @io_control << ['E', spooled] end return rescue ShutdownSignal @@ -171,118 +212,17 @@ def run_spooler end def run_io - # TODO: Make keepalive configurable? - @keepalive_timeout = 1800 - - # TODO: Make pending payload max configurable? - max_pending_payloads = 100 - - retry_payload = nil - - can_send = true - loop do # Reconnect loop @client.connect @io_control - reset_keepalive + @timeout = Time.now.to_i + @keepalive_timeout - # Capture send exceptions - begin - # IO loop - loop do - catch :keepalive do - begin - action = @io_control.pop @keepalive_next - Time.now.to_i - - # Process the action - case action[0] - when 'S' - # If we're flushing through the pending, pick from there - unless retry_payload.nil? - # Regenerate data if we need to - retry_payload.data = buffer_jdat_data(retry_payload.events, retry_payload.nonce) if retry_payload.data == nil - - # Send and move onto next - @client.send 'JDAT', retry_payload.data - - retry_payload = retry_payload.next - throw :keepalive - end - - # Ready to send, allow spooler to pass us something - @send_mutex.synchronize do - @send_ready = true - @send_cond.signal - end - - can_send = true - when 'E' - # If we have too many pending payloads, pause the IO - if @pending_payloads.length + 1 >= max_pending_payloads - @client.pause_send - end - - # Received some events - send them - send_jdat action[1] - - # The send action will trigger another "S" if we have more send buffer - can_send = false - when 'R' - # Received a message - signature, message = action[1..2] - case signature - when 'PONG' - process_pong message - when 'ACKN' - process_ackn message - else - # Unknown message - only listener is allowed to respond with a "????" message - # TODO: What should we do? Just ignore for now and let timeouts conquer - end - when 'F' - # Reconnect, an error occurred - break - end - rescue TimeoutError - # Keepalive timeout hit, send a PING unless we were awaiting a PONG - if @pending_ping - # Timed out, break into reconnect - fail TimeoutError - end - - # Is send full? can_send will be false if so - # We should've started receiving ACK by now so time out - fail TimeoutError unless can_send - - # Send PING - send_ping - - # We may have filled send buffer - can_send = false - end - end - - # Reset keepalive timeout - reset_keepalive - end - rescue ProtocolError => e - # Reconnect required due to a protocol error - @logger.warn 'Protocol error', :error => e.message unless @logger.nil? - rescue TimeoutError - # Reconnect due to timeout - @logger.warn 'Timeout occurred' unless @logger.nil? - rescue ShutdownSignal - # Shutdown, break out - break - rescue StandardError, NativeException => e - # Unknown error occurred - @logger.warn e, :hint => 'Unknown error' unless @logger.nil? - end + run_io_loop # Disconnect and retry payloads @client.disconnect - retry_payload = @first_payload + @retry_payload = @first_payload # TODO: Make reconnect time configurable? sleep 5 @@ -290,11 +230,164 @@ def run_io @client.disconnect return + rescue ShutdownSignal + # Ensure disconnected + @client.disconnect end - def reset_keepalive - @keepalive_next = Time.now.to_i + @keepalive_timeout - return + def run_io_loop() + io_stop = false + can_send = false + + # IO loop + loop do + begin + action = @io_control.pop @timeout - Time.now.to_i + + # Process the action + case action[0] + when 'S' + # If we're flushing through the pending, pick from there + unless @retry_payload.nil? + @logger.debug 'Send is ready, retrying previous payload' unless @logger.nil? + + # Regenerate data if we need to + @retry_payload.data = buffer_jdat_data(@retry_payload.events, @retry_payload.nonce) if @retry_payload.data == nil + + # Send and move onto next + @client.send 'JDAT', @retry_payload.data + + @retry_payload = @retry_payload.next + + # If first send, exit idle mode + if @retry_payload == @first_payload + @timeout = Time.now.to_i + @network_timeout + end + break + end + + # Ready to send, allow spooler to pass us something if we don't + # have something already + if @received_payloads.length != 0 + @logger.debug 'Send is ready, using events from backlog' unless @logger.nil? + send_payload @received_payloads.pop() + else + @logger.debug 'Send is ready, requesting events' unless @logger.nil? + + can_send = true + + @send_mutex.synchronize do + @send_ready = true + @send_cond.signal + end + end + when 'E' + # Were we expecting a payload? Store it if not + if can_send + @logger.debug 'Sending events', :events => action[1].length unless @logger.nil? + send_payload action[1] + can_send = false + else + @logger.debug 'Events received when not ready; saved to backlog' unless @logger.nil? + @received_payloads.push action[1] + end + when 'R' + # Received a message + signature, message = action[1..2] + case signature + when 'PONG' + process_pong message + when 'ACKN' + process_ackn message + else + # Unknown message - only listener is allowed to respond with a "????" message + # TODO: What should we do? Just ignore for now and let timeouts conquer + end + + # Any pending payloads left? + if @pending_payloads.length == 0 + # Handle shutdown + if io_stop + raise ShutdownSignal + end + + # Enter idle mode + @timeout = Time.now.to_i + @keepalive_timeout + else + # Set network timeout + @timeout = Time.now.to_i + @network_timeout + end + when 'F' + # Reconnect, an error occurred + break + when '!' + @logger.debug 'Shutdown request received' unless @logger.nil? + + # Shutdown request received + if @pending_payloads.length == 0 + raise ShutdownSignal + end + + @logger.debug 'Delaying shutdown due to pending payloads', :payloads => @pending_payloads.length unless @logger.nil? + + io_stop = true + + # Stop spooler sending + can_send = false + @send_mutex.synchronize do + @send_ready = false + end + end + rescue TimeoutError + if @pending_payloads != 0 + # Network timeout + fail TimeoutError + end + + # Keepalive timeout hit, send a PING unless we were awaiting a PONG + if @pending_ping + # Timed out, break into reconnect + fail TimeoutError + end + + # Stop spooler sending + can_send = false + @send_mutex.synchronize do + @send_ready = false + end + + # Send PING + send_ping + + @timeout = Time.now.to_i + @network_timeout + end + end + rescue ProtocolError => e + # Reconnect required due to a protocol error + @logger.warn 'Protocol error', :error => e.message unless @logger.nil? + rescue TimeoutError + # Reconnect due to timeout + @logger.warn 'Timeout occurred' unless @logger.nil? + rescue ShutdownSignal => e + raise + rescue StandardError, NativeException => e + # Unknown error occurred + @logger.warn e, :hint => 'Unknown error' unless @logger.nil? + end + + def send_payload(payload) + # If we have too many pending payloads, pause the IO + if @pending_payloads.length + 1 >= @max_pending_payloads + @client.pause_send + end + + # Received some events - send them + send_jdat payload + + # Leave idle mode if this is the first payload after idle + if @pending_payloads.length == 1 + @timeout = Time.now.to_i + @network_timeout + end end def generate_nonce diff --git a/lib/log-courier/client_tls.rb b/lib/log-courier/client_tcp.rb similarity index 70% rename from lib/log-courier/client_tls.rb rename to lib/log-courier/client_tcp.rb index dec33e85..5a5860f8 100644 --- a/lib/log-courier/client_tls.rb +++ b/lib/log-courier/client_tcp.rb @@ -23,16 +23,15 @@ module LogCourier # TLS transport implementation - class ClientTls + class ClientTcp def initialize(options = {}) @options = { logger: nil, - port: nil, - addresses: [], + transport: 'tls', ssl_ca: nil, ssl_certificate: nil, ssl_key: nil, - ssl_key_passphrase: nil + ssl_key_passphrase: nil, }.merge!(options) @logger = @options[:logger] @@ -41,14 +40,13 @@ def initialize(options = {}) fail "output/courier: '#{k}' is required" if @options[k].nil? end - fail 'output/courier: \'addresses\' must contain at least one address' if @options[:addresses].empty? - - c = 0 - [:ssl_certificate, :ssl_key].each do - c += 1 + if @options[:transport] == 'tls' + c = 0 + [:ssl_certificate, :ssl_key].each do + c += 1 + end + fail 'output/courier: \'ssl_certificate\' and \'ssl_key\' must be specified together' if c == 1 end - - fail 'output/courier: \'ssl_certificate\' and \'ssl_key\' must be specified together' if c == 1 end def connect(io_control) @@ -139,14 +137,18 @@ def run_send(io_control) end end return - rescue OpenSSL::SSL::SSLError, IOError, Errno::ECONNRESET => e + rescue OpenSSL::SSL::SSLError => e @logger.warn 'SSL write error', :error => e.message unless @logger.nil? io_control << ['F'] return + rescue IOError, Errno::ECONNRESET => e + @logger.warn 'Write error', :error => e.message unless @logger.nil? + io_control << ['F'] + return rescue ShutdownSignal return rescue StandardError, NativeException => e - @logger.warn e, :hint => 'Unknown SSL write error' unless @logger.nil? + @logger.warn e, :hint => 'Unknown write error' unless @logger.nil? io_control << ['F'] return end @@ -174,10 +176,14 @@ def run_recv(io_control) io_control << ['R', signature, message] end return - rescue OpenSSL::SSL::SSLError, IOError, Errno::ECONNRESET => e + rescue OpenSSL::SSL::SSLError => e @logger.warn 'SSL read error', :error => e.message unless @logger.nil? io_control << ['F'] return + rescue IOError, Errno::ECONNRESET => e + @logger.warn 'Read error', :error => e.message unless @logger.nil? + io_control << ['F'] + return rescue EOFError @logger.warn 'Connection closed by server' unless @logger.nil? io_control << ['F'] @@ -185,7 +191,7 @@ def run_recv(io_control) rescue ShutdownSignal return rescue => e - @logger.warn e, :hint => 'Unknown SSL read error' unless @logger.nil? + @logger.warn e, :hint => 'Unknown read error' unless @logger.nil? io_control << ['F'] return end @@ -200,24 +206,37 @@ def tls_connect begin tcp_socket = TCPSocket.new(address, port) - ssl = OpenSSL::SSL::SSLContext.new - - unless @options[:ssl_certificate].nil? - ssl.cert = OpenSSL::X509::Certificate.new(File.read(@options[:ssl_certificate])) - ssl.key = OpenSSL::PKey::RSA.new(File.read(@options[:ssl_key]), @options[:ssl_key_passphrase]) - end + if @options[:transport] == 'tls' + ssl = OpenSSL::SSL::SSLContext.new + + # Disable SSLv2 and SSLv3 + # Call set_params first to ensure options attribute is there (hmmmm?) + ssl.set_params + # Modify the default options to ensure SSLv2 and SSLv3 is disabled + # This retains any beneficial options set by default in the current Ruby implementation + ssl.options |= OpenSSL::SSL::OP_NO_SSLv2 if defined?(OpenSSL::SSL::OP_NO_SSLv2) + ssl.options |= OpenSSL::SSL::OP_NO_SSLv3 if defined?(OpenSSL::SSL::OP_NO_SSLv3) + + # Set the certificate file + unless @options[:ssl_certificate].nil? + ssl.cert = OpenSSL::X509::Certificate.new(File.read(@options[:ssl_certificate])) + ssl.key = OpenSSL::PKey::RSA.new(File.read(@options[:ssl_key]), @options[:ssl_key_passphrase]) + end - cert_store = OpenSSL::X509::Store.new - cert_store.add_file(@options[:ssl_ca]) - ssl.cert_store = cert_store - ssl.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT + cert_store = OpenSSL::X509::Store.new + cert_store.add_file(@options[:ssl_ca]) + ssl.cert_store = cert_store + ssl.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT - @ssl_client = OpenSSL::SSL::SSLSocket.new(tcp_socket) + @ssl_client = OpenSSL::SSL::SSLSocket.new(tcp_socket) - socket = @ssl_client.connect + socket = @ssl_client.connect - # Verify certificate - socket.post_connection_check(address) + # Verify certificate + socket.post_connection_check(address) + else + socket = tcp_socket.connect + end # Add extra logging data now we're connected @logger['address'] = address diff --git a/lib/log-courier/server.rb b/lib/log-courier/server.rb index 30d8312c..d5d06de7 100644 --- a/lib/log-courier/server.rb +++ b/lib/log-courier/server.rb @@ -36,11 +36,11 @@ class Server def initialize(options = {}) @options = { logger: nil, - transport: 'tls' + transport: 'tls', }.merge!(options) @logger = @options[:logger] - @logger['plugin'] = 'input/courier' + @logger['plugin'] = 'input/courier' unless @logger.nil? case @options[:transport] when 'tcp', 'tls' @@ -59,7 +59,7 @@ def initialize(options = {}) # Load the json adapter @json_adapter = MultiJson.adapter.instance - @json_options = { raw: true, use_bigdecimal: true } + @json_options = { raw: true } end def run(&block) diff --git a/lib/log-courier/server_tcp.rb b/lib/log-courier/server_tcp.rb index 8ef264c3..e17a80b1 100644 --- a/lib/log-courier/server_tcp.rb +++ b/lib/log-courier/server_tcp.rb @@ -87,6 +87,16 @@ def initialize(options = {}) if @options[:transport] == 'tls' ssl = OpenSSL::SSL::SSLContext.new + + # Disable SSLv2 and SSLv3 + # Call set_params first to ensure options attribute is there (hmmmm?) + ssl.set_params + # Modify the default options to ensure SSLv2 and SSLv3 is disabled + # This retains any beneficial options set by default in the current Ruby implementation + ssl.options |= OpenSSL::SSL::OP_NO_SSLv2 if defined?(OpenSSL::SSL::OP_NO_SSLv2) + ssl.options |= OpenSSL::SSL::OP_NO_SSLv3 if defined?(OpenSSL::SSL::OP_NO_SSLv3) + + # Set the certificate file ssl.cert = OpenSSL::X509::Certificate.new(File.read(@options[:ssl_certificate])) ssl.key = OpenSSL::PKey::RSA.new(File.read(@options[:ssl_key]), @options[:ssl_key_passphrase]) @@ -134,7 +144,7 @@ def run(&block) client = @server.accept rescue EOFError, OpenSSL::SSL::SSLError, IOError => e # Accept failure or other issue - @logger.warn 'Connection failed to accept', :error => e.message, :peer => @tcp_server.peer unless @logger.nil + @logger.warn 'Connection failed to accept', :error => e.message, :peer => @tcp_server.peer unless @logger.nil? client.close rescue nil unless client.nil? next end @@ -254,10 +264,14 @@ def run @logger.info 'Connection closed', :peer => @peer unless @logger.nil? end return - rescue OpenSSL::SSL::SSLError, IOError, Errno::ECONNRESET => e + rescue OpenSSL::SSL::SSLError => e # Read errors, only action is to shutdown which we'll do in ensure @logger.warn 'SSL error, connection aborted', :error => e.message, :peer => @peer unless @logger.nil? return + rescue IOError, Errno::ECONNRESET => e + # Read errors, only action is to shutdown which we'll do in ensure + @logger.warn 'Connection aborted', :error => e.message, :peer => @peer unless @logger.nil? + return rescue ProtocolError => e # Connection abort request due to a protocol error @logger.warn 'Protocol error, connection aborted', :error => e.message, :peer => @peer unless @logger.nil? diff --git a/lib/log-courier/server_zmq.rb b/lib/log-courier/server_zmq.rb index a0540a20..363a4a71 100644 --- a/lib/log-courier/server_zmq.rb +++ b/lib/log-courier/server_zmq.rb @@ -85,7 +85,7 @@ def initialize(options = {}) bind = 'tcp://' + @options[:address] + (@options[:port] == 0 ? ':*' : ':' + @options[:port].to_s) rc = @socket.bind(bind) - fail 'failed to bind at ' + bind + ': ' + rZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc) + fail 'failed to bind at ' + bind + ': ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc) # Lookup port number that was allocated in case it was set to 0 endpoint = '' @@ -273,8 +273,12 @@ def deliver(source, data, &block) } end - # Existing thread, throw on the queue, if not enough room drop the message - index['']['client'].push data, 0 + # Existing thread, throw on the queue, if not enough room (timeout) drop the message + begin + index['']['client'].push data, 0 + rescue LogCourier::TimeoutError + # TODO: Log a warning about this? + end end return end diff --git a/lib/logstash/inputs/courier.rb b/lib/logstash/inputs/courier.rb index 57d35c83..0efc41c7 100644 --- a/lib/logstash/inputs/courier.rb +++ b/lib/logstash/inputs/courier.rb @@ -79,7 +79,7 @@ class Courier < LogStash::Inputs::Base public def register - @logger.info('Starting courier input listener', :address => "#{@host}:#{@port}") + @logger.info 'Starting courier input listener', :address => "#{@host}:#{@port}" options = { logger: @logger, @@ -108,6 +108,7 @@ def register def run(output_queue) @log_courier.run do |event| + event['tags'] = [event['tags']] if event.has_key?('tags') && !event['tags'].is_a?(Array) event = LogStash::Event.new(event) decorate event output_queue << event diff --git a/lib/logstash/outputs/courier.rb b/lib/logstash/outputs/courier.rb index b72fcb7c..a87b1d2f 100644 --- a/lib/logstash/outputs/courier.rb +++ b/lib/logstash/outputs/courier.rb @@ -20,7 +20,7 @@ module LogStash module Outputs # Send events using the Log Courier protocol - class LogCourier < LogStash::Outputs::Base + class Courier < LogStash::Outputs::Base config_name 'courier' milestone 1 @@ -51,9 +51,10 @@ class LogCourier < LogStash::Outputs::Base public def register - require 'log-courier/client' + @logger.info 'Starting courier output' - @client = LogCourier::Client.new( + options = { + logger: @logger, addresses: @hosts, port: @port, ssl_ca: @ssl_ca, @@ -61,8 +62,11 @@ def register ssl_key: @ssl_key, ssl_key_passphrase: @ssl_key_passphrase, spool_size: @spool_size, - idle_timeout: @idle_timeout - ) + idle_timeout: @idle_timeout, + } + + require 'log-courier/client' + @client = LogCourier::Client.new(options) end public diff --git a/log-courier.gemspec b/log-courier.gemspec.tmpl similarity index 92% rename from log-courier.gemspec rename to log-courier.gemspec.tmpl index 242d474a..a081390b 100644 --- a/log-courier.gemspec +++ b/log-courier.gemspec.tmpl @@ -1,6 +1,6 @@ Gem::Specification.new do |gem| gem.name = 'log-courier' - gem.version = '1.3' + gem.version = '' gem.description = 'Log Courier library' gem.summary = 'Receive events from Log Courier and transmit between LogStash instances' gem.homepage = 'https://github.com/driskell/log-courier' @@ -11,7 +11,7 @@ Gem::Specification.new do |gem| gem.require_paths = ['lib'] gem.files = %w( lib/log-courier/client.rb - lib/log-courier/client_tls.rb + lib/log-courier/client_tcp.rb lib/log-courier/event_queue.rb lib/log-courier/server.rb lib/log-courier/server_tcp.rb diff --git a/logstash-input-log-courier.gemspec b/logstash-input-log-courier.gemspec.tmpl similarity index 88% rename from logstash-input-log-courier.gemspec rename to logstash-input-log-courier.gemspec.tmpl index 638af4a5..08cb77e8 100644 --- a/logstash-input-log-courier.gemspec +++ b/logstash-input-log-courier.gemspec.tmpl @@ -1,6 +1,6 @@ Gem::Specification.new do |gem| gem.name = 'logstash-input-log-courier' - gem.version = '1.3' + gem.version = '' gem.description = 'Log Courier Input Logstash Plugin' gem.summary = 'Receive events from Log Courier and Logstash using the Log Courier protocol' gem.homepage = 'https://github.com/driskell/log-courier' @@ -16,5 +16,5 @@ Gem::Specification.new do |gem| gem.metadata = { 'logstash_plugin' => 'true', 'group' => 'input' } gem.add_runtime_dependency 'logstash', '~> 1.4' - gem.add_runtime_dependency 'log-courier', '= 1.3' + gem.add_runtime_dependency 'log-courier', '= ' end diff --git a/logstash-output-log-courier.gemspec b/logstash-output-log-courier.gemspec.tmpl similarity index 88% rename from logstash-output-log-courier.gemspec rename to logstash-output-log-courier.gemspec.tmpl index 3b68ee77..c8f6c5b2 100644 --- a/logstash-output-log-courier.gemspec +++ b/logstash-output-log-courier.gemspec.tmpl @@ -1,6 +1,6 @@ Gem::Specification.new do |gem| gem.name = 'logstash-output-log-courier' - gem.version = '1.3' + gem.version = '' gem.description = 'Log Courier Output Logstash Plugin' gem.summary = 'Transmit events from one Logstash instance to another using the Log Courier protocol' gem.homepage = 'https://github.com/driskell/log-courier' @@ -16,5 +16,5 @@ Gem::Specification.new do |gem| gem.metadata = { 'logstash_plugin' => 'true', 'group' => 'input' } gem.add_runtime_dependency 'logstash', '~> 1.4' - gem.add_runtime_dependency 'log-courier', '= 1.3' + gem.add_runtime_dependency 'log-courier', '= ' end diff --git a/spec/courier_spec.rb b/spec/courier_spec.rb index 12ebace5..50e497df 100644 --- a/spec/courier_spec.rb +++ b/spec/courier_spec.rb @@ -29,11 +29,9 @@ "ssl ca": "#{@ssl_cert.path}", "servers": [ "localhost:#{server_port}" ] }, - "files": [ - { - "paths": [ "-" ] - } - ] + "stdin": { + "fields": { "type": "stdin" } + } } config @@ -49,8 +47,11 @@ expect(e['message']).to eq "stdin line test #{i}" expect(e['host']).to eq host expect(e['path']).to eq '-' + expect(e['type']).to eq 'stdin' i += 1 end + + stdin_shutdown end it 'should split lines that are too long' do @@ -59,12 +60,7 @@ "network": { "ssl ca": "#{@ssl_cert.path}", "servers": [ "127.0.0.1:#{server_port}" ] - }, - "files": [ - { - "paths": [ "-" ] - } - ] + } } config @@ -90,6 +86,8 @@ expect(e['path']).to eq '-' i += 1 end + + stdin_shutdown end it 'should follow a file from the end' do @@ -496,12 +494,9 @@ "ssl ca": "#{@ssl_cert.path}", "servers": [ "127.0.0.1:#{server_port}" ] }, - "files": [ - { - "paths": [ "-" ], - "fields": { "array": [ 1, 2 ] } - } - ] + "stdin": { + "fields": { "array": [ 1, 2 ] } + } } config @@ -521,6 +516,8 @@ expect(e['path']).to eq '-' i += 1 end + + stdin_shutdown end it 'should allow dictionaries inside field configuration' do @@ -530,12 +527,9 @@ "ssl ca": "#{@ssl_cert.path}", "servers": [ "127.0.0.1:#{server_port}" ] }, - "files": [ - { - "paths": [ "-" ], - "fields": { "dict": { "first": "first", "second": 5 } } - } - ] + "stdin": { + "fields": { "dict": { "first": "first", "second": 5 } } + } } config @@ -555,6 +549,8 @@ expect(e['path']).to eq '-' i += 1 end + + stdin_shutdown end it 'should accept globs of configuration files to include' do diff --git a/spec/lib/helpers/common.rb b/spec/lib/helpers/common.rb index bf93570d..5ae29a27 100644 --- a/spec/lib/helpers/common.rb +++ b/spec/lib/helpers/common.rb @@ -194,7 +194,7 @@ def receive_and_check(args = {}, &block) if block.nil? found = @files.find do |f| next unless f.pending? - f.logged?(event: e, **args) + f.logged?({event: e}.merge!(args)) end expect(found).to_not be_nil, "Event received not recognised: #{e}" else diff --git a/spec/lib/helpers/log-courier.rb b/spec/lib/helpers/log-courier.rb index 12a4f089..0cd88b83 100644 --- a/spec/lib/helpers/log-courier.rb +++ b/spec/lib/helpers/log-courier.rb @@ -74,6 +74,8 @@ def startup(args = {}) _write_config config if args[:stdin] + args[:args] += ' ' if args[:args] + args[:args] += '-stdin=true' @log_courier_mode = 'r+' else @log_courier_mode = 'r' @@ -113,6 +115,21 @@ def _write_config(config) @config.close end + def stdin_shutdown + return unless @log_courier_mode == 'r+' + begin + # If this fails, don't bother closing write again + @log_courier_mode = 'r' + Timeout.timeout(30) do + # Close and wait + @log_courier.close_write + @log_courier_reader.join + end + rescue Timeout::Error + fail "Log-courier did not shutdown on stdin EOF" + end + end + def shutdown puts 'Shutting down Log Courier' return if @log_courier.nil? diff --git a/src/lc-admin/lc-admin.go b/src/lc-admin/lc-admin.go index d0353099..fbce8ebc 100644 --- a/src/lc-admin/lc-admin.go +++ b/src/lc-admin/lc-admin.go @@ -12,401 +12,401 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. -*/ + */ package main import ( - "bufio" - "flag" - "fmt" - "lc-lib/admin" - "lc-lib/core" - "os" - "os/signal" - "strings" - "text/scanner" - "time" + "bufio" + "flag" + "fmt" + "github.com/driskell/log-courier/src/lc-lib/admin" + "github.com/driskell/log-courier/src/lc-lib/core" + "os" + "os/signal" + "strings" + "text/scanner" + "time" ) type CommandError struct { - message string + message string } func (c *CommandError) Error() string { - return c.message + return c.message } var CommandEOF *CommandError = &CommandError{"EOF"} var CommandTooManyArgs *CommandError = &CommandError{"Too many arguments"} type Admin struct { - client *admin.Client - connected bool - quiet bool - admin_connect string - scanner scanner.Scanner - scanner_err error + client *admin.Client + connected bool + quiet bool + admin_connect string + scanner scanner.Scanner + scanner_err error } func NewAdmin(quiet bool, admin_connect string) *Admin { - return &Admin{ - quiet: quiet, - admin_connect: admin_connect, - } + return &Admin{ + quiet: quiet, + admin_connect: admin_connect, + } } func (a *Admin) connect() error { - if !a.connected { - var err error + if !a.connected { + var err error - if !a.quiet { - fmt.Printf("Attempting connection to %s...\n", a.admin_connect) - } + if !a.quiet { + fmt.Printf("Attempting connection to %s...\n", a.admin_connect) + } - if a.client, err = admin.NewClient(a.admin_connect); err != nil { - fmt.Printf("Failed to connect: %s\n", err) - return err - } + if a.client, err = admin.NewClient(a.admin_connect); err != nil { + fmt.Printf("Failed to connect: %s\n", err) + return err + } - if !a.quiet { - fmt.Printf("Connected\n\n") - } + if !a.quiet { + fmt.Printf("Connected\n\n") + } - a.connected = true - } + a.connected = true + } - return nil + return nil } func (a *Admin) ProcessCommand(command string) bool { - var reconnected bool - - for { - if !a.connected { - if err := a.connect(); err != nil { - return false - } - - reconnected = true - } - - var err error - - a.initScanner(command) - if command, err = a.scanIdent(); err != nil { - goto Error - } - - switch command { - case "reload": - if !a.scanEOF() { - err = CommandTooManyArgs - break - } - - err = a.client.Reload() - if err != nil { - break - } - - fmt.Printf("Configuration reload successful\n") - case "status": - var format string - format, err = a.scanIdent() - if err != nil && err != CommandEOF { - break - } - - if !a.scanEOF() { - err = CommandTooManyArgs - break - } - - var snaps *core.Snapshot - snaps, err = a.client.FetchSnapshot() - if err != nil { - break - } - - a.renderSnap(format, snaps) - case "help": - if !a.scanEOF() { - err = CommandTooManyArgs - break - } - - PrintHelp() - default: - err = &CommandError{fmt.Sprintf("Unknown command: %s", command)} - } - - if err == nil { - return true - } - - Error: - if _, ok := err.(*CommandError); ok { - fmt.Printf("Parse error: %s\n", err) - return false - } else if _, ok := err.(*admin.ErrorResponse); ok { - fmt.Printf("Log Courier returned an error: %s\n", err) - return false - } else { - a.connected = false - fmt.Printf("Connection error: %s\n", err) - } - - if reconnected { - break - } - } - - return false + var reconnected bool + + for { + if !a.connected { + if err := a.connect(); err != nil { + return false + } + + reconnected = true + } + + var err error + + a.initScanner(command) + if command, err = a.scanIdent(); err != nil { + goto Error + } + + switch command { + case "reload": + if !a.scanEOF() { + err = CommandTooManyArgs + break + } + + err = a.client.Reload() + if err != nil { + break + } + + fmt.Printf("Configuration reload successful\n") + case "status": + var format string + format, err = a.scanIdent() + if err != nil && err != CommandEOF { + break + } + + if !a.scanEOF() { + err = CommandTooManyArgs + break + } + + var snaps *core.Snapshot + snaps, err = a.client.FetchSnapshot() + if err != nil { + break + } + + a.renderSnap(format, snaps) + case "help": + if !a.scanEOF() { + err = CommandTooManyArgs + break + } + + PrintHelp() + default: + err = &CommandError{fmt.Sprintf("Unknown command: %s", command)} + } + + if err == nil { + return true + } + + Error: + if _, ok := err.(*CommandError); ok { + fmt.Printf("Parse error: %s\n", err) + return false + } else if _, ok := err.(*admin.ErrorResponse); ok { + fmt.Printf("Log Courier returned an error: %s\n", err) + return false + } else { + a.connected = false + fmt.Printf("Connection error: %s\n", err) + } + + if reconnected { + break + } + } + + return false } func (a *Admin) initScanner(command string) { - a.scanner.Init(strings.NewReader(command)) - a.scanner.Mode = scanner.ScanIdents | scanner.ScanInts | scanner.ScanStrings - a.scanner.Whitespace = 1<<' ' + a.scanner.Init(strings.NewReader(command)) + a.scanner.Mode = scanner.ScanIdents | scanner.ScanInts | scanner.ScanStrings + a.scanner.Whitespace = 1 << ' ' - a.scanner.Error = func(s *scanner.Scanner, msg string) { - a.scanner_err = &CommandError{msg} - } + a.scanner.Error = func(s *scanner.Scanner, msg string) { + a.scanner_err = &CommandError{msg} + } } func (a *Admin) scanIdent() (string, error) { - r := a.scanner.Scan() - if a.scanner_err != nil { - return "", a.scanner_err - } - switch r { - case scanner.Ident: - return a.scanner.TokenText(), nil - case scanner.EOF: - return "", CommandEOF - } - return "", &CommandError{"Invalid token"} + r := a.scanner.Scan() + if a.scanner_err != nil { + return "", a.scanner_err + } + switch r { + case scanner.Ident: + return a.scanner.TokenText(), nil + case scanner.EOF: + return "", CommandEOF + } + return "", &CommandError{"Invalid token"} } func (a *Admin) scanEOF() bool { - r := a.scanner.Scan() - if a.scanner_err == nil && r == scanner.EOF { - return true - } - return false + r := a.scanner.Scan() + if a.scanner_err == nil && r == scanner.EOF { + return true + } + return false } func (a *Admin) renderSnap(format string, snap *core.Snapshot) { - switch format { - case "json": - fmt.Printf("{\n") - a.renderSnapJSON("\t", snap) - fmt.Printf("}\n") - default: - a.renderSnapYAML("", snap) - } + switch format { + case "json": + fmt.Printf("{\n") + a.renderSnapJSON("\t", snap) + fmt.Printf("}\n") + default: + a.renderSnapYAML("", snap) + } } func (a *Admin) renderSnapJSON(indent string, snap *core.Snapshot) { - if snap.NumEntries() != 0 { - for i, j := 0, snap.NumEntries(); i < j; i = i+1 { - k, v := snap.Entry(i) - switch t := v.(type) { - case string: - fmt.Printf(indent + "%q: %q", k, t) - case int8, int16, int32, int64, uint8, uint16, uint32, uint64: - fmt.Printf(indent + "%q: %d", k, t) - case float32, float64: - fmt.Printf(indent + "%q: %.2f", k, t) - case time.Time: - fmt.Printf(indent + "%q: %q", k, t.Format("_2 Jan 2006 15.04.05")) - case time.Duration: - fmt.Printf(indent + "%q: %q", k, (t-(t%time.Second)).String()) - default: - fmt.Printf(indent + "%q: %q", k, fmt.Sprintf("%v", t)) - } - if i + 1 < j || snap.NumSubs() != 0 { - fmt.Printf(",\n") - } else { - fmt.Printf("\n") - } - } - } - if snap.NumSubs() != 0 { - for i, j := 0, snap.NumSubs(); i < j; i = i+1 { - sub_snap := snap.Sub(i) - fmt.Printf(indent + "%q: {\n", sub_snap.Description()) - a.renderSnapJSON(indent + "\t", sub_snap) - if i + 1 < j { - fmt.Printf(indent + "},\n") - } else { - fmt.Printf(indent + "}\n") - } - } - } + if snap.NumEntries() != 0 { + for i, j := 0, snap.NumEntries(); i < j; i = i + 1 { + k, v := snap.Entry(i) + switch t := v.(type) { + case string: + fmt.Printf(indent+"%q: %q", k, t) + case int8, int16, int32, int64, uint8, uint16, uint32, uint64: + fmt.Printf(indent+"%q: %d", k, t) + case float32, float64: + fmt.Printf(indent+"%q: %.2f", k, t) + case time.Time: + fmt.Printf(indent+"%q: %q", k, t.Format("_2 Jan 2006 15.04.05")) + case time.Duration: + fmt.Printf(indent+"%q: %q", k, (t - (t % time.Second)).String()) + default: + fmt.Printf(indent+"%q: %q", k, fmt.Sprintf("%v", t)) + } + if i+1 < j || snap.NumSubs() != 0 { + fmt.Printf(",\n") + } else { + fmt.Printf("\n") + } + } + } + if snap.NumSubs() != 0 { + for i, j := 0, snap.NumSubs(); i < j; i = i + 1 { + sub_snap := snap.Sub(i) + fmt.Printf(indent+"%q: {\n", sub_snap.Description()) + a.renderSnapJSON(indent+"\t", sub_snap) + if i+1 < j { + fmt.Printf(indent + "},\n") + } else { + fmt.Printf(indent + "}\n") + } + } + } } func (a *Admin) renderSnapYAML(indent string, snap *core.Snapshot) { - if snap.NumEntries() != 0 { - for i, j := 0, snap.NumEntries(); i < j; i = i+1 { - k, v := snap.Entry(i) - switch t := v.(type) { - case string: - fmt.Printf(indent + "%s: %s\n", k, t) - case int, int8, int16, int32, int64, uint8, uint16, uint32, uint64: - fmt.Printf(indent + "%s: %d\n", k, t) - case float32, float64: - fmt.Printf(indent + "%s: %.2f\n", k, t) - case time.Time: - fmt.Printf(indent + "%s: %s\n", k, t.Format("_2 Jan 2006 15.04.05")) - case time.Duration: - fmt.Printf(indent + "%s: %s\n", k, (t-(t%time.Second)).String()) - default: - fmt.Printf(indent + "%s: %v\n", k, t) - } - } - } - if snap.NumSubs() != 0 { - for i, j := 0, snap.NumSubs(); i < j; i = i+1 { - sub_snap := snap.Sub(i) - fmt.Printf(indent + "%s:\n", sub_snap.Description()) - a.renderSnapYAML(indent + " ", sub_snap) - } - } + if snap.NumEntries() != 0 { + for i, j := 0, snap.NumEntries(); i < j; i = i + 1 { + k, v := snap.Entry(i) + switch t := v.(type) { + case string: + fmt.Printf(indent+"%s: %s\n", k, t) + case int, int8, int16, int32, int64, uint8, uint16, uint32, uint64: + fmt.Printf(indent+"%s: %d\n", k, t) + case float32, float64: + fmt.Printf(indent+"%s: %.2f\n", k, t) + case time.Time: + fmt.Printf(indent+"%s: %s\n", k, t.Format("_2 Jan 2006 15.04.05")) + case time.Duration: + fmt.Printf(indent+"%s: %s\n", k, (t - (t % time.Second)).String()) + default: + fmt.Printf(indent+"%s: %v\n", k, t) + } + } + } + if snap.NumSubs() != 0 { + for i, j := 0, snap.NumSubs(); i < j; i = i + 1 { + sub_snap := snap.Sub(i) + fmt.Printf(indent+"%s:\n", sub_snap.Description()) + a.renderSnapYAML(indent+" ", sub_snap) + } + } } func (a *Admin) Run() { - signal_chan := make(chan os.Signal, 1) - signal.Notify(signal_chan, os.Interrupt) - - command_chan := make(chan string) - go func() { - var discard bool - reader := bufio.NewReader(os.Stdin) - for { - line, prefix, err := reader.ReadLine() - if err != nil { - break - } else if prefix { - discard = true - } else if discard { - fmt.Printf("Line too long!\n") - discard = false - } else { - command_chan <- string(line) - } - } - }() + signal_chan := make(chan os.Signal, 1) + signal.Notify(signal_chan, os.Interrupt) + + command_chan := make(chan string) + go func() { + var discard bool + reader := bufio.NewReader(os.Stdin) + for { + line, prefix, err := reader.ReadLine() + if err != nil { + break + } else if prefix { + discard = true + } else if discard { + fmt.Printf("Line too long!\n") + discard = false + } else { + command_chan <- string(line) + } + } + }() CommandLoop: - for { - fmt.Printf("> ") - select { - case command := <-command_chan: - if command == "exit" { - break CommandLoop - } - a.ProcessCommand(command) - case <-signal_chan: - fmt.Printf("\n> exit\n") - break CommandLoop - } - } + for { + fmt.Printf("> ") + select { + case command := <-command_chan: + if command == "exit" { + break CommandLoop + } + a.ProcessCommand(command) + case <-signal_chan: + fmt.Printf("\n> exit\n") + break CommandLoop + } + } } func (a *Admin) argsCommand(args []string, watch bool) bool { - var signal_chan chan os.Signal + var signal_chan chan os.Signal - if watch { - signal_chan = make(chan os.Signal, 1) - signal.Notify(signal_chan, os.Interrupt) - } + if watch { + signal_chan = make(chan os.Signal, 1) + signal.Notify(signal_chan, os.Interrupt) + } WatchLoop: - for { - if !a.ProcessCommand(strings.Join(args, " ")) { - if !watch { - return false - } - } - - if !watch { - break - } - - // Gap between repeats - fmt.Printf("\n") - - select { - case <-signal_chan: - break WatchLoop - case <-time.After(time.Second): - } - } - - return true + for { + if !a.ProcessCommand(strings.Join(args, " ")) { + if !watch { + return false + } + } + + if !watch { + break + } + + // Gap between repeats + fmt.Printf("\n") + + select { + case <-signal_chan: + break WatchLoop + case <-time.After(time.Second): + } + } + + return true } func PrintHelp() { - fmt.Printf("Available commands:\n") - fmt.Printf(" reload Reload configuration\n") - fmt.Printf(" status Display the current shipping status\n") - fmt.Printf(" exit Exit\n") + fmt.Printf("Available commands:\n") + fmt.Printf(" reload Reload configuration\n") + fmt.Printf(" status Display the current shipping status\n") + fmt.Printf(" exit Exit\n") } func main() { - var version bool - var quiet bool - var watch bool - var admin_connect string - - flag.BoolVar(&version, "version", false, "display the Log Courier client version") - flag.BoolVar(&quiet, "quiet", false, "quietly execute the command line argument and output only the result") - flag.BoolVar(&watch, "watch", false, "repeat the command specified on the command line every second") - flag.StringVar(&admin_connect, "connect", "tcp:127.0.0.1:1234", "the Log Courier instance to connect to (default tcp:127.0.0.1:1234)") - - flag.Parse() - - if version { - fmt.Printf("Log Courier version %s\n", core.Log_Courier_Version) - os.Exit(0) - } - - if !quiet { - fmt.Printf("Log Courier version %s client\n\n", core.Log_Courier_Version) - } - - args := flag.Args() - - if len(args) != 0 { - // Don't require a connection to display the help message - if args[0] == "help" { - PrintHelp() - os.Exit(0) - } - - admin := NewAdmin(quiet, admin_connect) - if admin.argsCommand(args, watch) { - os.Exit(0) - } - os.Exit(1) - } - - if quiet { - fmt.Printf("No command specified on the command line for quiet execution\n") - os.Exit(1) - } - - if watch { - fmt.Printf("No command specified on the command line to watch\n") - os.Exit(1) - } - - admin := NewAdmin(quiet, admin_connect) - if err := admin.connect(); err != nil { - return - } - - admin.Run() + var version bool + var quiet bool + var watch bool + var admin_connect string + + flag.BoolVar(&version, "version", false, "display the Log Courier client version") + flag.BoolVar(&quiet, "quiet", false, "quietly execute the command line argument and output only the result") + flag.BoolVar(&watch, "watch", false, "repeat the command specified on the command line every second") + flag.StringVar(&admin_connect, "connect", "tcp:127.0.0.1:1234", "the Log Courier instance to connect to (default tcp:127.0.0.1:1234)") + + flag.Parse() + + if version { + fmt.Printf("Log Courier version %s\n", core.Log_Courier_Version) + os.Exit(0) + } + + if !quiet { + fmt.Printf("Log Courier version %s client\n\n", core.Log_Courier_Version) + } + + args := flag.Args() + + if len(args) != 0 { + // Don't require a connection to display the help message + if args[0] == "help" { + PrintHelp() + os.Exit(0) + } + + admin := NewAdmin(quiet, admin_connect) + if admin.argsCommand(args, watch) { + os.Exit(0) + } + os.Exit(1) + } + + if quiet { + fmt.Printf("No command specified on the command line for quiet execution\n") + os.Exit(1) + } + + if watch { + fmt.Printf("No command specified on the command line to watch\n") + os.Exit(1) + } + + admin := NewAdmin(quiet, admin_connect) + if err := admin.connect(); err != nil { + return + } + + admin.Run() } diff --git a/src/lc-curvekey/lc-curvekey.go b/src/lc-curvekey/lc-curvekey.go index 2d09c349..71339f22 100644 --- a/src/lc-curvekey/lc-curvekey.go +++ b/src/lc-curvekey/lc-curvekey.go @@ -17,10 +17,10 @@ package main import ( - "flag" - "fmt" - zmq "github.com/alecthomas/gozmq" - "syscall" + "flag" + "fmt" + zmq "github.com/alecthomas/gozmq" + "syscall" ) /* @@ -31,70 +31,70 @@ import ( import "C" func main() { - var single bool - - flag.BoolVar(&single, "single", false, "generate a single keypair") - flag.Parse() - - if single { - fmt.Println("Generating single keypair...") - - pub, priv, err := CurveKeyPair() - if err != nil { - fmt.Println("An error occurred:", err) - if err == syscall.ENOTSUP { - fmt.Print("Please ensure that your zeromq installation was built with libsodium support") - } - return - } - - fmt.Println("Public key: ", pub) - fmt.Println("Private key: ", priv) - return - } - - fmt.Println("Generating configuration keys...") - fmt.Println("(Use 'genkey --single' to generate a single keypair.)") - fmt.Println("") - - server_pub, server_priv, err := CurveKeyPair() - if err != nil { - fmt.Println("An error occurred:", err) - if err == syscall.ENOTSUP { - fmt.Print("Please ensure that your zeromq installation was built with libsodium support") - } - return - } - - client_pub, client_priv, err := CurveKeyPair() - if err != nil { - fmt.Println("An error occurred:", err) - if err == syscall.ENOTSUP { - fmt.Println("Please ensure that your zeromq installation was built with libsodium support") - } - return - } - - fmt.Println("Copy and paste the following into your Log Courier configuration:") - fmt.Printf(" \"curve server key\": \"%s\",\n", server_pub) - fmt.Printf(" \"curve public key\": \"%s\",\n", client_pub) - fmt.Printf(" \"curve secret key\": \"%s\",\n", client_priv) - fmt.Println("") - fmt.Println("Copy and paste the following into your LogStash configuration:") - fmt.Printf(" curve_secret_key => \"%s\",\n", server_priv) + var single bool + + flag.BoolVar(&single, "single", false, "generate a single keypair") + flag.Parse() + + if single { + fmt.Println("Generating single keypair...") + + pub, priv, err := CurveKeyPair() + if err != nil { + fmt.Println("An error occurred:", err) + if err == syscall.ENOTSUP { + fmt.Print("Please ensure that your zeromq installation was built with libsodium support") + } + return + } + + fmt.Println("Public key: ", pub) + fmt.Println("Private key: ", priv) + return + } + + fmt.Println("Generating configuration keys...") + fmt.Println("(Use 'genkey --single' to generate a single keypair.)") + fmt.Println("") + + server_pub, server_priv, err := CurveKeyPair() + if err != nil { + fmt.Println("An error occurred:", err) + if err == syscall.ENOTSUP { + fmt.Print("Please ensure that your zeromq installation was built with libsodium support") + } + return + } + + client_pub, client_priv, err := CurveKeyPair() + if err != nil { + fmt.Println("An error occurred:", err) + if err == syscall.ENOTSUP { + fmt.Println("Please ensure that your zeromq installation was built with libsodium support") + } + return + } + + fmt.Println("Copy and paste the following into your Log Courier configuration:") + fmt.Printf(" \"curve server key\": \"%s\",\n", server_pub) + fmt.Printf(" \"curve public key\": \"%s\",\n", client_pub) + fmt.Printf(" \"curve secret key\": \"%s\",\n", client_priv) + fmt.Println("") + fmt.Println("Copy and paste the following into your LogStash configuration:") + fmt.Printf(" curve_secret_key => \"%s\",\n", server_priv) } // Because gozmq does not yet expose this for us, we have to expose it ourselves func CurveKeyPair() (string, string, error) { - var pub [41]C.char - var priv [41]C.char + var pub [41]C.char + var priv [41]C.char - // Because gozmq does not yet expose this for us, we have to expose it ourselves - if rc, err := C.zmq_curve_keypair(&pub[0], &priv[0]); rc != 0 { - return "", "", casterr(err) - } + // Because gozmq does not yet expose this for us, we have to expose it ourselves + if rc, err := C.zmq_curve_keypair(&pub[0], &priv[0]); rc != 0 { + return "", "", casterr(err) + } - return C.GoString(&pub[0]), C.GoString(&priv[0]), nil + return C.GoString(&pub[0]), C.GoString(&priv[0]), nil } // The following is copy-pasted from gozmq's zmq.go @@ -102,21 +102,21 @@ func CurveKeyPair() (string, string, error) { type zmqErrno syscall.Errno func (e zmqErrno) Error() string { - return C.GoString(C.zmq_strerror(C.int(e))) + return C.GoString(C.zmq_strerror(C.int(e))) } func casterr(fromcgo error) error { - errno, ok := fromcgo.(syscall.Errno) - if !ok { - return fromcgo - } - zmqerrno := zmqErrno(errno) - switch zmqerrno { - case zmq.ENOTSOCK: - return zmqerrno - } - if zmqerrno >= C.ZMQ_HAUSNUMERO { - return zmqerrno - } - return errno + errno, ok := fromcgo.(syscall.Errno) + if !ok { + return fromcgo + } + zmqerrno := zmqErrno(errno) + switch zmqerrno { + case zmq.ENOTSOCK: + return zmqerrno + } + if zmqerrno >= C.ZMQ_HAUSNUMERO { + return zmqerrno + } + return errno } diff --git a/src/lc-lib/admin/client.go b/src/lc-lib/admin/client.go index 0109be0f..7ff172fe 100644 --- a/src/lc-lib/admin/client.go +++ b/src/lc-lib/admin/client.go @@ -12,142 +12,142 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. -*/ + */ package admin import ( - "encoding/gob" - "fmt" - "lc-lib/core" - "net" - "strings" - "time" + "encoding/gob" + "fmt" + "github.com/driskell/log-courier/src/lc-lib/core" + "net" + "strings" + "time" ) type Client struct { - admin_connect string - conn net.Conn - decoder *gob.Decoder + admin_connect string + conn net.Conn + decoder *gob.Decoder } func NewClient(admin_connect string) (*Client, error) { - var err error + var err error - ret := &Client{} + ret := &Client{} - // TODO: handle the connection in a goroutine that can PING - // on idle, and implement a close member to shut it - // it down. For now we'll rely on the auto-reconnect - if ret.conn, err = ret.connect(admin_connect); err != nil { - return nil, err - } + // TODO: handle the connection in a goroutine that can PING + // on idle, and implement a close member to shut it + // it down. For now we'll rely on the auto-reconnect + if ret.conn, err = ret.connect(admin_connect); err != nil { + return nil, err + } - ret.decoder = gob.NewDecoder(ret.conn) + ret.decoder = gob.NewDecoder(ret.conn) - return ret, nil + return ret, nil } func (c *Client) connect(admin_connect string) (net.Conn, error) { - connect := strings.SplitN(admin_connect, ":", 2) - if len(connect) == 1 { - connect = append(connect, connect[0]) - connect[0] = "tcp" - } + connect := strings.SplitN(admin_connect, ":", 2) + if len(connect) == 1 { + connect = append(connect, connect[0]) + connect[0] = "tcp" + } - if connector, ok := registeredConnectors[connect[0]]; ok { - return connector(connect[0], connect[1]) - } + if connector, ok := registeredConnectors[connect[0]]; ok { + return connector(connect[0], connect[1]) + } - return nil, fmt.Errorf("Unknown transport specified in connection address: '%s'", connect[0]) + return nil, fmt.Errorf("Unknown transport specified in connection address: '%s'", connect[0]) } func (c *Client) request(command string) (*Response, error) { - if err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Second)); err != nil { - return nil, err - } + if err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Second)); err != nil { + return nil, err + } - total_written := 0 + total_written := 0 - for { - wrote, err := c.conn.Write([]byte(command[total_written:4])) - if err != nil { - return nil, err - } + for { + wrote, err := c.conn.Write([]byte(command[total_written:4])) + if err != nil { + return nil, err + } - total_written += wrote - if total_written == 4 { - break - } - } + total_written += wrote + if total_written == 4 { + break + } + } - var response Response + var response Response - if err := c.conn.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil { - return nil, err - } + if err := c.conn.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil { + return nil, err + } - if err := c.decoder.Decode(&response); err != nil { - return nil, err - } + if err := c.decoder.Decode(&response); err != nil { + return nil, err + } - return &response, nil + return &response, nil } func (c *Client) resolveError(response *Response) error { - ret, ok := response.Response.(*ErrorResponse) - if ok { - return ret - } + ret, ok := response.Response.(*ErrorResponse) + if ok { + return ret + } - return &ErrorResponse{Message: fmt.Sprintf("Unrecognised response: %v\n", ret)} + return &ErrorResponse{Message: fmt.Sprintf("Unrecognised response: %v\n", ret)} } func (c *Client) Ping() error { - response, err := c.request("PING") - if err != nil { - return err - } + response, err := c.request("PING") + if err != nil { + return err + } - if _, ok := response.Response.(*PongResponse); ok { - return nil - } + if _, ok := response.Response.(*PongResponse); ok { + return nil + } - return c.resolveError(response) + return c.resolveError(response) } func (c *Client) Reload() error { - response, err := c.request("RELD") - if err != nil { - return err - } + response, err := c.request("RELD") + if err != nil { + return err + } - if _, ok := response.Response.(*ReloadResponse); ok { - return nil - } + if _, ok := response.Response.(*ReloadResponse); ok { + return nil + } - return c.resolveError(response) + return c.resolveError(response) } func (c *Client) FetchSnapshot() (*core.Snapshot, error) { - response, err := c.request("SNAP") - if err != nil { - return nil, err - } - - if ret, ok := response.Response.(*core.Snapshot); ok { - return ret, nil - } - - // Backwards compatibility - if ret, ok := response.Response.([]*core.Snapshot); ok { - snap := core.NewSnapshot("Log Courier") - for _, sub := range ret { - snap.AddSub(sub) - } - snap.Sort() - return snap, nil - } - - return nil, c.resolveError(response) + response, err := c.request("SNAP") + if err != nil { + return nil, err + } + + if ret, ok := response.Response.(*core.Snapshot); ok { + return ret, nil + } + + // Backwards compatibility + if ret, ok := response.Response.([]*core.Snapshot); ok { + snap := core.NewSnapshot("Log Courier") + for _, sub := range ret { + snap.AddSub(sub) + } + snap.Sort() + return snap, nil + } + + return nil, c.resolveError(response) } diff --git a/src/lc-lib/admin/listener.go b/src/lc-lib/admin/listener.go index 0de9146c..94ed0887 100644 --- a/src/lc-lib/admin/listener.go +++ b/src/lc-lib/admin/listener.go @@ -12,154 +12,154 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. -*/ + */ package admin import ( - "fmt" - "lc-lib/core" - "net" - "strings" - "time" + "fmt" + "github.com/driskell/log-courier/src/lc-lib/core" + "net" + "strings" + "time" ) type Listener struct { - core.PipelineSegment - core.PipelineConfigReceiver - - config *core.GeneralConfig - command_chan chan string - response_chan chan *Response - listener NetListener - client_shutdown chan interface{} - client_started chan interface{} - client_ended chan interface{} + core.PipelineSegment + core.PipelineConfigReceiver + + config *core.GeneralConfig + command_chan chan string + response_chan chan *Response + listener NetListener + client_shutdown chan interface{} + client_started chan interface{} + client_ended chan interface{} } func NewListener(pipeline *core.Pipeline, config *core.GeneralConfig) (*Listener, error) { - var err error + var err error - ret := &Listener{ - config: config, - command_chan: make(chan string), - response_chan: make(chan *Response), - client_shutdown: make(chan interface{}), - // TODO: Make this limit configurable - client_started: make(chan interface{}, 50), - client_ended: make(chan interface{}, 50), - } + ret := &Listener{ + config: config, + command_chan: make(chan string), + response_chan: make(chan *Response), + client_shutdown: make(chan interface{}), + // TODO: Make this limit configurable + client_started: make(chan interface{}, 50), + client_ended: make(chan interface{}, 50), + } - if ret.listener, err = ret.listen(config); err != nil { - return nil, err - } + if ret.listener, err = ret.listen(config); err != nil { + return nil, err + } - pipeline.Register(ret) + pipeline.Register(ret) - return ret, nil + return ret, nil } func (l *Listener) listen(config *core.GeneralConfig) (NetListener, error) { - bind := strings.SplitN(config.AdminBind, ":", 2) - if len(bind) == 1 { - bind = append(bind, bind[0]) - bind[0] = "tcp" - } + bind := strings.SplitN(config.AdminBind, ":", 2) + if len(bind) == 1 { + bind = append(bind, bind[0]) + bind[0] = "tcp" + } - if listener, ok := registeredListeners[bind[0]]; ok { - return listener(bind[0], bind[1]) - } + if listener, ok := registeredListeners[bind[0]]; ok { + return listener(bind[0], bind[1]) + } - return nil, fmt.Errorf("Unknown transport specified for admin bind: '%s'", bind[0]) + return nil, fmt.Errorf("Unknown transport specified for admin bind: '%s'", bind[0]) } func (l *Listener) OnCommand() <-chan string { - return l.command_chan + return l.command_chan } func (l *Listener) Respond(response *Response) { - l.response_chan <- response + l.response_chan <- response } func (l *Listener) Run() { - defer func(){ - l.Done() - }() + defer func() { + l.Done() + }() ListenerLoop: - for { - select { - case <-l.OnShutdown(): - break ListenerLoop - case config := <-l.OnConfig(): - // We can't yet disable admin during a reload - if config.General.AdminEnabled { - if config.General.AdminBind != l.config.AdminBind { - new_listener, err := l.listen(&config.General) - if err != nil { - log.Error("The new admin configuration failed to apply: %s", err) - continue - } - - l.listener.Close() - l.listener = new_listener - l.config = &config.General - } - } - default: - } - - l.listener.SetDeadline(time.Now().Add(time.Second)) - - conn, err := l.listener.Accept() - if err != nil { - if net_err, ok := err.(*net.OpError); ok && net_err.Timeout() { - continue - } - log.Warning("Failed to accept admin connection: %s", err) - } - - log.Debug("New admin connection from %s", conn.RemoteAddr()) - - l.startServer(conn) - } - - // Shutdown listener - l.listener.Close() - - // Trigger shutdowns - close(l.client_shutdown) - - // Wait for shutdowns - for { - if len(l.client_started) == 0 { - break - } - - select { - case <-l.client_ended: - <-l.client_started - default: - } - } + for { + select { + case <-l.OnShutdown(): + break ListenerLoop + case config := <-l.OnConfig(): + // We can't yet disable admin during a reload + if config.General.AdminEnabled { + if config.General.AdminBind != l.config.AdminBind { + new_listener, err := l.listen(&config.General) + if err != nil { + log.Error("The new admin configuration failed to apply: %s", err) + continue + } + + l.listener.Close() + l.listener = new_listener + l.config = &config.General + } + } + default: + } + + l.listener.SetDeadline(time.Now().Add(time.Second)) + + conn, err := l.listener.Accept() + if err != nil { + if net_err, ok := err.(*net.OpError); ok && net_err.Timeout() { + continue + } + log.Warning("Failed to accept admin connection: %s", err) + } + + log.Debug("New admin connection from %s", conn.RemoteAddr()) + + l.startServer(conn) + } + + // Shutdown listener + l.listener.Close() + + // Trigger shutdowns + close(l.client_shutdown) + + // Wait for shutdowns + for { + if len(l.client_started) == 0 { + break + } + + select { + case <-l.client_ended: + <-l.client_started + default: + } + } } func (l *Listener) startServer(conn net.Conn) { - server := newServer(l, conn) - - select { - case <-l.client_ended: - <-l.client_started - default: - } - - select { - case l.client_started <- 1: - default: - // TODO: Make this limit configurable - log.Warning("Refused admin connection: Admin connection limit (50) reached") - return - } - - go server.Run() + server := newServer(l, conn) + + select { + case <-l.client_ended: + <-l.client_started + default: + } + + select { + case l.client_started <- 1: + default: + // TODO: Make this limit configurable + log.Warning("Refused admin connection: Admin connection limit (50) reached") + return + } + + go server.Run() } diff --git a/src/lc-lib/admin/logging.go b/src/lc-lib/admin/logging.go index efc0a01b..c0806433 100644 --- a/src/lc-lib/admin/logging.go +++ b/src/lc-lib/admin/logging.go @@ -21,5 +21,5 @@ import "github.com/op/go-logging" var log *logging.Logger func init() { - log = logging.MustGetLogger("admin") + log = logging.MustGetLogger("admin") } diff --git a/src/lc-lib/admin/responses.go b/src/lc-lib/admin/responses.go index 5545b5f6..3e1c78e0 100644 --- a/src/lc-lib/admin/responses.go +++ b/src/lc-lib/admin/responses.go @@ -12,18 +12,18 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. -*/ + */ package admin import ( - "encoding/gob" - "lc-lib/core" - "time" + "encoding/gob" + "github.com/driskell/log-courier/src/lc-lib/core" + "time" ) type Response struct { - Response interface{} + Response interface{} } type PongResponse struct { @@ -33,30 +33,30 @@ type ReloadResponse struct { } type ErrorResponse struct { - Message string + Message string } func (e *ErrorResponse) Error() string { - return e.Message + return e.Message } func init() { - // Response structure - gob.Register(&Response{}) + // Response structure + gob.Register(&Response{}) - // General error - gob.Register(&ErrorResponse{}) + // General error + gob.Register(&ErrorResponse{}) - // PONG - gob.Register(&PongResponse{}) + // PONG + gob.Register(&PongResponse{}) - // RELD - gob.Register(&ReloadResponse{}) + // RELD + gob.Register(&ReloadResponse{}) - // SNAP - gob.Register(&core.Snapshot{}) - // SNAP - time.Time - gob.Register(time.Now()) - // SNAP - time.Duration - gob.Register(time.Since(time.Now())) + // SNAP + gob.Register(&core.Snapshot{}) + // SNAP - time.Time + gob.Register(time.Now()) + // SNAP - time.Duration + gob.Register(time.Since(time.Now())) } diff --git a/src/lc-lib/admin/server.go b/src/lc-lib/admin/server.go index 7c7e0e14..dbb25b47 100644 --- a/src/lc-lib/admin/server.go +++ b/src/lc-lib/admin/server.go @@ -12,136 +12,136 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. -*/ + */ package admin import ( - "encoding/gob" - "fmt" - "io" - "net" - "time" + "encoding/gob" + "fmt" + "io" + "net" + "time" ) type server struct { - listener *Listener - conn net.Conn + listener *Listener + conn net.Conn - encoder *gob.Encoder + encoder *gob.Encoder } func newServer(listener *Listener, conn net.Conn) *server { - return &server{ - listener: listener, - conn: conn, - } + return &server{ + listener: listener, + conn: conn, + } } func (s *server) Run() { - if err := s.loop(); err != nil { - log.Warning("Error on admin connection from %s: %s", s.conn.RemoteAddr(), err) - } else { - log.Debug("Admin connection from %s closed", s.conn.RemoteAddr()) - } + if err := s.loop(); err != nil { + log.Warning("Error on admin connection from %s: %s", s.conn.RemoteAddr(), err) + } else { + log.Debug("Admin connection from %s closed", s.conn.RemoteAddr()) + } - if conn, ok := s.conn.(*net.TCPConn); ok { - // TODO: Make linger time configurable? - conn.SetLinger(5) - } + if conn, ok := s.conn.(*net.TCPConn); ok { + // TODO: Make linger time configurable? + conn.SetLinger(5) + } - s.conn.Close() + s.conn.Close() - s.listener.client_ended <- 1 + s.listener.client_ended <- 1 } func (s *server) loop() (err error) { - var result *Response -// TODO : Obey shutdown request on s.listener.client_shutdown channel close - s.encoder = gob.NewEncoder(s.conn) - - command := make([]byte, 4) - - for { - if err = s.readCommand(command); err != nil { - if err == io.EOF { - err = nil - } - return - } - - log.Debug("Command from %s: %s", s.conn.RemoteAddr(), command) - - if string(command) == "PING" { - result = &Response{&PongResponse{}} - } else { - result = s.processCommand(string(command)) - } - - if err = s.sendResponse(result); err != nil { - return - } - } + var result *Response + // TODO : Obey shutdown request on s.listener.client_shutdown channel close + s.encoder = gob.NewEncoder(s.conn) + + command := make([]byte, 4) + + for { + if err = s.readCommand(command); err != nil { + if err == io.EOF { + err = nil + } + return + } + + log.Debug("Command from %s: %s", s.conn.RemoteAddr(), command) + + if string(command) == "PING" { + result = &Response{&PongResponse{}} + } else { + result = s.processCommand(string(command)) + } + + if err = s.sendResponse(result); err != nil { + return + } + } } func (s *server) readCommand(command []byte) error { - total_read := 0 - start_time := time.Now() - - for { - // Poll every second for shutdown - if err := s.conn.SetReadDeadline(time.Now().Add(time.Second)); err != nil { - return err - } - - read, err := s.conn.Read(command[total_read:4]) - if err != nil { - if op_err, ok := err.(*net.OpError); ok && op_err.Timeout() { - // TODO: Make idle timeout configurable - if time.Now().Sub(start_time) <= 1800 * time.Second { - // Check shutdown at each interval - select { - case <-s.listener.client_shutdown: - return io.EOF - default: - } - - continue - } - } else if total_read != 0 && op_err == io.EOF { - return fmt.Errorf("EOF") - } - return err - } - - total_read += read - if total_read == 4 { - break - } - } - - return nil + total_read := 0 + start_time := time.Now() + + for { + // Poll every second for shutdown + if err := s.conn.SetReadDeadline(time.Now().Add(time.Second)); err != nil { + return err + } + + read, err := s.conn.Read(command[total_read:4]) + if err != nil { + if op_err, ok := err.(*net.OpError); ok && op_err.Timeout() { + // TODO: Make idle timeout configurable + if time.Now().Sub(start_time) <= 1800*time.Second { + // Check shutdown at each interval + select { + case <-s.listener.client_shutdown: + return io.EOF + default: + } + + continue + } + } else if total_read != 0 && op_err == io.EOF { + return fmt.Errorf("EOF") + } + return err + } + + total_read += read + if total_read == 4 { + break + } + } + + return nil } func (s *server) sendResponse(response *Response) error { - if err := s.conn.SetWriteDeadline(time.Now().Add(5 * time.Second)); err != nil { - return err - } + if err := s.conn.SetWriteDeadline(time.Now().Add(5 * time.Second)); err != nil { + return err + } - if err := s.encoder.Encode(response); err != nil { - return err - } + if err := s.encoder.Encode(response); err != nil { + return err + } - return nil + return nil } func (s *server) processCommand(command string) *Response { - select { - case s.listener.command_chan <- command: - // Listener immediately stops processing commands on shutdown, so catch it here - case <-s.listener.client_shutdown: - return &Response{&ErrorResponse{Message: "Log Courier is shutting down"}} - } - - return <-s.listener.response_chan + select { + case s.listener.command_chan <- command: + // Listener immediately stops processing commands on shutdown, so catch it here + case <-s.listener.client_shutdown: + return &Response{&ErrorResponse{Message: "Log Courier is shutting down"}} + } + + return <-s.listener.response_chan } diff --git a/src/lc-lib/admin/transport.go b/src/lc-lib/admin/transport.go index 483ffad7..3632aa82 100644 --- a/src/lc-lib/admin/transport.go +++ b/src/lc-lib/admin/transport.go @@ -12,7 +12,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. -*/ + */ package admin @@ -33,7 +33,7 @@ type listenerFunc func(string, string) (NetListener, error) var ( registeredConnectors map[string]connectorFunc = make(map[string]connectorFunc) - registeredListeners map[string]listenerFunc = make(map[string]listenerFunc) + registeredListeners map[string]listenerFunc = make(map[string]listenerFunc) ) func registerTransport(name string, connector connectorFunc, listener listenerFunc) { diff --git a/src/lc-lib/admin/transport_tcp.go b/src/lc-lib/admin/transport_tcp.go index 44ab237c..fb96e75b 100644 --- a/src/lc-lib/admin/transport_tcp.go +++ b/src/lc-lib/admin/transport_tcp.go @@ -12,7 +12,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. -*/ + */ package admin diff --git a/src/lc-lib/admin/transport_unix.go b/src/lc-lib/admin/transport_unix.go index 02c6d6e9..febd39dd 100644 --- a/src/lc-lib/admin/transport_unix.go +++ b/src/lc-lib/admin/transport_unix.go @@ -14,13 +14,14 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. -*/ + */ package admin import ( "fmt" "net" + "os" ) func init() { @@ -49,6 +50,14 @@ func listenUnix(transport, addr string) (NetListener, error) { return nil, fmt.Errorf("The admin bind address specified is not valid: %s", err) } + // Remove previous socket file if it's still there or we'll get address + // already in use error + if _, err = os.Stat(addr); err == nil || !os.IsNotExist(err) { + if err := os.Remove(addr); err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("Failed to remove the existing socket file: %s", err) + } + } + listener, err := net.ListenUnix("unix", uaddr) if err != nil { return nil, err diff --git a/src/lc-lib/codecs/filter.go b/src/lc-lib/codecs/filter.go index 47116141..171e7980 100644 --- a/src/lc-lib/codecs/filter.go +++ b/src/lc-lib/codecs/filter.go @@ -17,90 +17,93 @@ package codecs import ( - "errors" - "fmt" - "lc-lib/core" - "regexp" + "errors" + "fmt" + "github.com/driskell/log-courier/src/lc-lib/core" + "regexp" ) type CodecFilterFactory struct { - Patterns []string `config:"patterns"` - Negate bool `config:"negate"` + Patterns []string `config:"patterns"` + Negate bool `config:"negate"` - matchers []*regexp.Regexp + matchers []*regexp.Regexp } type CodecFilter struct { - config *CodecFilterFactory - last_offset int64 - filtered_lines uint64 - callback_func core.CodecCallbackFunc - meter_filtered uint64 + config *CodecFilterFactory + last_offset int64 + filtered_lines uint64 + callback_func core.CodecCallbackFunc + meter_filtered uint64 } func NewFilterCodecFactory(config *core.Config, config_path string, unused map[string]interface{}, name string) (core.CodecFactory, error) { - var err error - - result := &CodecFilterFactory{} - if err = config.PopulateConfig(result, config_path, unused); err != nil { - return nil, err - } - - if len(result.Patterns) == 0 { - return nil, errors.New("Filter codec pattern must be specified.") - } - - result.matchers = make([]*regexp.Regexp, len(result.Patterns)) - for k, pattern := range result.Patterns { - result.matchers[k], err = regexp.Compile(pattern) - if err != nil { - return nil, fmt.Errorf("Failed to compile filter codec pattern, '%s'.", err) - } - } - - return result, nil + var err error + + result := &CodecFilterFactory{} + if err = config.PopulateConfig(result, config_path, unused); err != nil { + return nil, err + } + + if len(result.Patterns) == 0 { + return nil, errors.New("Filter codec pattern must be specified.") + } + + result.matchers = make([]*regexp.Regexp, len(result.Patterns)) + for k, pattern := range result.Patterns { + result.matchers[k], err = regexp.Compile(pattern) + if err != nil { + return nil, fmt.Errorf("Failed to compile filter codec pattern, '%s'.", err) + } + } + + return result, nil } func (f *CodecFilterFactory) NewCodec(callback_func core.CodecCallbackFunc, offset int64) core.Codec { - return &CodecFilter{ - config: f, - last_offset: offset, - callback_func: callback_func, - } + return &CodecFilter{ + config: f, + last_offset: offset, + callback_func: callback_func, + } } func (c *CodecFilter) Teardown() int64 { - return c.last_offset + return c.last_offset } func (c *CodecFilter) Event(start_offset int64, end_offset int64, text string) { - // Only flush the event if it matches a filter - var match bool - for _, matcher := range c.config.matchers { - if matcher.MatchString(text) { - match = true - break - } - } - - if c.config.Negate != match { - c.callback_func(start_offset, end_offset, text) - } else { - c.filtered_lines++ - } + // Only flush the event if it matches a filter + var match bool + + for _, matcher := range c.config.matchers { + if matcher.MatchString(text) { + match = true + break + } + } + + if c.config.Negate != match { + c.callback_func(start_offset, end_offset, text) + } else { + c.filtered_lines++ + } + + c.last_offset = end_offset } func (c *CodecFilter) Meter() { - c.meter_filtered = c.filtered_lines + c.meter_filtered = c.filtered_lines } func (c *CodecFilter) Snapshot() *core.Snapshot { - snap := core.NewSnapshot("Filter Codec") - snap.AddEntry("Filtered lines", c.meter_filtered) - return snap + snap := core.NewSnapshot("Filter Codec") + snap.AddEntry("Filtered lines", c.meter_filtered) + return snap } // Register the codec func init() { - core.RegisterCodec("filter", NewFilterCodecFactory) + core.RegisterCodec("filter", NewFilterCodecFactory) } diff --git a/src/lc-lib/codecs/filter_test.go b/src/lc-lib/codecs/filter_test.go index 4d34624a..3c1d0c19 100644 --- a/src/lc-lib/codecs/filter_test.go +++ b/src/lc-lib/codecs/filter_test.go @@ -1,102 +1,108 @@ package codecs import ( - "lc-lib/core" - "testing" + "github.com/driskell/log-courier/src/lc-lib/core" + "testing" ) var filter_lines []string func createFilterCodec(unused map[string]interface{}, callback core.CodecCallbackFunc, t *testing.T) core.Codec { - config := core.NewConfig() + config := core.NewConfig() - factory, err := NewFilterCodecFactory(config, "", unused, "filter") - if err != nil { - t.Logf("Failed to create filter codec: %s", err) - t.FailNow() - } + factory, err := NewFilterCodecFactory(config, "", unused, "filter") + if err != nil { + t.Logf("Failed to create filter codec: %s", err) + t.FailNow() + } - return factory.NewCodec(callback, 0) + return factory.NewCodec(callback, 0) } func checkFilter(start_offset int64, end_offset int64, text string) { - filter_lines = append(filter_lines, text) + filter_lines = append(filter_lines, text) } func TestFilter(t *testing.T) { - filter_lines = make([]string, 0, 1) - - codec := createFilterCodec(map[string]interface{}{ - "patterns": []string{"^NEXT line$"}, - "negate": false, - }, checkFilter, t) - - // Send some data - codec.Event(0, 1, "DEBUG First line") - codec.Event(2, 3, "NEXT line") - codec.Event(4, 5, "ANOTHER line") - codec.Event(6, 7, "DEBUG Next line") - - if len(filter_lines) != 1 { - t.Logf("Wrong line count received") - t.FailNow() - } else if filter_lines[0] != "NEXT line" { - t.Logf("Wrong line[0] received: %s", filter_lines[0]) - t.FailNow() - } + filter_lines = make([]string, 0, 1) + + codec := createFilterCodec(map[string]interface{}{ + "patterns": []string{"^NEXT line$"}, + "negate": false, + }, checkFilter, t) + + // Send some data + codec.Event(0, 1, "DEBUG First line") + codec.Event(2, 3, "NEXT line") + codec.Event(4, 5, "ANOTHER line") + codec.Event(6, 7, "DEBUG Next line") + + if len(filter_lines) != 1 { + t.Error("Wrong line count received") + } else if filter_lines[0] != "NEXT line" { + t.Error("Wrong line[0] received: %s", filter_lines[0]) + } + + offset := codec.Teardown() + if offset != 7 { + t.Error("Teardown returned incorrect offset: ", offset) + } } func TestFilterNegate(t *testing.T) { - filter_lines = make([]string, 0, 1) - - codec := createFilterCodec(map[string]interface{}{ - "patterns": []string{"^NEXT line$"}, - "negate": true, - }, checkFilter, t) - - // Send some data - codec.Event(0, 1, "DEBUG First line") - codec.Event(2, 3, "NEXT line") - codec.Event(4, 5, "ANOTHER line") - codec.Event(6, 7, "DEBUG Next line") - - if len(filter_lines) != 3 { - t.Logf("Wrong line count received") - t.FailNow() - } else if filter_lines[0] != "DEBUG First line" { - t.Logf("Wrong line[0] received: %s", filter_lines[0]) - t.FailNow() - } else if filter_lines[1] != "ANOTHER line" { - t.Logf("Wrong line[1] received: %s", filter_lines[1]) - t.FailNow() - } else if filter_lines[2] != "DEBUG Next line" { - t.Logf("Wrong line[2] received: %s", filter_lines[2]) - t.FailNow() - } + filter_lines = make([]string, 0, 1) + + codec := createFilterCodec(map[string]interface{}{ + "patterns": []string{"^NEXT line$"}, + "negate": true, + }, checkFilter, t) + + // Send some data + codec.Event(0, 1, "DEBUG First line") + codec.Event(2, 3, "NEXT line") + codec.Event(4, 5, "ANOTHER line") + codec.Event(6, 7, "DEBUG Next line") + + if len(filter_lines) != 3 { + t.Error("Wrong line count received") + } else if filter_lines[0] != "DEBUG First line" { + t.Error("Wrong line[0] received: %s", filter_lines[0]) + } else if filter_lines[1] != "ANOTHER line" { + t.Error("Wrong line[1] received: %s", filter_lines[1]) + } else if filter_lines[2] != "DEBUG Next line" { + t.Error("Wrong line[2] received: %s", filter_lines[2]) + } + + offset := codec.Teardown() + if offset != 7 { + t.Error("Teardown returned incorrect offset: ", offset) + } } func TestFilterMultiple(t *testing.T) { - filter_lines = make([]string, 0, 1) - - codec := createFilterCodec(map[string]interface{}{ - "patterns": []string{"^NEXT line$", "^DEBUG First line$"}, - "negate": false, - }, checkFilter, t) - - // Send some data - codec.Event(0, 1, "DEBUG First line") - codec.Event(2, 3, "NEXT line") - codec.Event(4, 5, "ANOTHER line") - codec.Event(6, 7, "DEBUG Next line") - - if len(filter_lines) != 2 { - t.Logf("Wrong line count received") - t.FailNow() - } else if filter_lines[0] != "DEBUG First line" { - t.Logf("Wrong line[0] received: %s", filter_lines[0]) - t.FailNow() - } else if filter_lines[1] != "NEXT line" { - t.Logf("Wrong line[1] received: %s", filter_lines[1]) - t.FailNow() - } + filter_lines = make([]string, 0, 1) + + codec := createFilterCodec(map[string]interface{}{ + "patterns": []string{"^NEXT line$", "^DEBUG First line$"}, + "negate": false, + }, checkFilter, t) + + // Send some data + codec.Event(0, 1, "DEBUG First line") + codec.Event(2, 3, "NEXT line") + codec.Event(4, 5, "ANOTHER line") + codec.Event(6, 7, "DEBUG Next line") + + if len(filter_lines) != 2 { + t.Error("Wrong line count received") + } else if filter_lines[0] != "DEBUG First line" { + t.Error("Wrong line[0] received: %s", filter_lines[0]) + } else if filter_lines[1] != "NEXT line" { + t.Error("Wrong line[1] received: %s", filter_lines[1]) + } + + offset := codec.Teardown() + if offset != 7 { + t.Error("Teardown returned incorrect offset: ", offset) + } } diff --git a/src/lc-lib/codecs/multiline.go b/src/lc-lib/codecs/multiline.go index 88bc5595..1a6ec8e4 100644 --- a/src/lc-lib/codecs/multiline.go +++ b/src/lc-lib/codecs/multiline.go @@ -17,236 +17,236 @@ package codecs import ( - "errors" - "fmt" - "lc-lib/core" - "regexp" - "strings" - "sync" - "time" + "errors" + "fmt" + "github.com/driskell/log-courier/src/lc-lib/core" + "regexp" + "strings" + "sync" + "time" ) const ( - codecMultiline_What_Previous = 0x00000001 - codecMultiline_What_Next = 0x00000002 + codecMultiline_What_Previous = 0x00000001 + codecMultiline_What_Next = 0x00000002 ) type CodecMultilineFactory struct { - Pattern string `config:"pattern"` - What string `config:"what"` - Negate bool `config:"negate"` - PreviousTimeout time.Duration `config:"previous timeout"` - MaxMultilineBytes int64 `config:"max multiline bytes"` - - matcher *regexp.Regexp - what int + Pattern string `config:"pattern"` + What string `config:"what"` + Negate bool `config:"negate"` + PreviousTimeout time.Duration `config:"previous timeout"` + MaxMultilineBytes int64 `config:"max multiline bytes"` + + matcher *regexp.Regexp + what int } type CodecMultiline struct { - config *CodecMultilineFactory - last_offset int64 - callback_func core.CodecCallbackFunc - - end_offset int64 - start_offset int64 - buffer []string - buffer_lines int64 - buffer_len int64 - timer_lock sync.Mutex - timer_stop chan interface{} - timer_wait sync.WaitGroup - timer_deadline time.Time - - meter_lines int64 - meter_bytes int64 + config *CodecMultilineFactory + last_offset int64 + callback_func core.CodecCallbackFunc + + end_offset int64 + start_offset int64 + buffer []string + buffer_lines int64 + buffer_len int64 + timer_lock sync.Mutex + timer_stop chan interface{} + timer_wait sync.WaitGroup + timer_deadline time.Time + + meter_lines int64 + meter_bytes int64 } func NewMultilineCodecFactory(config *core.Config, config_path string, unused map[string]interface{}, name string) (core.CodecFactory, error) { - var err error - - result := &CodecMultilineFactory{} - if err = config.PopulateConfig(result, config_path, unused); err != nil { - return nil, err - } - - if result.Pattern == "" { - return nil, errors.New("Multiline codec pattern must be specified.") - } - - result.matcher, err = regexp.Compile(result.Pattern) - if err != nil { - return nil, fmt.Errorf("Failed to compile multiline codec pattern, '%s'.", err) - } - - if result.What == "" || result.What == "previous" { - result.what = codecMultiline_What_Previous - } else if result.What == "next" { - result.what = codecMultiline_What_Next - } - - if result.MaxMultilineBytes == 0 { - result.MaxMultilineBytes = config.General.SpoolMaxBytes - } - - // We conciously allow a line 4 bytes longer what we would normally have as the limit - // This 4 bytes is the event header size. It's not worth considering though - if result.MaxMultilineBytes > config.General.SpoolMaxBytes { - return nil, fmt.Errorf("max multiline bytes cannot be greater than /general/spool max bytes") - } - - return result, nil + var err error + + result := &CodecMultilineFactory{} + if err = config.PopulateConfig(result, config_path, unused); err != nil { + return nil, err + } + + if result.Pattern == "" { + return nil, errors.New("Multiline codec pattern must be specified.") + } + + result.matcher, err = regexp.Compile(result.Pattern) + if err != nil { + return nil, fmt.Errorf("Failed to compile multiline codec pattern, '%s'.", err) + } + + if result.What == "" || result.What == "previous" { + result.what = codecMultiline_What_Previous + } else if result.What == "next" { + result.what = codecMultiline_What_Next + } + + if result.MaxMultilineBytes == 0 { + result.MaxMultilineBytes = config.General.SpoolMaxBytes + } + + // We conciously allow a line 4 bytes longer what we would normally have as the limit + // This 4 bytes is the event header size. It's not worth considering though + if result.MaxMultilineBytes > config.General.SpoolMaxBytes { + return nil, fmt.Errorf("max multiline bytes cannot be greater than /general/spool max bytes") + } + + return result, nil } func (f *CodecMultilineFactory) NewCodec(callback_func core.CodecCallbackFunc, offset int64) core.Codec { - c := &CodecMultiline{ - config: f, - end_offset: offset, - last_offset: offset, - callback_func: callback_func, - } - - // Start the "previous timeout" routine that will auto flush at deadline - if f.PreviousTimeout != 0 { - c.timer_stop = make(chan interface{}) - c.timer_wait.Add(1) - - c.timer_deadline = time.Now().Add(f.PreviousTimeout) - - go c.deadlineRoutine() - } - return c + c := &CodecMultiline{ + config: f, + end_offset: offset, + last_offset: offset, + callback_func: callback_func, + } + + // Start the "previous timeout" routine that will auto flush at deadline + if f.PreviousTimeout != 0 { + c.timer_stop = make(chan interface{}) + c.timer_wait.Add(1) + + c.timer_deadline = time.Now().Add(f.PreviousTimeout) + + go c.deadlineRoutine() + } + return c } func (c *CodecMultiline) Teardown() int64 { - if c.config.PreviousTimeout != 0 { - close(c.timer_stop) - c.timer_wait.Wait() - } + if c.config.PreviousTimeout != 0 { + close(c.timer_stop) + c.timer_wait.Wait() + } - return c.last_offset + return c.last_offset } func (c *CodecMultiline) Event(start_offset int64, end_offset int64, text string) { - // TODO(driskell): If we are using previous and we match on the very first line read, - // then this is because we've started in the middle of a multiline event (the first line - // should never match) - so we could potentially offer an option to discard this. - // The benefit would be that when using previous_timeout, we could discard any extraneous - // event data that did not get written in time, if the user so wants it, in order to prevent - // odd incomplete data. It would be a signal from the user, "I will worry about the buffering - // issues my programs may have - you just make sure to write each event either completely or - // partially, always with the FIRST line correct (which could be the important one)." - match_failed := c.config.Negate == c.config.matcher.MatchString(text) - if c.config.what == codecMultiline_What_Previous { - if c.config.PreviousTimeout != 0 { - // Prevent a flush happening while we're modifying the stored data - c.timer_lock.Lock() - } - if match_failed { - c.flush() - } - } - - var text_len int64 = int64(len(text)) - - // Check we don't exceed the max multiline bytes - if check_len := c.buffer_len + text_len + c.buffer_lines; check_len > c.config.MaxMultilineBytes { - // Store partial and flush - overflow := check_len - c.config.MaxMultilineBytes - cut := text_len - overflow - c.end_offset = end_offset - overflow - - c.buffer = append(c.buffer, text[:cut]) - c.buffer_lines++ - c.buffer_len += cut - - c.flush() - - // Append the remaining data to the buffer - start_offset += cut - text = text[cut:] - } - - if len(c.buffer) == 0 { - c.start_offset = start_offset - } - c.end_offset = end_offset - - c.buffer = append(c.buffer, text) - c.buffer_lines++ - c.buffer_len += text_len - - if c.config.what == codecMultiline_What_Previous { - if c.config.PreviousTimeout != 0 { - // Reset the timer and unlock - c.timer_deadline = time.Now().Add(c.config.PreviousTimeout) - c.timer_lock.Unlock() - } - } else if c.config.what == codecMultiline_What_Next && match_failed { - c.flush() - } - // TODO: Split the line if its too big + // TODO(driskell): If we are using previous and we match on the very first line read, + // then this is because we've started in the middle of a multiline event (the first line + // should never match) - so we could potentially offer an option to discard this. + // The benefit would be that when using previous_timeout, we could discard any extraneous + // event data that did not get written in time, if the user so wants it, in order to prevent + // odd incomplete data. It would be a signal from the user, "I will worry about the buffering + // issues my programs may have - you just make sure to write each event either completely or + // partially, always with the FIRST line correct (which could be the important one)." + match_failed := c.config.Negate == c.config.matcher.MatchString(text) + if c.config.what == codecMultiline_What_Previous { + if c.config.PreviousTimeout != 0 { + // Prevent a flush happening while we're modifying the stored data + c.timer_lock.Lock() + } + if match_failed { + c.flush() + } + } + + var text_len int64 = int64(len(text)) + + // Check we don't exceed the max multiline bytes + if check_len := c.buffer_len + text_len + c.buffer_lines; check_len > c.config.MaxMultilineBytes { + // Store partial and flush + overflow := check_len - c.config.MaxMultilineBytes + cut := text_len - overflow + c.end_offset = end_offset - overflow + + c.buffer = append(c.buffer, text[:cut]) + c.buffer_lines++ + c.buffer_len += cut + + c.flush() + + // Append the remaining data to the buffer + start_offset += cut + text = text[cut:] + } + + if len(c.buffer) == 0 { + c.start_offset = start_offset + } + c.end_offset = end_offset + + c.buffer = append(c.buffer, text) + c.buffer_lines++ + c.buffer_len += text_len + + if c.config.what == codecMultiline_What_Previous { + if c.config.PreviousTimeout != 0 { + // Reset the timer and unlock + c.timer_deadline = time.Now().Add(c.config.PreviousTimeout) + c.timer_lock.Unlock() + } + } else if c.config.what == codecMultiline_What_Next && match_failed { + c.flush() + } + // TODO: Split the line if its too big } func (c *CodecMultiline) flush() { - if len(c.buffer) == 0 { - return - } + if len(c.buffer) == 0 { + return + } - text := strings.Join(c.buffer, "\n") + text := strings.Join(c.buffer, "\n") - // Set last offset - this is returned in Teardown so if we're mid multiline and crash, we start this multiline again - c.last_offset = c.end_offset - c.buffer = nil - c.buffer_len = 0 - c.buffer_lines = 0 + // Set last offset - this is returned in Teardown so if we're mid multiline and crash, we start this multiline again + c.last_offset = c.end_offset + c.buffer = nil + c.buffer_len = 0 + c.buffer_lines = 0 - c.callback_func(c.start_offset, c.end_offset, text) + c.callback_func(c.start_offset, c.end_offset, text) } func (c *CodecMultiline) Meter() { - c.meter_lines = c.buffer_lines - c.meter_bytes = c.end_offset - c.last_offset + c.meter_lines = c.buffer_lines + c.meter_bytes = c.end_offset - c.last_offset } func (c *CodecMultiline) Snapshot() *core.Snapshot { - snap := core.NewSnapshot("Multiline Codec") - snap.AddEntry("Pending lines", c.meter_lines) - snap.AddEntry("Pending bytes", c.meter_bytes) - return snap + snap := core.NewSnapshot("Multiline Codec") + snap.AddEntry("Pending lines", c.meter_lines) + snap.AddEntry("Pending bytes", c.meter_bytes) + return snap } func (c *CodecMultiline) deadlineRoutine() { - timer := time.NewTimer(0) + timer := time.NewTimer(0) DeadlineLoop: - for { - select { - case <-c.timer_stop: - timer.Stop() - - // Shutdown signal so end the routine - break DeadlineLoop - case now := <-timer.C: - c.timer_lock.Lock() - - // Have we reached the target time? - if !now.After(c.timer_deadline) { - // Deadline moved, update the timer - timer.Reset(c.timer_deadline.Sub(now)) - c.timer_lock.Unlock() - continue - } - - c.flush() - timer.Reset(c.config.PreviousTimeout) - c.timer_lock.Unlock() - } - } - - c.timer_wait.Done() + for { + select { + case <-c.timer_stop: + timer.Stop() + + // Shutdown signal so end the routine + break DeadlineLoop + case now := <-timer.C: + c.timer_lock.Lock() + + // Have we reached the target time? + if !now.After(c.timer_deadline) { + // Deadline moved, update the timer + timer.Reset(c.timer_deadline.Sub(now)) + c.timer_lock.Unlock() + continue + } + + c.flush() + timer.Reset(c.config.PreviousTimeout) + c.timer_lock.Unlock() + } + } + + c.timer_wait.Done() } // Register the codec func init() { - core.RegisterCodec("multiline", NewMultilineCodecFactory) + core.RegisterCodec("multiline", NewMultilineCodecFactory) } diff --git a/src/lc-lib/codecs/multiline_test.go b/src/lc-lib/codecs/multiline_test.go index 7c8069a5..3ca7ba51 100644 --- a/src/lc-lib/codecs/multiline_test.go +++ b/src/lc-lib/codecs/multiline_test.go @@ -1,10 +1,10 @@ package codecs import ( - "lc-lib/core" - "sync" - "testing" - "time" + "github.com/driskell/log-courier/src/lc-lib/core" + "sync" + "testing" + "time" ) var multiline_t *testing.T @@ -12,223 +12,251 @@ var multiline_lines int var multiline_lock sync.Mutex func createMultilineCodec(unused map[string]interface{}, callback core.CodecCallbackFunc, t *testing.T) core.Codec { - config := core.NewConfig() - config.General.MaxLineBytes = 1048576 - config.General.SpoolMaxBytes = 10485760 + config := core.NewConfig() + config.General.MaxLineBytes = 1048576 + config.General.SpoolMaxBytes = 10485760 - factory, err := NewMultilineCodecFactory(config, "", unused, "multiline") - if err != nil { - t.Logf("Failed to create multiline codec: %s", err) - t.FailNow() - } + factory, err := NewMultilineCodecFactory(config, "", unused, "multiline") + if err != nil { + t.Logf("Failed to create multiline codec: %s", err) + t.FailNow() + } - return factory.NewCodec(callback, 0) + return factory.NewCodec(callback, 0) } func checkMultiline(start_offset int64, end_offset int64, text string) { - multiline_lock.Lock() - defer multiline_lock.Unlock() - multiline_lines++ - - if multiline_lines == 1 { - if text != "DEBUG First line\nNEXT line\nANOTHER line" { - multiline_t.Logf("Event data incorrect [% X]", text) - multiline_t.FailNow() - } - - if start_offset != 0 { - multiline_t.Logf("Event start offset is incorrect [%d]", start_offset) - multiline_t.FailNow() - } - - if end_offset != 5 { - multiline_t.Logf("Event end offset is incorrect [%d]", end_offset) - multiline_t.FailNow() - } - - return - } - - if text != "DEBUG Next line" { - multiline_t.Logf("Event data incorrect [% X]", text) - multiline_t.FailNow() - } - - if start_offset != 6 { - multiline_t.Logf("Event start offset is incorrect [%d]", start_offset) - multiline_t.FailNow() - } - - if end_offset != 7 { - multiline_t.Logf("Event end offset is incorrect [%d]", end_offset) - multiline_t.FailNow() - } + multiline_lock.Lock() + defer multiline_lock.Unlock() + multiline_lines++ + + if multiline_lines == 1 { + if text != "DEBUG First line\nNEXT line\nANOTHER line" { + multiline_t.Logf("Event data incorrect [% X]", text) + multiline_t.FailNow() + } + + if start_offset != 0 { + multiline_t.Logf("Event start offset is incorrect [%d]", start_offset) + multiline_t.FailNow() + } + + if end_offset != 5 { + multiline_t.Logf("Event end offset is incorrect [%d]", end_offset) + multiline_t.FailNow() + } + + return + } + + if text != "DEBUG Next line" { + multiline_t.Logf("Event data incorrect [% X]", text) + multiline_t.FailNow() + } + + if start_offset != 6 { + multiline_t.Logf("Event start offset is incorrect [%d]", start_offset) + multiline_t.FailNow() + } + + if end_offset != 7 { + multiline_t.Logf("Event end offset is incorrect [%d]", end_offset) + multiline_t.FailNow() + } } func TestMultilinePrevious(t *testing.T) { - multiline_t = t - multiline_lines = 0 - - codec := createMultilineCodec(map[string]interface{}{ - "pattern": "^(ANOTHER|NEXT) ", - "what": "previous", - "negate": false, - }, checkMultiline, t) - - // Send some data - codec.Event(0, 1, "DEBUG First line") - codec.Event(2, 3, "NEXT line") - codec.Event(4, 5, "ANOTHER line") - codec.Event(6, 7, "DEBUG Next line") - - if multiline_lines != 1 { - t.Logf("Wrong line count received") - t.FailNow() - } + multiline_t = t + multiline_lines = 0 + + codec := createMultilineCodec(map[string]interface{}{ + "pattern": "^(ANOTHER|NEXT) ", + "what": "previous", + "negate": false, + }, checkMultiline, t) + + // Send some data + codec.Event(0, 1, "DEBUG First line") + codec.Event(2, 3, "NEXT line") + codec.Event(4, 5, "ANOTHER line") + codec.Event(6, 7, "DEBUG Next line") + + if multiline_lines != 1 { + t.Logf("Wrong line count received") + t.FailNow() + } + + offset := codec.Teardown() + if offset != 5 { + t.Error("Teardown returned incorrect offset: ", offset) + } } func TestMultilinePreviousNegate(t *testing.T) { - multiline_t = t - multiline_lines = 0 - - codec := createMultilineCodec(map[string]interface{}{ - "pattern": "^DEBUG ", - "what": "previous", - "negate": true, - }, checkMultiline, t) - - // Send some data - codec.Event(0, 1, "DEBUG First line") - codec.Event(2, 3, "NEXT line") - codec.Event(4, 5, "ANOTHER line") - codec.Event(6, 7, "DEBUG Next line") - - if multiline_lines != 1 { - t.Logf("Wrong line count received") - t.FailNow() - } + multiline_t = t + multiline_lines = 0 + + codec := createMultilineCodec(map[string]interface{}{ + "pattern": "^DEBUG ", + "what": "previous", + "negate": true, + }, checkMultiline, t) + + // Send some data + codec.Event(0, 1, "DEBUG First line") + codec.Event(2, 3, "NEXT line") + codec.Event(4, 5, "ANOTHER line") + codec.Event(6, 7, "DEBUG Next line") + + if multiline_lines != 1 { + t.Logf("Wrong line count received") + t.FailNow() + } + + offset := codec.Teardown() + if offset != 5 { + t.Error("Teardown returned incorrect offset: ", offset) + } } func TestMultilinePreviousTimeout(t *testing.T) { - multiline_t = t - multiline_lines = 0 - - codec := createMultilineCodec(map[string]interface{}{ - "pattern": "^(ANOTHER|NEXT) ", - "what": "previous", - "negate": false, - "previous timeout": "5s", - }, checkMultiline, t) - - // Send some data - codec.Event(0, 1, "DEBUG First line") - codec.Event(2, 3, "NEXT line") - codec.Event(4, 5, "ANOTHER line") - codec.Event(6, 7, "DEBUG Next line") - - // Allow 3 seconds - time.Sleep(3 * time.Second) - - multiline_lock.Lock() - if multiline_lines != 1 { - t.Logf("Timeout triggered too early") - t.FailNow() - } - multiline_lock.Unlock() - - // Allow 7 seconds - time.Sleep(7 * time.Second) - - multiline_lock.Lock() - if multiline_lines != 2 { - t.Logf("Wrong line count received") - t.FailNow() - } - multiline_lock.Unlock() - - codec.Teardown() + multiline_t = t + multiline_lines = 0 + + codec := createMultilineCodec(map[string]interface{}{ + "pattern": "^(ANOTHER|NEXT) ", + "what": "previous", + "negate": false, + "previous timeout": "3s", + }, checkMultiline, t) + + // Send some data + codec.Event(0, 1, "DEBUG First line") + codec.Event(2, 3, "NEXT line") + codec.Event(4, 5, "ANOTHER line") + codec.Event(6, 7, "DEBUG Next line") + + // Allow a second + time.Sleep(time.Second) + + multiline_lock.Lock() + if multiline_lines != 1 { + t.Logf("Timeout triggered too early") + t.FailNow() + } + multiline_lock.Unlock() + + // Allow 5 seconds + time.Sleep(5 * time.Second) + + multiline_lock.Lock() + if multiline_lines != 2 { + t.Logf("Wrong line count received") + t.FailNow() + } + multiline_lock.Unlock() + + offset := codec.Teardown() + if offset != 7 { + t.Error("Teardown returned incorrect offset: ", offset) + } } func TestMultilineNext(t *testing.T) { - multiline_t = t - multiline_lines = 0 - - codec := createMultilineCodec(map[string]interface{}{ - "pattern": "^(DEBUG|NEXT) ", - "what": "next", - "negate": false, - }, checkMultiline, t) - - // Send some data - codec.Event(0, 1, "DEBUG First line") - codec.Event(2, 3, "NEXT line") - codec.Event(4, 5, "ANOTHER line") - codec.Event(6, 7, "DEBUG Next line") - - if multiline_lines != 1 { - t.Logf("Wrong line count received") - t.FailNow() - } + multiline_t = t + multiline_lines = 0 + + codec := createMultilineCodec(map[string]interface{}{ + "pattern": "^(DEBUG|NEXT) ", + "what": "next", + "negate": false, + }, checkMultiline, t) + + // Send some data + codec.Event(0, 1, "DEBUG First line") + codec.Event(2, 3, "NEXT line") + codec.Event(4, 5, "ANOTHER line") + codec.Event(6, 7, "DEBUG Next line") + + if multiline_lines != 1 { + t.Logf("Wrong line count received") + t.FailNow() + } + + offset := codec.Teardown() + if offset != 5 { + t.Error("Teardown returned incorrect offset: ", offset) + } } func TestMultilineNextNegate(t *testing.T) { - multiline_t = t - multiline_lines = 0 - - codec := createMultilineCodec(map[string]interface{}{ - "pattern": "^ANOTHER ", - "what": "next", - "negate": true, - }, checkMultiline, t) - - // Send some data - codec.Event(0, 1, "DEBUG First line") - codec.Event(2, 3, "NEXT line") - codec.Event(4, 5, "ANOTHER line") - codec.Event(6, 7, "DEBUG Next line") - - if multiline_lines != 1 { - t.Logf("Wrong line count received") - t.FailNow() - } + multiline_t = t + multiline_lines = 0 + + codec := createMultilineCodec(map[string]interface{}{ + "pattern": "^ANOTHER ", + "what": "next", + "negate": true, + }, checkMultiline, t) + + // Send some data + codec.Event(0, 1, "DEBUG First line") + codec.Event(2, 3, "NEXT line") + codec.Event(4, 5, "ANOTHER line") + codec.Event(6, 7, "DEBUG Next line") + + if multiline_lines != 1 { + t.Logf("Wrong line count received") + t.FailNow() + } + + offset := codec.Teardown() + if offset != 5 { + t.Error("Teardown returned incorrect offset: ", offset) + } } func checkMultilineMaxBytes(start_offset int64, end_offset int64, text string) { - multiline_lines++ + multiline_lines++ - if multiline_lines == 1 { - if text != "DEBUG First line\nsecond line\nthi" { - multiline_t.Logf("Event data incorrect [% X]", text) - multiline_t.FailNow() - } + if multiline_lines == 1 { + if text != "DEBUG First line\nsecond line\nthi" { + multiline_t.Logf("Event data incorrect [% X]", text) + multiline_t.FailNow() + } - return - } + return + } - if text != "rd line" { - multiline_t.Logf("Second event data incorrect [% X]", text) - multiline_t.FailNow() - } + if text != "rd line" { + multiline_t.Logf("Second event data incorrect [% X]", text) + multiline_t.FailNow() + } } func TestMultilineMaxBytes(t *testing.T) { - multiline_t = t - multiline_lines = 0 - - codec := createMultilineCodec(map[string]interface{}{ - "max multiline bytes": int64(32), - "pattern": "^DEBUG ", - "negate": true, - }, checkMultilineMaxBytes, t) - - // Send some data - codec.Event(0, 1, "DEBUG First line") - codec.Event(2, 3, "second line") - codec.Event(4, 5, "third line") - codec.Event(6, 7, "DEBUG Next line") - - if multiline_lines != 2 { - t.Logf("Wrong line count received") - t.FailNow() - } + multiline_t = t + multiline_lines = 0 + + codec := createMultilineCodec(map[string]interface{}{ + "max multiline bytes": int64(32), + "pattern": "^DEBUG ", + "negate": true, + }, checkMultilineMaxBytes, t) + + // Send some data + codec.Event(0, 1, "DEBUG First line") + codec.Event(2, 3, "second line") + codec.Event(4, 5, "third line") + codec.Event(6, 7, "DEBUG Next line") + + if multiline_lines != 2 { + t.Logf("Wrong line count received") + t.FailNow() + } + + offset := codec.Teardown() + if offset != 5 { + t.Error("Teardown returned incorrect offset: ", offset) + } } diff --git a/src/lc-lib/codecs/plain.go b/src/lc-lib/codecs/plain.go index 865e70e8..00ac97e2 100644 --- a/src/lc-lib/codecs/plain.go +++ b/src/lc-lib/codecs/plain.go @@ -17,49 +17,49 @@ package codecs import ( - "lc-lib/core" + "github.com/driskell/log-courier/src/lc-lib/core" ) type CodecPlainFactory struct { } type CodecPlain struct { - last_offset int64 - callback_func core.CodecCallbackFunc + last_offset int64 + callback_func core.CodecCallbackFunc } func NewPlainCodecFactory(config *core.Config, config_path string, unused map[string]interface{}, name string) (core.CodecFactory, error) { - if err := config.ReportUnusedConfig(config_path, unused); err != nil { - return nil, err - } - return &CodecPlainFactory{}, nil + if err := config.ReportUnusedConfig(config_path, unused); err != nil { + return nil, err + } + return &CodecPlainFactory{}, nil } func (f *CodecPlainFactory) NewCodec(callback_func core.CodecCallbackFunc, offset int64) core.Codec { - return &CodecPlain{ - last_offset: offset, - callback_func: callback_func, - } + return &CodecPlain{ + last_offset: offset, + callback_func: callback_func, + } } func (c *CodecPlain) Teardown() int64 { - return c.last_offset + return c.last_offset } func (c *CodecPlain) Event(start_offset int64, end_offset int64, text string) { - c.last_offset = end_offset + c.last_offset = end_offset - c.callback_func(start_offset, end_offset, text) + c.callback_func(start_offset, end_offset, text) } func (c *CodecPlain) Meter() { } func (c *CodecPlain) Snapshot() *core.Snapshot { - return nil + return nil } // Register the codec func init() { - core.RegisterCodec("plain", NewPlainCodecFactory) + core.RegisterCodec("plain", NewPlainCodecFactory) } diff --git a/src/lc-lib/core/codec.go b/src/lc-lib/core/codec.go index e4d73571..5c2cf03a 100644 --- a/src/lc-lib/core/codec.go +++ b/src/lc-lib/core/codec.go @@ -17,16 +17,16 @@ package core type Codec interface { - Teardown() int64 - Event(int64, int64, string) - Meter() - Snapshot() *Snapshot + Teardown() int64 + Event(int64, int64, string) + Meter() + Snapshot() *Snapshot } type CodecCallbackFunc func(int64, int64, string) type CodecFactory interface { - NewCodec(CodecCallbackFunc, int64) Codec + NewCodec(CodecCallbackFunc, int64) Codec } type CodecRegistrarFunc func(*Config, string, map[string]interface{}, string) (CodecFactory, error) @@ -34,13 +34,13 @@ type CodecRegistrarFunc func(*Config, string, map[string]interface{}, string) (C var registered_Codecs map[string]CodecRegistrarFunc = make(map[string]CodecRegistrarFunc) func RegisterCodec(codec string, registrar_func CodecRegistrarFunc) { - registered_Codecs[codec] = registrar_func + registered_Codecs[codec] = registrar_func } func AvailableCodecs() (ret []string) { - ret = make([]string, 0, len(registered_Codecs)) - for k := range registered_Codecs { - ret = append(ret, k) - } - return + ret = make([]string, 0, len(registered_Codecs)) + for k := range registered_Codecs { + ret = append(ret, k) + } + return } diff --git a/src/lc-lib/core/config.go b/src/lc-lib/core/config.go index a63dad52..65eb05ca 100644 --- a/src/lc-lib/core/config.go +++ b/src/lc-lib/core/config.go @@ -20,350 +20,396 @@ package core import ( - "bytes" - "encoding/json" - "fmt" - "github.com/op/go-logging" - "math" - "os" - "path/filepath" - "reflect" - "time" + "bytes" + "encoding/json" + "fmt" + "github.com/op/go-logging" + "math" + "os" + "path/filepath" + "reflect" + "strings" + "time" ) const ( - default_GeneralConfig_AdminEnabled bool = false - default_GeneralConfig_AdminBind string = "tcp:127.0.0.1:1234" - default_GeneralConfig_PersistDir string = "." - default_GeneralConfig_ProspectInterval time.Duration = 10 * time.Second - default_GeneralConfig_SpoolSize int64 = 1024 - default_GeneralConfig_SpoolMaxBytes int64 = 10485760 - default_GeneralConfig_SpoolTimeout time.Duration = 5 * time.Second - default_GeneralConfig_LineBufferBytes int64 = 16384 - default_GeneralConfig_MaxLineBytes int64 = 1048576 - default_GeneralConfig_LogLevel logging.Level = logging.INFO - default_GeneralConfig_LogStdout bool = true - default_GeneralConfig_LogSyslog bool = false - default_NetworkConfig_Transport string = "tls" - default_NetworkConfig_Timeout time.Duration = 15 * time.Second - default_NetworkConfig_Reconnect time.Duration = 1 * time.Second - default_NetworkConfig_MaxPendingPayloads int64 = 10 - default_FileConfig_Codec string = "plain" - default_FileConfig_DeadTime int64 = 86400 + default_GeneralConfig_AdminEnabled bool = false + default_GeneralConfig_AdminBind string = "tcp:127.0.0.1:1234" + default_GeneralConfig_PersistDir string = "." + default_GeneralConfig_ProspectInterval time.Duration = 10 * time.Second + default_GeneralConfig_SpoolSize int64 = 1024 + default_GeneralConfig_SpoolMaxBytes int64 = 10485760 + default_GeneralConfig_SpoolTimeout time.Duration = 5 * time.Second + default_GeneralConfig_LineBufferBytes int64 = 16384 + default_GeneralConfig_MaxLineBytes int64 = 1048576 + default_GeneralConfig_LogLevel logging.Level = logging.INFO + default_GeneralConfig_LogStdout bool = true + default_GeneralConfig_LogSyslog bool = false + default_NetworkConfig_Transport string = "tls" + default_NetworkConfig_Rfc2782Srv bool = true + default_NetworkConfig_Rfc2782Service string = "courier" + default_NetworkConfig_Timeout time.Duration = 15 * time.Second + default_NetworkConfig_Reconnect time.Duration = 1 * time.Second + default_NetworkConfig_MaxPendingPayloads int64 = 10 + default_StreamConfig_Codec string = "plain" + default_StreamConfig_DeadTime int64 = 86400 ) var ( - default_GeneralConfig_Host string = "localhost.localdomain" + default_GeneralConfig_Host string = "localhost.localdomain" ) type Config struct { - General GeneralConfig `config:"general"` - Network NetworkConfig `config:"network"` - Files []FileConfig `config:"files"` - Includes []string `config:"includes"` + General GeneralConfig `config:"general"` + Network NetworkConfig `config:"network"` + Files []FileConfig `config:"files"` + Includes []string `config:"includes"` + Stdin StreamConfig `config:"stdin"` } type GeneralConfig struct { - AdminEnabled bool `config:"admin enabled"` - AdminBind string `config:"admin listen address"` - PersistDir string `config:"persist directory"` - ProspectInterval time.Duration `config:"prospect interval"` - SpoolSize int64 `config:"spool size"` - SpoolMaxBytes int64 `config:"spool max bytes"` - SpoolTimeout time.Duration `config:"spool timeout"` - LineBufferBytes int64 `config:"line buffer bytes"` - MaxLineBytes int64 `config:"max line bytes"` - LogLevel logging.Level `config:"log level"` - LogStdout bool `config:"log stdout"` - LogSyslog bool `config:"log syslog"` - LogFile string `config:"log file"` - Host string `config:"host"` + AdminEnabled bool `config:"admin enabled"` + AdminBind string `config:"admin listen address"` + PersistDir string `config:"persist directory"` + ProspectInterval time.Duration `config:"prospect interval"` + SpoolSize int64 `config:"spool size"` + SpoolMaxBytes int64 `config:"spool max bytes"` + SpoolTimeout time.Duration `config:"spool timeout"` + LineBufferBytes int64 `config:"line buffer bytes"` + MaxLineBytes int64 `config:"max line bytes"` + LogLevel logging.Level `config:"log level"` + LogStdout bool `config:"log stdout"` + LogSyslog bool `config:"log syslog"` + LogFile string `config:"log file"` + Host string `config:"host"` } type NetworkConfig struct { - Transport string `config:"transport"` - Servers []string `config:"servers"` - Timeout time.Duration `config:"timeout"` - Reconnect time.Duration `config:"reconnect"` - MaxPendingPayloads int64 `config:"max pending payloads"` - - Unused map[string]interface{} - TransportFactory TransportFactory + Transport string `config:"transport"` + Servers []string `config:"servers"` + Rfc2782Srv bool `config:"rfc 2782 srv"` + Rfc2782Service string `config:"rfc 2782 service"` + Timeout time.Duration `config:"timeout"` + Reconnect time.Duration `config:"reconnect"` + MaxPendingPayloads int64 `config:"max pending payloads"` + + Unused map[string]interface{} + TransportFactory TransportFactory } type CodecConfigStub struct { - Name string `config:"name"` + Name string `config:"name"` - Unused map[string]interface{} + Unused map[string]interface{} +} + +type StreamConfig struct { + Fields map[string]interface{} `config:"fields"` + Codec CodecConfigStub `config:"codec"` + DeadTime time.Duration `config:"dead time"` + + CodecFactory CodecFactory } type FileConfig struct { - Paths []string `config:"paths"` - Fields map[string]interface{} `config:"fields"` - Codec CodecConfigStub `config:"codec"` - DeadTime time.Duration `config:"dead time"` + Paths []string `config:"paths"` - CodecFactory CodecFactory + StreamConfig `config:",embed"` } func NewConfig() *Config { - return &Config{} + return &Config{} } func (c *Config) loadFile(path string) (stripped *bytes.Buffer, err error) { - stripped = new(bytes.Buffer) - - file, err := os.Open(path) - if err != nil { - err = fmt.Errorf("Failed to open config file: %s", err) - return - } - defer file.Close() - - stat, err := file.Stat() - if err != nil { - err = fmt.Errorf("Stat failed for config file: %s", err) - return - } - if stat.Size() > (10 << 20) { - err = fmt.Errorf("Config file too large (%s)", stat.Size()) - return - } - - // Strip comments and read config into stripped - var s, p, state int - { - // Pull the config file into memory - buffer := make([]byte, stat.Size()) - _, err = file.Read(buffer) - if err != nil { - return - } - - for p < len(buffer) { - b := buffer[p] - if state == 0 { - // Main body - if b == '"' { - state = 1 - } else if b == '\'' { - state = 2 - } else if b == '#' { - state = 3 - stripped.Write(buffer[s:p]) - } else if b == '/' { - state = 4 - } - } else if state == 1 { - // Double-quoted string - if b == '\\' { - state = 5 - } else if b == '"' { - state = 0 - } - } else if state == 2 { - // Single-quoted string - if b == '\\' { - state = 6 - } else if b == '\'' { - state = 0 - } - } else if state == 3 { - // End of line comment (#) - if b == '\r' || b == '\n' { - state = 0 - s = p + 1 - } - } else if state == 4 { - // Potential start of multiline comment - if b == '*' { - state = 7 - stripped.Write(buffer[s : p-1]) - } else { - state = 0 - } - } else if state == 5 { - // Escape within double quote - state = 1 - } else if state == 6 { - // Escape within single quote - state = 2 - } else if state == 7 { - // Multiline comment (/**/) - if b == '*' { - state = 8 - } - } else { // state == 8 - // Potential end of multiline comment - if b == '/' { - state = 0 - s = p + 1 - } else { - state = 7 - } - } - p++ - } - stripped.Write(buffer[s:p]) - } - - return + stripped = new(bytes.Buffer) + + file, err := os.Open(path) + if err != nil { + err = fmt.Errorf("Failed to open config file: %s", err) + return + } + defer file.Close() + + stat, err := file.Stat() + if err != nil { + err = fmt.Errorf("Stat failed for config file: %s", err) + return + } + if stat.Size() > (10 << 20) { + err = fmt.Errorf("Config file too large (%s)", stat.Size()) + return + } + + // Strip comments and read config into stripped + var s, p, state int + { + // Pull the config file into memory + buffer := make([]byte, stat.Size()) + _, err = file.Read(buffer) + if err != nil { + return + } + + for p < len(buffer) { + b := buffer[p] + if state == 0 { + // Main body + if b == '"' { + state = 1 + } else if b == '\'' { + state = 2 + } else if b == '#' { + state = 3 + stripped.Write(buffer[s:p]) + } else if b == '/' { + state = 4 + } + } else if state == 1 { + // Double-quoted string + if b == '\\' { + state = 5 + } else if b == '"' { + state = 0 + } + } else if state == 2 { + // Single-quoted string + if b == '\\' { + state = 6 + } else if b == '\'' { + state = 0 + } + } else if state == 3 { + // End of line comment (#) + if b == '\r' || b == '\n' { + state = 0 + s = p + 1 + } + } else if state == 4 { + // Potential start of multiline comment + if b == '*' { + state = 7 + stripped.Write(buffer[s : p-1]) + } else { + state = 0 + } + } else if state == 5 { + // Escape within double quote + state = 1 + } else if state == 6 { + // Escape within single quote + state = 2 + } else if state == 7 { + // Multiline comment (/**/) + if b == '*' { + state = 8 + } + } else { // state == 8 + // Potential end of multiline comment + if b == '/' { + state = 0 + s = p + 1 + } else { + state = 7 + } + } + p++ + } + stripped.Write(buffer[s:p]) + } + + return +} + +// Parse a *json.SyntaxError into a pretty error message +func (c *Config) parseSyntaxError(js []byte, err error) error { + json_err, ok := err.(*json.SyntaxError) + if !ok { + return err + } + + start := bytes.LastIndex(js[:json_err.Offset], []byte("\n"))+1 + end := bytes.Index(js[start:], []byte("\n")) + if end >= 0 { + end += start + } else { + end = len(js) + } + + line, pos := bytes.Count(js[:start], []byte("\n")), int(json_err.Offset) - start - 1 + + return fmt.Errorf("%s on line %d\n%s\n%s^", err, line, js[start:end], strings.Repeat(" ", pos)) } // TODO: Config from a TOML? Maybe a custom one func (c *Config) Load(path string) (err error) { - var data *bytes.Buffer - - // Read the main config file - if data, err = c.loadFile(path); err != nil { - return - } - - // Pull the entire structure into raw_config - raw_config := make(map[string]interface{}) - if err = json.Unmarshal(data.Bytes(), &raw_config); err != nil { - return - } - - // Fill in defaults where the zero-value is a valid setting - c.General.AdminEnabled = default_GeneralConfig_AdminEnabled - c.General.LogLevel = default_GeneralConfig_LogLevel - c.General.LogStdout = default_GeneralConfig_LogStdout - c.General.LogSyslog = default_GeneralConfig_LogSyslog - - // Populate configuration - reporting errors on spelling mistakes etc. - if err = c.PopulateConfig(c, "/", raw_config); err != nil { - return - } - - // Iterate includes - for _, glob := range c.Includes { - // Glob the path - var matches []string - if matches, err = filepath.Glob(glob); err != nil { - return - } - - for _, include := range matches { - // Read the include - if data, err = c.loadFile(include); err != nil { - return - } - - // Pull the structure into raw_include - raw_include := make([]interface{}, 0) - if err = json.Unmarshal(data.Bytes(), &raw_include); err != nil { - return - } - - // Append to configuration - if err = c.populateConfigSlice(reflect.ValueOf(c).Elem().FieldByName("Files"), fmt.Sprintf("%s/", include), raw_include); err != nil { - return - } - } - } - - if c.General.AdminBind == "" { - c.General.AdminBind = default_GeneralConfig_AdminBind - } - - if c.General.PersistDir == "" { - c.General.PersistDir = default_GeneralConfig_PersistDir - } - - if c.General.ProspectInterval == time.Duration(0) { - c.General.ProspectInterval = default_GeneralConfig_ProspectInterval - } - - if c.General.SpoolSize == 0 { - c.General.SpoolSize = default_GeneralConfig_SpoolSize - } - - // Enforce maximum of 2 GB since event transmit length is uint32 - if c.General.SpoolMaxBytes == 0 { - c.General.SpoolMaxBytes = default_GeneralConfig_SpoolMaxBytes - } - if c.General.SpoolMaxBytes > 2*1024*1024*1024 { - err = fmt.Errorf("/general/spool max bytes can not be greater than 2 GiB") - return - } - - if c.General.SpoolTimeout == time.Duration(0) { - c.General.SpoolTimeout = default_GeneralConfig_SpoolTimeout - } - - if c.General.LineBufferBytes == 0 { - c.General.LineBufferBytes = default_GeneralConfig_LineBufferBytes - } - if c.General.LineBufferBytes < 1 { - err = fmt.Errorf("/general/line buffer bytes must be greater than 1") - return - } - - // Max line bytes can not be larger than spool max bytes - if c.General.MaxLineBytes == 0 { - c.General.MaxLineBytes = default_GeneralConfig_MaxLineBytes - } - if c.General.MaxLineBytes > c.General.SpoolMaxBytes { - err = fmt.Errorf("/general/max line bytes can not be greater than /general/spool max bytes") - return - } - - if c.General.Host == "" { - ret, err := os.Hostname() - if err == nil { - c.General.Host = ret - } else { - c.General.Host = default_GeneralConfig_Host - log.Warning("Failed to determine the FQDN; using '%s'.", c.General.Host) - } - } - - if c.Network.Transport == "" { - c.Network.Transport = default_NetworkConfig_Transport - } - - if registrar_func, ok := registered_Transports[c.Network.Transport]; ok { - if c.Network.TransportFactory, err = registrar_func(c, "/network/", c.Network.Unused, c.Network.Transport); err != nil { - return - } - } else { - err = fmt.Errorf("Unrecognised transport '%s'", c.Network.Transport) - return - } - - if c.Network.Timeout == time.Duration(0) { - c.Network.Timeout = default_NetworkConfig_Timeout - } - if c.Network.Reconnect == time.Duration(0) { - c.Network.Reconnect = default_NetworkConfig_Reconnect - } - if c.Network.MaxPendingPayloads == 0 { - c.Network.MaxPendingPayloads = default_NetworkConfig_MaxPendingPayloads - } - - for k := range c.Files { - if c.Files[k].Codec.Name == "" { - c.Files[k].Codec.Name = default_FileConfig_Codec - } - - if registrar_func, ok := registered_Codecs[c.Files[k].Codec.Name]; ok { - if c.Files[k].CodecFactory, err = registrar_func(c, fmt.Sprintf("/files[%d]/codec/", k), c.Files[k].Codec.Unused, c.Files[k].Codec.Name); err != nil { - return - } - } else { - err = fmt.Errorf("Unrecognised codec '%s' for 'files' entry %d", c.Files[k].Codec.Name, k) - return - } - - if c.Files[k].DeadTime == time.Duration(0) { - c.Files[k].DeadTime = time.Duration(default_FileConfig_DeadTime) * time.Second - } - - // TODO: Event transmit length is uint32, if fields length is rediculous we will fail - } - - return + var data *bytes.Buffer + + // Read the main config file + if data, err = c.loadFile(path); err != nil { + return + } + + // Pull the entire structure into raw_config + raw_config := make(map[string]interface{}) + if err = json.Unmarshal(data.Bytes(), &raw_config); err != nil { + return c.parseSyntaxError(data.Bytes(), err) + } + + // Fill in defaults where the zero-value is a valid setting + c.General.AdminEnabled = default_GeneralConfig_AdminEnabled + c.General.LogLevel = default_GeneralConfig_LogLevel + c.General.LogStdout = default_GeneralConfig_LogStdout + c.General.LogSyslog = default_GeneralConfig_LogSyslog + c.Network.Rfc2782Srv = default_NetworkConfig_Rfc2782Srv + + // Populate configuration - reporting errors on spelling mistakes etc. + if err = c.PopulateConfig(c, "/", raw_config); err != nil { + return + } + + // Iterate includes + for _, glob := range c.Includes { + // Glob the path + var matches []string + if matches, err = filepath.Glob(glob); err != nil { + return + } + + for _, include := range matches { + // Read the include + if data, err = c.loadFile(include); err != nil { + return + } + + // Pull the structure into raw_include + raw_include := make([]interface{}, 0) + if err = json.Unmarshal(data.Bytes(), &raw_include); err != nil { + return + } + + // Append to configuration + if err = c.populateConfigSlice(reflect.ValueOf(c).Elem().FieldByName("Files"), fmt.Sprintf("%s/", include), raw_include); err != nil { + return + } + } + } + + if c.General.AdminBind == "" { + c.General.AdminBind = default_GeneralConfig_AdminBind + } + + if c.General.PersistDir == "" { + c.General.PersistDir = default_GeneralConfig_PersistDir + } + + if c.General.ProspectInterval == time.Duration(0) { + c.General.ProspectInterval = default_GeneralConfig_ProspectInterval + } + + if c.General.SpoolSize == 0 { + c.General.SpoolSize = default_GeneralConfig_SpoolSize + } + + // Enforce maximum of 2 GB since event transmit length is uint32 + if c.General.SpoolMaxBytes == 0 { + c.General.SpoolMaxBytes = default_GeneralConfig_SpoolMaxBytes + } + if c.General.SpoolMaxBytes > 2*1024*1024*1024 { + err = fmt.Errorf("/general/spool max bytes can not be greater than 2 GiB") + return + } + + if c.General.SpoolTimeout == time.Duration(0) { + c.General.SpoolTimeout = default_GeneralConfig_SpoolTimeout + } + + if c.General.LineBufferBytes == 0 { + c.General.LineBufferBytes = default_GeneralConfig_LineBufferBytes + } + if c.General.LineBufferBytes < 1 { + err = fmt.Errorf("/general/line buffer bytes must be greater than 1") + return + } + + // Max line bytes can not be larger than spool max bytes + if c.General.MaxLineBytes == 0 { + c.General.MaxLineBytes = default_GeneralConfig_MaxLineBytes + } + if c.General.MaxLineBytes > c.General.SpoolMaxBytes { + err = fmt.Errorf("/general/max line bytes can not be greater than /general/spool max bytes") + return + } + + if c.General.Host == "" { + ret, err := os.Hostname() + if err == nil { + c.General.Host = ret + } else { + c.General.Host = default_GeneralConfig_Host + log.Warning("Failed to determine the FQDN; using '%s'.", c.General.Host) + } + } + + if c.Network.Transport == "" { + c.Network.Transport = default_NetworkConfig_Transport + } + + if registrar_func, ok := registered_Transports[c.Network.Transport]; ok { + if c.Network.TransportFactory, err = registrar_func(c, "/network/", c.Network.Unused, c.Network.Transport); err != nil { + return + } + } else { + err = fmt.Errorf("Unrecognised transport '%s'", c.Network.Transport) + return + } + + if c.Network.Rfc2782Service == "" { + c.Network.Rfc2782Service = default_NetworkConfig_Rfc2782Service + } + if c.Network.Timeout == time.Duration(0) { + c.Network.Timeout = default_NetworkConfig_Timeout + } + if c.Network.Reconnect == time.Duration(0) { + c.Network.Reconnect = default_NetworkConfig_Reconnect + } + if c.Network.MaxPendingPayloads == 0 { + c.Network.MaxPendingPayloads = default_NetworkConfig_MaxPendingPayloads + } + + for k := range c.Files { + if err = c.initStreamConfig(fmt.Sprintf("/files[%d]/codec/", k), &c.Files[k].StreamConfig); err != nil { + return + } + } + + if err = c.initStreamConfig("/stdin", &c.Stdin); err != nil { + return + } + + return +} + +func (c *Config) initStreamConfig(path string, stream_config *StreamConfig) (err error) { + if stream_config.Codec.Name == "" { + stream_config.Codec.Name = default_StreamConfig_Codec + } + + if registrar_func, ok := registered_Codecs[stream_config.Codec.Name]; ok { + if stream_config.CodecFactory, err = registrar_func(c, path, stream_config.Codec.Unused, stream_config.Codec.Name); err != nil { + return + } + } else { + return fmt.Errorf("Unrecognised codec '%s' for %s", stream_config.Codec.Name, path) + } + + if stream_config.DeadTime == time.Duration(0) { + stream_config.DeadTime = time.Duration(default_StreamConfig_DeadTime) * time.Second + } + + // TODO: EDGE CASE: Event transmit length is uint32, if fields length is rediculous we will fail + + return nil } // TODO: This should be pushed to a wrapper or module @@ -372,151 +418,163 @@ func (c *Config) Load(path string) (err error) { // or an error is reported if "Unused" is not available // We can then take the unused configuration dynamically at runtime based on another value func (c *Config) PopulateConfig(config interface{}, config_path string, raw_config map[string]interface{}) (err error) { - vconfig := reflect.ValueOf(config).Elem() - for i := 0; i < vconfig.NumField(); i++ { - field := vconfig.Field(i) - if !field.CanSet() { - continue - } - fieldtype := vconfig.Type().Field(i) - tag := fieldtype.Tag.Get("config") - if tag == "" { - continue - } - if _, ok := raw_config[tag]; ok { - if field.Kind() == reflect.Struct { - if reflect.TypeOf(raw_config[tag]).Kind() == reflect.Map { - if err = c.PopulateConfig(field.Addr().Interface(), fmt.Sprintf("%s%s/", config_path, tag), raw_config[tag].(map[string]interface{})); err != nil { - return - } - delete(raw_config, tag) - continue - } else { - err = fmt.Errorf("Option %s%s must be a hash", config_path, tag) - return - } - } - value := reflect.ValueOf(raw_config[tag]) - if value.Type().AssignableTo(field.Type()) { - field.Set(value) - } else if value.Kind() == reflect.Slice && field.Kind() == reflect.Slice { - if err = c.populateConfigSlice(field, fmt.Sprintf("%s%s/", config_path, tag), raw_config[tag].([]interface{})); err != nil { - return - } - } else if value.Kind() == reflect.Map && field.Kind() == reflect.Map { - if field.IsNil() { - field.Set(reflect.MakeMap(field.Type())) - } - for _, j := range value.MapKeys() { - item := value.MapIndex(j) - if item.Elem().Type().AssignableTo(field.Type().Elem()) { - field.SetMapIndex(j, item.Elem()) - } else { - err = fmt.Errorf("Option %s%s[%s] must be %s or similar", config_path, tag, j, field.Type().Elem()) - return - } - } - } else if field.Type().String() == "time.Duration" { - var duration float64 - vduration := reflect.ValueOf(duration) - fail := true - if value.Type().AssignableTo(vduration.Type()) { - duration = value.Float() - if duration >= math.MinInt64 && duration <= math.MaxInt64 { - field.Set(reflect.ValueOf(time.Duration(int64(duration)) * time.Second)) - fail = false - } - } else if value.Kind() == reflect.String { - var tduration time.Duration - if tduration, err = time.ParseDuration(value.String()); err == nil { - field.Set(reflect.ValueOf(tduration)) - fail = false - } - } - if fail { - err = fmt.Errorf("Option %s%s must be a valid numeric or string duration", config_path, tag) - return - } - } else if field.Type().String() == "logging.Level" { - fail := true - if value.Kind() == reflect.String { - var llevel logging.Level - if llevel, err = logging.LogLevel(value.String()); err == nil { - fail = false - field.Set(reflect.ValueOf(llevel)) - } - } - if fail { - err = fmt.Errorf("Option %s%s is not a valid log level (critical, error, warning, notice, info, debug)", config_path, tag) - return - } - } else if field.Kind() == reflect.Int64 { - fail := true - if value.Kind() == reflect.Float64 { - number := value.Float() - if math.Floor(number) == number { - fail = false - field.Set(reflect.ValueOf(int64(number))) - } - } - if fail { - err = fmt.Errorf("Option %s%s is not a valid integer", config_path, tag, field.Type()) - return - } - } else if field.Kind() == reflect.Int { - fail := true - if value.Kind() == reflect.Float64 { - number := value.Float() - if math.Floor(number) == number { - fail = false - field.Set(reflect.ValueOf(int(number))) - } - } - if fail { - err = fmt.Errorf("Option %s%s is not a valid integer", config_path, tag, field.Type()) - return - } - } else { - err = fmt.Errorf("Option %s%s must be %s or similar (%s provided)", config_path, tag, field.Type(), value.Type()) - return - } - delete(raw_config, tag) - } - } - if unused := vconfig.FieldByName("Unused"); unused.IsValid() { - if unused.IsNil() { - unused.Set(reflect.MakeMap(unused.Type())) - } - for k, v := range raw_config { - unused.SetMapIndex(reflect.ValueOf(k), reflect.ValueOf(v)) - } - return - } - return c.ReportUnusedConfig(config_path, raw_config) + vconfig := reflect.ValueOf(config).Elem() +FieldLoop: + for i := 0; i < vconfig.NumField(); i++ { + field := vconfig.Field(i) + if !field.CanSet() { + continue + } + fieldtype := vconfig.Type().Field(i) + tag := fieldtype.Tag.Get("config") + mods := strings.Split(tag, ",") + tag = mods[0] + mods = mods[1:] + for _, mod := range mods { + if mod == "embed" && field.Kind() == reflect.Struct { + if err = c.PopulateConfig(field.Addr().Interface(), config_path, raw_config); err != nil { + return + } + continue FieldLoop + } + } + if tag == "" { + continue + } + if _, ok := raw_config[tag]; ok { + if field.Kind() == reflect.Struct { + if reflect.TypeOf(raw_config[tag]).Kind() == reflect.Map { + if err = c.PopulateConfig(field.Addr().Interface(), fmt.Sprintf("%s%s/", config_path, tag), raw_config[tag].(map[string]interface{})); err != nil { + return + } + delete(raw_config, tag) + continue + } else { + err = fmt.Errorf("Option %s%s must be a hash", config_path, tag) + return + } + } + value := reflect.ValueOf(raw_config[tag]) + if value.Type().AssignableTo(field.Type()) { + field.Set(value) + } else if value.Kind() == reflect.Slice && field.Kind() == reflect.Slice { + if err = c.populateConfigSlice(field, fmt.Sprintf("%s%s/", config_path, tag), raw_config[tag].([]interface{})); err != nil { + return + } + } else if value.Kind() == reflect.Map && field.Kind() == reflect.Map { + if field.IsNil() { + field.Set(reflect.MakeMap(field.Type())) + } + for _, j := range value.MapKeys() { + item := value.MapIndex(j) + if item.Elem().Type().AssignableTo(field.Type().Elem()) { + field.SetMapIndex(j, item.Elem()) + } else { + err = fmt.Errorf("Option %s%s[%s] must be %s or similar", config_path, tag, j, field.Type().Elem()) + return + } + } + } else if field.Type().String() == "time.Duration" { + var duration float64 + vduration := reflect.ValueOf(duration) + fail := true + if value.Type().AssignableTo(vduration.Type()) { + duration = value.Float() + if duration >= math.MinInt64 && duration <= math.MaxInt64 { + field.Set(reflect.ValueOf(time.Duration(int64(duration)) * time.Second)) + fail = false + } + } else if value.Kind() == reflect.String { + var tduration time.Duration + if tduration, err = time.ParseDuration(value.String()); err == nil { + field.Set(reflect.ValueOf(tduration)) + fail = false + } + } + if fail { + err = fmt.Errorf("Option %s%s must be a valid numeric or string duration", config_path, tag) + return + } + } else if field.Type().String() == "logging.Level" { + fail := true + if value.Kind() == reflect.String { + var llevel logging.Level + if llevel, err = logging.LogLevel(value.String()); err == nil { + fail = false + field.Set(reflect.ValueOf(llevel)) + } + } + if fail { + err = fmt.Errorf("Option %s%s is not a valid log level (critical, error, warning, notice, info, debug)", config_path, tag) + return + } + } else if field.Kind() == reflect.Int64 { + fail := true + if value.Kind() == reflect.Float64 { + number := value.Float() + if math.Floor(number) == number { + fail = false + field.Set(reflect.ValueOf(int64(number))) + } + } + if fail { + err = fmt.Errorf("Option %s%s is not a valid integer", config_path, tag, field.Type()) + return + } + } else if field.Kind() == reflect.Int { + fail := true + if value.Kind() == reflect.Float64 { + number := value.Float() + if math.Floor(number) == number { + fail = false + field.Set(reflect.ValueOf(int(number))) + } + } + if fail { + err = fmt.Errorf("Option %s%s is not a valid integer", config_path, tag, field.Type()) + return + } + } else { + err = fmt.Errorf("Option %s%s must be %s or similar (%s provided)", config_path, tag, field.Type(), value.Type()) + return + } + delete(raw_config, tag) + } + } + if unused := vconfig.FieldByName("Unused"); unused.IsValid() { + if unused.IsNil() { + unused.Set(reflect.MakeMap(unused.Type())) + } + for k, v := range raw_config { + unused.SetMapIndex(reflect.ValueOf(k), reflect.ValueOf(v)) + } + return + } + return c.ReportUnusedConfig(config_path, raw_config) } func (c *Config) populateConfigSlice(field reflect.Value, config_path string, raw_config []interface{}) (err error) { - elemtype := field.Type().Elem() - if elemtype.Kind() == reflect.Struct { - for j := 0; j < len(raw_config); j++ { - item := reflect.New(elemtype) - if err = c.PopulateConfig(item.Interface(), fmt.Sprintf("%s[%d]/", config_path, j), raw_config[j].(map[string]interface{})); err != nil { - return - } - field.Set(reflect.Append(field, item.Elem())) - } - } else { - for j := 0; j < len(raw_config); j++ { - field.Set(reflect.Append(field, reflect.ValueOf(raw_config[j]))) - } - } - return + elemtype := field.Type().Elem() + if elemtype.Kind() == reflect.Struct { + for j := 0; j < len(raw_config); j++ { + item := reflect.New(elemtype) + if err = c.PopulateConfig(item.Interface(), fmt.Sprintf("%s[%d]/", config_path, j), raw_config[j].(map[string]interface{})); err != nil { + return + } + field.Set(reflect.Append(field, item.Elem())) + } + } else { + for j := 0; j < len(raw_config); j++ { + field.Set(reflect.Append(field, reflect.ValueOf(raw_config[j]))) + } + } + return } func (c *Config) ReportUnusedConfig(config_path string, raw_config map[string]interface{}) (err error) { - for k := range raw_config { - err = fmt.Errorf("Option %s%s is not available", config_path, k) - return - } - return + for k := range raw_config { + err = fmt.Errorf("Option %s%s is not available", config_path, k) + return + } + return } diff --git a/src/lc-lib/core/event.go b/src/lc-lib/core/event.go index 5b9c83a1..d11c0483 100644 --- a/src/lc-lib/core/event.go +++ b/src/lc-lib/core/event.go @@ -17,17 +17,17 @@ package core import ( - "encoding/json" + "encoding/json" ) type Event map[string]interface{} type EventDescriptor struct { - Stream Stream - Offset int64 - Event []byte + Stream Stream + Offset int64 + Event []byte } func (e *Event) Encode() ([]byte, error) { - return json.Marshal(e) + return json.Marshal(e) } diff --git a/src/lc-lib/core/logging.go b/src/lc-lib/core/logging.go index 7a5fd43c..515be966 100644 --- a/src/lc-lib/core/logging.go +++ b/src/lc-lib/core/logging.go @@ -12,7 +12,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. -*/ + */ package core diff --git a/src/lc-lib/core/pipeline.go b/src/lc-lib/core/pipeline.go index 480e4ddb..d22e8c2e 100644 --- a/src/lc-lib/core/pipeline.go +++ b/src/lc-lib/core/pipeline.go @@ -17,136 +17,136 @@ package core import ( - "sync" + "sync" ) type Pipeline struct { - pipes []IPipelineSegment - signal chan interface{} - group sync.WaitGroup - config_sinks map[*PipelineConfigReceiver]chan *Config - snapshot_chan chan []*Snapshot - snapshot_pipes map[IPipelineSnapshotProvider]IPipelineSnapshotProvider + pipes []IPipelineSegment + signal chan interface{} + group sync.WaitGroup + config_sinks map[*PipelineConfigReceiver]chan *Config + snapshot_chan chan []*Snapshot + snapshot_pipes map[IPipelineSnapshotProvider]IPipelineSnapshotProvider } func NewPipeline() *Pipeline { - return &Pipeline{ - pipes: make([]IPipelineSegment, 0, 5), - signal: make(chan interface{}), - config_sinks: make(map[*PipelineConfigReceiver]chan *Config), - snapshot_chan: make(chan []*Snapshot), - snapshot_pipes: make(map[IPipelineSnapshotProvider]IPipelineSnapshotProvider), - } + return &Pipeline{ + pipes: make([]IPipelineSegment, 0, 5), + signal: make(chan interface{}), + config_sinks: make(map[*PipelineConfigReceiver]chan *Config), + snapshot_chan: make(chan []*Snapshot), + snapshot_pipes: make(map[IPipelineSnapshotProvider]IPipelineSnapshotProvider), + } } func (p *Pipeline) Register(ipipe IPipelineSegment) { - p.group.Add(1) + p.group.Add(1) - pipe := ipipe.getStruct() - pipe.signal = p.signal - pipe.group = &p.group + pipe := ipipe.getStruct() + pipe.signal = p.signal + pipe.group = &p.group - p.pipes = append(p.pipes, ipipe) + p.pipes = append(p.pipes, ipipe) - if ipipe_ext, ok := ipipe.(IPipelineConfigReceiver); ok { - pipe_ext := ipipe_ext.getConfigReceiverStruct() - sink := make(chan *Config) - p.config_sinks[pipe_ext] = sink - pipe_ext.config_chan = sink - } + if ipipe_ext, ok := ipipe.(IPipelineConfigReceiver); ok { + pipe_ext := ipipe_ext.getConfigReceiverStruct() + sink := make(chan *Config) + p.config_sinks[pipe_ext] = sink + pipe_ext.config_chan = sink + } - if ipipe_ext, ok := ipipe.(IPipelineSnapshotProvider); ok { - p.snapshot_pipes[ipipe_ext] = ipipe_ext - } + if ipipe_ext, ok := ipipe.(IPipelineSnapshotProvider); ok { + p.snapshot_pipes[ipipe_ext] = ipipe_ext + } } func (p *Pipeline) Start() { - for _, ipipe := range p.pipes { - go ipipe.Run() - } + for _, ipipe := range p.pipes { + go ipipe.Run() + } } func (p *Pipeline) Shutdown() { - close(p.signal) + close(p.signal) } func (p *Pipeline) Wait() { - p.group.Wait() + p.group.Wait() } func (p *Pipeline) SendConfig(config *Config) { - for _, sink := range p.config_sinks { - sink <- config - } + for _, sink := range p.config_sinks { + sink <- config + } } func (p *Pipeline) Snapshot() *Snapshot { - snap := NewSnapshot("Log Courier") + snap := NewSnapshot("Log Courier") - for _, sink := range p.snapshot_pipes { - for _, sub := range sink.Snapshot() { - snap.AddSub(sub) - } - } + for _, sink := range p.snapshot_pipes { + for _, sub := range sink.Snapshot() { + snap.AddSub(sub) + } + } - return snap + return snap } type IPipelineSegment interface { - Run() - getStruct() *PipelineSegment + Run() + getStruct() *PipelineSegment } type PipelineSegment struct { - signal <-chan interface{} - group *sync.WaitGroup + signal <-chan interface{} + group *sync.WaitGroup } func (s *PipelineSegment) Run() { - panic("Run() not implemented") + panic("Run() not implemented") } func (s *PipelineSegment) getStruct() *PipelineSegment { - return s + return s } func (s *PipelineSegment) OnShutdown() <-chan interface{} { - return s.signal + return s.signal } func (s *PipelineSegment) Done() { - s.group.Done() + s.group.Done() } type IPipelineConfigReceiver interface { - getConfigReceiverStruct() *PipelineConfigReceiver + getConfigReceiverStruct() *PipelineConfigReceiver } type PipelineConfigReceiver struct { - config_chan <-chan *Config + config_chan <-chan *Config } func (s *PipelineConfigReceiver) getConfigReceiverStruct() *PipelineConfigReceiver { - return s + return s } func (s *PipelineConfigReceiver) OnConfig() <-chan *Config { - return s.config_chan + return s.config_chan } type IPipelineSnapshotProvider interface { - Snapshot() []*Snapshot + Snapshot() []*Snapshot } type PipelineSnapshotProvider struct { } func (s *PipelineSnapshotProvider) getSnapshotProviderStruct() *PipelineSnapshotProvider { - return s + return s } func (s *PipelineSnapshotProvider) Snapshot() []*Snapshot { - ret := NewSnapshot("Unknown") - ret.AddEntry("Error", "NotImplemented") - return []*Snapshot{ret} + ret := NewSnapshot("Unknown") + ret.AddEntry("Error", "NotImplemented") + return []*Snapshot{ret} } diff --git a/src/lc-lib/core/snapshot.go b/src/lc-lib/core/snapshot.go index 12551f41..ead2602f 100644 --- a/src/lc-lib/core/snapshot.go +++ b/src/lc-lib/core/snapshot.go @@ -12,80 +12,80 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. -*/ + */ package core import ( - "sort" + "sort" ) type Snapshot struct { - Desc string - Entries map[string]interface{} - Keys []string - Subs map[string]*Snapshot - SubKeys []string + Desc string + Entries map[string]interface{} + Keys []string + Subs map[string]*Snapshot + SubKeys []string } func NewSnapshot(desc string) *Snapshot { - return &Snapshot{ - Desc: desc, - Entries: make(map[string]interface{}), - Keys: make([]string, 0), - Subs: make(map[string]*Snapshot), - SubKeys: make([]string, 0), - } + return &Snapshot{ + Desc: desc, + Entries: make(map[string]interface{}), + Keys: make([]string, 0), + Subs: make(map[string]*Snapshot), + SubKeys: make([]string, 0), + } } func (s *Snapshot) Sort() { - sort.Strings(s.Keys) - sort.Strings(s.SubKeys) + sort.Strings(s.Keys) + sort.Strings(s.SubKeys) } func (s *Snapshot) Description() string { - return s.Desc + return s.Desc } func (s *Snapshot) AddEntry(name string, value interface{}) { - s.Entries[name] = value - s.Keys = append(s.Keys, name) + s.Entries[name] = value + s.Keys = append(s.Keys, name) } func (s *Snapshot) EntryByName(name string) (interface{}, bool) { - if v, ok := s.Entries[name]; ok { - return v, true - } + if v, ok := s.Entries[name]; ok { + return v, true + } - return nil, false + return nil, false } func (s *Snapshot) Entry(i int) (string, interface{}) { - if i < 0 || i >= len(s.Keys) { - panic("Out of bounds") - } + if i < 0 || i >= len(s.Keys) { + panic("Out of bounds") + } - return s.Keys[i], s.Entries[s.Keys[i]] + return s.Keys[i], s.Entries[s.Keys[i]] } func (s *Snapshot) NumEntries() int { - return len(s.Keys) + return len(s.Keys) } func (s *Snapshot) AddSub(sub *Snapshot) { - desc := sub.Description() - s.Subs[desc] = sub - s.SubKeys = append(s.SubKeys, desc) + desc := sub.Description() + s.Subs[desc] = sub + s.SubKeys = append(s.SubKeys, desc) } func (s *Snapshot) Sub(i int) *Snapshot { - if i < 0 || i >= len(s.SubKeys) { - panic("Out of bounds") - } + if i < 0 || i >= len(s.SubKeys) { + panic("Out of bounds") + } - return s.Subs[s.SubKeys[i]] + return s.Subs[s.SubKeys[i]] } func (s *Snapshot) NumSubs() int { - return len(s.SubKeys) + return len(s.SubKeys) } diff --git a/src/lc-lib/core/stream.go b/src/lc-lib/core/stream.go index 5b096fa3..e07f4ef1 100644 --- a/src/lc-lib/core/stream.go +++ b/src/lc-lib/core/stream.go @@ -20,5 +20,5 @@ import "os" // A stream should be a pointer object that uniquely identified a file stream type Stream interface { - Info() (string, os.FileInfo) + Info() (string, os.FileInfo) } diff --git a/src/lc-lib/core/transport.go b/src/lc-lib/core/transport.go index 1ab49a7f..83e22c97 100644 --- a/src/lc-lib/core/transport.go +++ b/src/lc-lib/core/transport.go @@ -17,22 +17,22 @@ package core const ( - Reload_None = iota - Reload_Servers - Reload_Transport + Reload_None = iota + Reload_Servers + Reload_Transport ) type Transport interface { - ReloadConfig(*NetworkConfig) int - Init() error - CanSend() <-chan int - Write(string, []byte) error - Read() <-chan interface{} - Shutdown() + ReloadConfig(*NetworkConfig) int + Init() error + CanSend() <-chan int + Write(string, []byte) error + Read() <-chan interface{} + Shutdown() } type TransportFactory interface { - NewTransport(*NetworkConfig) (Transport, error) + NewTransport(*NetworkConfig) (Transport, error) } type TransportRegistrarFunc func(*Config, string, map[string]interface{}, string) (TransportFactory, error) @@ -40,13 +40,13 @@ type TransportRegistrarFunc func(*Config, string, map[string]interface{}, string var registered_Transports map[string]TransportRegistrarFunc = make(map[string]TransportRegistrarFunc) func RegisterTransport(transport string, registrar_func TransportRegistrarFunc) { - registered_Transports[transport] = registrar_func + registered_Transports[transport] = registrar_func } func AvailableTransports() (ret []string) { - ret = make([]string, 0, len(registered_Transports)) - for k := range registered_Transports { - ret = append(ret, k) - } - return + ret = make([]string, 0, len(registered_Transports)) + for k := range registered_Transports { + ret = append(ret, k) + } + return } diff --git a/src/lc-lib/core/util.go b/src/lc-lib/core/util.go index f09b6d9f..2bb6ac5e 100644 --- a/src/lc-lib/core/util.go +++ b/src/lc-lib/core/util.go @@ -12,31 +12,31 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. -*/ + */ package core import ( - "math" - "time" + "math" + "time" ) func CalculateSpeed(duration time.Duration, speed float64, count float64, seconds_no_change *int) float64 { - if count == 0 { - *seconds_no_change++ - } else { - *seconds_no_change = 0 - } + if count == 0 { + *seconds_no_change++ + } else { + *seconds_no_change = 0 + } - if speed == 0. { - return count - } + if speed == 0. { + return count + } - if *seconds_no_change >= 5 { - *seconds_no_change = 0 - return 0. - } + if *seconds_no_change >= 5 { + *seconds_no_change = 0 + return 0. + } - // Calculate a moving average over 5 seconds - use similiar weight as load average - return count + math.Exp(float64(duration) / float64(time.Second) / -5.) * (speed - count) + // Calculate a moving average over 5 seconds - use similiar weight as load average + return count + math.Exp(float64(duration)/float64(time.Second)/-5.)*(speed-count) } diff --git a/src/lc-lib/core/version.go b/src/lc-lib/core/version.go deleted file mode 100644 index 5fea5ae1..00000000 --- a/src/lc-lib/core/version.go +++ /dev/null @@ -1,19 +0,0 @@ -/* -* Copyright 2014 Jason Woods. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package core - -const Log_Courier_Version string = "1.3" diff --git a/src/lc-lib/core/version.go.template b/src/lc-lib/core/version.go.tmpl similarity index 100% rename from src/lc-lib/core/version.go.template rename to src/lc-lib/core/version.go.tmpl diff --git a/src/lc-lib/harvester/harvester.go b/src/lc-lib/harvester/harvester.go index f9cf6637..c5c3a58d 100644 --- a/src/lc-lib/harvester/harvester.go +++ b/src/lc-lib/harvester/harvester.go @@ -20,389 +20,406 @@ package harvester import ( - "fmt" - "io" - "lc-lib/core" - "os" - "sync" - "time" + "fmt" + "github.com/driskell/log-courier/src/lc-lib/core" + "io" + "os" + "sync" + "time" ) type HarvesterFinish struct { - Last_Offset int64 - Error error + Last_Event_Offset int64 + Last_Read_Offset int64 + Error error + Last_Stat os.FileInfo } type Harvester struct { - sync.RWMutex - - stop_chan chan interface{} - return_chan chan *HarvesterFinish - stream core.Stream - fileinfo os.FileInfo - path string - config *core.Config - fileconfig *core.FileConfig - offset int64 - output chan<- *core.EventDescriptor - codec core.Codec - file *os.File - split bool - - line_speed float64 - byte_speed float64 - line_count uint64 - byte_count uint64 - last_eof_off *int64 - last_eof *time.Time + sync.RWMutex + + stop_chan chan interface{} + return_chan chan *HarvesterFinish + stream core.Stream + fileinfo os.FileInfo + path string + config *core.Config + stream_config *core.StreamConfig + offset int64 + output chan<- *core.EventDescriptor + codec core.Codec + file *os.File + split bool + + line_speed float64 + byte_speed float64 + line_count uint64 + byte_count uint64 + last_eof_off *int64 + last_eof *time.Time } -func NewHarvester(stream core.Stream, config *core.Config, fileconfig *core.FileConfig, offset int64) *Harvester { - var fileinfo os.FileInfo - var path string - - if stream != nil { - // Grab now so we can safely use them even if prospector changes them - path, fileinfo = stream.Info() - } else { - // This is stdin - path, fileinfo = "-", nil - } - - ret := &Harvester{ - stop_chan: make(chan interface{}), - return_chan: make(chan *HarvesterFinish, 1), - stream: stream, - fileinfo: fileinfo, - path: path, - config: config, - fileconfig: fileconfig, - offset: offset, - last_eof: nil, - } - - ret.codec = fileconfig.CodecFactory.NewCodec(ret.eventCallback, ret.offset) - - return ret +func NewHarvester(stream core.Stream, config *core.Config, stream_config *core.StreamConfig, offset int64) *Harvester { + var fileinfo os.FileInfo + var path string + + if stream != nil { + // Grab now so we can safely use them even if prospector changes them + path, fileinfo = stream.Info() + } else { + // This is stdin + path, fileinfo = "-", nil + } + + ret := &Harvester{ + stop_chan: make(chan interface{}), + stream: stream, + fileinfo: fileinfo, + path: path, + config: config, + stream_config: stream_config, + offset: offset, + last_eof: nil, + } + + ret.codec = stream_config.CodecFactory.NewCodec(ret.eventCallback, ret.offset) + + return ret } func (h *Harvester) Start(output chan<- *core.EventDescriptor) { - go func() { - status := &HarvesterFinish{} - status.Last_Offset, status.Error = h.harvest(output) - h.return_chan <- status - }() + if h.return_chan != nil { + h.Stop() + <-h.return_chan + } + + h.return_chan = make(chan *HarvesterFinish, 1) + + go func() { + status := &HarvesterFinish{} + status.Last_Event_Offset, status.Error = h.harvest(output) + status.Last_Read_Offset = h.offset + status.Last_Stat = h.fileinfo + h.return_chan <- status + close(h.return_chan) + }() } func (h *Harvester) Stop() { - close(h.stop_chan) + close(h.stop_chan) } func (h *Harvester) OnFinish() <-chan *HarvesterFinish { - return h.return_chan + return h.return_chan } func (h *Harvester) harvest(output chan<- *core.EventDescriptor) (int64, error) { - if err := h.prepareHarvester(); err != nil { - return h.offset, err - } - - defer h.file.Close() - - h.output = output - - if h.path == "-" { - log.Info("Started stdin harvester") - h.offset = 0 - } else { - // Get current offset in file - offset, err := h.file.Seek(0, os.SEEK_CUR) - if err != nil { - log.Warning("Failed to determine start offset for %s: %s", h.path, err) - return h.offset, err - } - - if h.offset != offset { - log.Warning("Started harvester at position %d (requested %d): %s", offset, h.offset, h.path) - } else { - log.Info("Started harvester at position %d (requested %d): %s", offset, h.offset, h.path) - } - - h.offset = offset - } - - // The buffer size limits the maximum line length we can read, including terminator - reader := NewLineReader(h.file, int(h.config.General.LineBufferBytes), int(h.config.General.MaxLineBytes)) - - // TODO: Make configurable? - read_timeout := 10 * time.Second - - last_read_time := time.Now() - last_line_count := uint64(0) - last_byte_count := uint64(0) - last_measurement := last_read_time - seconds_without_events := 0 + if err := h.prepareHarvester(); err != nil { + return h.offset, err + } + + defer h.file.Close() + + h.output = output + + if h.path == "-" { + log.Info("Started stdin harvester") + h.offset = 0 + } else { + // Get current offset in file + offset, err := h.file.Seek(0, os.SEEK_CUR) + if err != nil { + log.Warning("Failed to determine start offset for %s: %s", h.path, err) + return h.offset, err + } + + if h.offset != offset { + log.Warning("Started harvester at position %d (requested %d): %s", offset, h.offset, h.path) + } else { + log.Info("Started harvester at position %d (requested %d): %s", offset, h.offset, h.path) + } + + h.offset = offset + } + + // The buffer size limits the maximum line length we can read, including terminator + reader := NewLineReader(h.file, int(h.config.General.LineBufferBytes), int(h.config.General.MaxLineBytes)) + + // TODO: Make configurable? + read_timeout := 10 * time.Second + + last_read_time := time.Now() + last_line_count := uint64(0) + last_byte_count := uint64(0) + last_measurement := last_read_time + seconds_without_events := 0 ReadLoop: - for { - text, bytesread, err := h.readline(reader) - - if duration := time.Since(last_measurement); duration >= time.Second { - h.Lock() - - h.line_speed = core.CalculateSpeed(duration, h.line_speed, float64(h.line_count - last_line_count), &seconds_without_events) - h.byte_speed = core.CalculateSpeed(duration, h.byte_speed, float64(h.byte_count - last_byte_count), &seconds_without_events) - - last_byte_count = h.byte_count - last_line_count = h.line_count - last_measurement = time.Now() - - h.codec.Meter() - - h.last_eof = nil - - h.Unlock() - - // Check shutdown - select { - case <-h.stop_chan: - break ReadLoop - default: - } - } - - if err == nil { - line_offset := h.offset - h.offset += int64(bytesread) - - // Codec is last - it forwards harvester state for us such as offset for resume - h.codec.Event(line_offset, h.offset, text) - - last_read_time = time.Now() - h.line_count++ - h.byte_count += uint64(bytesread) - - continue - } - - if err != io.EOF { - if h.path == "-" { - log.Error("Unexpected error reading from stdin: %s", err) - } else { - log.Error("Unexpected error reading from %s: %s", h.path, err) - } - return h.codec.Teardown(), err - } - - if h.path == "-" { - // Stdin has finished - stdin blocks permanently until the stream ends - // Once the stream ends, finish the harvester - log.Info("Stopping harvest of stdin; EOF reached") - return h.codec.Teardown(), nil - } - - // Check shutdown - select { - case <-h.stop_chan: - break ReadLoop - default: - } - - h.Lock() - if h.last_eof_off == nil { - h.last_eof_off = new(int64) - } - *h.last_eof_off = h.offset - - if h.last_eof == nil { - h.last_eof = new(time.Time) - } - *h.last_eof = last_read_time - h.Unlock() - - // Don't check for truncation until we hit the full read_timeout - if time.Since(last_read_time) < read_timeout { - continue - } - - info, err := h.file.Stat() - if err != nil { - log.Error("Unexpected error checking status of %s: %s", h.path, err) - return h.codec.Teardown(), err - } - - if info.Size() < h.offset { - log.Warning("Unexpected file truncation, seeking to beginning: %s", h.path) - h.file.Seek(0, os.SEEK_SET) - h.offset = 0 - // TODO: How does this impact a partial line reader buffer? - // TODO: How does this imapct multiline? - continue - } - - if age := time.Since(last_read_time); age > h.fileconfig.DeadTime { - // if last_read_time was more than dead time, this file is probably dead. Stop watching it. - log.Info("Stopping harvest of %s; last change was %v ago", h.path, age-(age%time.Second)) - // TODO: We should return a Stat() from before we attempted to read - // In prospector we use that for comparison to resume - // This prevents a potential race condition if we stop just as the - // file is modified with extra lines... - return h.codec.Teardown(), nil - } - } - - log.Info("Harvester for %s exiting", h.path) - return h.codec.Teardown(), nil + for { + text, bytesread, err := h.readline(reader) + + if duration := time.Since(last_measurement); duration >= time.Second { + h.Lock() + + h.line_speed = core.CalculateSpeed(duration, h.line_speed, float64(h.line_count-last_line_count), &seconds_without_events) + h.byte_speed = core.CalculateSpeed(duration, h.byte_speed, float64(h.byte_count-last_byte_count), &seconds_without_events) + + last_byte_count = h.byte_count + last_line_count = h.line_count + last_measurement = time.Now() + + h.codec.Meter() + + h.last_eof = nil + + h.Unlock() + + // Check shutdown + select { + case <-h.stop_chan: + break ReadLoop + default: + } + } + + if err == nil { + line_offset := h.offset + h.offset += int64(bytesread) + + // Codec is last - it forwards harvester state for us such as offset for resume + h.codec.Event(line_offset, h.offset, text) + + last_read_time = time.Now() + h.line_count++ + h.byte_count += uint64(bytesread) + + continue + } + + if err != io.EOF { + if h.path == "-" { + log.Error("Unexpected error reading from stdin: %s", err) + } else { + log.Error("Unexpected error reading from %s: %s", h.path, err) + } + return h.codec.Teardown(), err + } + + if h.path == "-" { + // Stdin has finished - stdin blocks permanently until the stream ends + // Once the stream ends, finish the harvester + log.Info("Stopping harvest of stdin; EOF reached") + return h.codec.Teardown(), nil + } + + // Check shutdown + select { + case <-h.stop_chan: + break ReadLoop + default: + } + + h.Lock() + if h.last_eof_off == nil { + h.last_eof_off = new(int64) + } + *h.last_eof_off = h.offset + + if h.last_eof == nil { + h.last_eof = new(time.Time) + } + *h.last_eof = last_read_time + h.Unlock() + + // Don't check for truncation until we hit the full read_timeout + if time.Since(last_read_time) < read_timeout { + continue + } + + info, err := h.file.Stat() + if err != nil { + log.Error("Unexpected error checking status of %s: %s", h.path, err) + return h.codec.Teardown(), err + } + + if info.Size() < h.offset { + log.Warning("Unexpected file truncation, seeking to beginning: %s", h.path) + h.file.Seek(0, os.SEEK_SET) + h.offset = 0 + // TODO: How does this impact a partial line reader buffer? + // TODO: How does this impact multiline? + continue + } + + // If last_read_time was more than dead time, this file is probably dead. + // Stop only if the mtime did not change since last check - this stops a + // race where we hit EOF but as we Stat() the mtime is updated - this mtime + // is the one we monitor in order to resume checking, so we need to check it + // didn't already update + if age := time.Since(last_read_time); age > h.stream_config.DeadTime && h.fileinfo.ModTime() == info.ModTime() { + log.Info("Stopping harvest of %s; last change was %v ago", h.path, age-(age%time.Second)) + return h.codec.Teardown(), nil + } + + // Store latest stat() + h.fileinfo = info + } + + log.Info("Harvester for %s exiting", h.path) + return h.codec.Teardown(), nil } func (h *Harvester) eventCallback(start_offset int64, end_offset int64, text string) { - event := core.Event{ - "host": h.config.General.Host, - "path": h.path, - "offset": start_offset, - "message": text, - } - for k := range h.fileconfig.Fields { - event[k] = h.fileconfig.Fields[k] - } - - // If we split any of the line data, tag it - if h.split { - if v, ok := event["tags"]; ok { - if v, ok := v.([]string); ok { - v = append(v, "splitline") - } - } else { - event["tags"] = []string{"splitline"} - } - h.split = false - } - - encoded, err := event.Encode() - if err != nil { - // This should never happen - log and skip if it does - log.Warning("Skipping line in %s at offset %d due to encoding failure: %s", h.path, start_offset, err) - return - } - - desc := &core.EventDescriptor{ - Stream: h.stream, - Offset: end_offset, - Event: encoded, - } + event := core.Event{ + "host": h.config.General.Host, + "path": h.path, + "offset": start_offset, + "message": text, + } + for k := range h.stream_config.Fields { + event[k] = h.stream_config.Fields[k] + } + + // If we split any of the line data, tag it + if h.split { + if v, ok := event["tags"]; ok { + if v, ok := v.([]string); ok { + v = append(v, "splitline") + } + } else { + event["tags"] = []string{"splitline"} + } + h.split = false + } + + encoded, err := event.Encode() + if err != nil { + // This should never happen - log and skip if it does + log.Warning("Skipping line in %s at offset %d due to encoding failure: %s", h.path, start_offset, err) + return + } + + desc := &core.EventDescriptor{ + Stream: h.stream, + Offset: end_offset, + Event: encoded, + } EventLoop: - for { - select { - case <-h.stop_chan: - break EventLoop - case h.output <- desc: - break EventLoop - } - } + for { + select { + case <-h.stop_chan: + break EventLoop + case h.output <- desc: + break EventLoop + } + } } func (h *Harvester) prepareHarvester() error { - // Special handling that "-" means to read from standard input - if h.path == "-" { - h.file = os.Stdin - return nil - } - - var err error - h.file, err = h.openFile(h.path) - if err != nil { - log.Error("Failed opening %s: %s", h.path, err) - return err - } - - // Check we opened the right file - info, err := h.file.Stat() - if err != nil { - h.file.Close() - return err - } - - if !os.SameFile(info, h.fileinfo) { - h.file.Close() - return fmt.Errorf("Not the same file") - } - - // TODO: Check error? - h.file.Seek(h.offset, os.SEEK_SET) - - return nil + // Special handling that "-" means to read from standard input + if h.path == "-" { + h.file = os.Stdin + return nil + } + + var err error + h.file, err = h.openFile(h.path) + if err != nil { + log.Error("Failed opening %s: %s", h.path, err) + return err + } + + // Check we opened the right file + info, err := h.file.Stat() + if err != nil { + h.file.Close() + return err + } + + if !os.SameFile(info, h.fileinfo) { + h.file.Close() + return fmt.Errorf("Not the same file") + } + + // Store latest stat() + h.fileinfo = info + + // TODO: Check error? + h.file.Seek(h.offset, os.SEEK_SET) + + return nil } func (h *Harvester) readline(reader *LineReader) (string, int, error) { - var newline int - - line, err := reader.ReadSlice() - - if line != nil { - if err == nil { - // Line will always end in '\n' if no error, but check also for CR - if len(line) > 1 && line[len(line)-2] == '\r' { - newline = 2 - } else { - newline = 1 - } - } else if err == ErrLineTooLong { - h.split = true - err = nil - } - - // Return the line along with the length including line ending - length := len(line) - // We use string() to copy the memory, which is a slice of the line buffer we need to re-use - return string(line[:length-newline]), length, err - } - - if err != nil { - if err != io.EOF { - // Pass back error to tear down harvester - return "", 0, err - } - - // Backoff - select { - case <-h.stop_chan: - case <-time.After(1 * time.Second): - } - } - - return "", 0, io.EOF + var newline int + + line, err := reader.ReadSlice() + + if line != nil { + if err == nil { + // Line will always end in '\n' if no error, but check also for CR + if len(line) > 1 && line[len(line)-2] == '\r' { + newline = 2 + } else { + newline = 1 + } + } else if err == ErrLineTooLong { + h.split = true + err = nil + } + + // Return the line along with the length including line ending + length := len(line) + // We use string() to copy the memory, which is a slice of the line buffer we need to re-use + return string(line[:length-newline]), length, err + } + + if err != nil { + if err != io.EOF { + // Pass back error to tear down harvester + return "", 0, err + } + + // Backoff + select { + case <-h.stop_chan: + case <-time.After(1 * time.Second): + } + } + + return "", 0, io.EOF } func (h *Harvester) Snapshot() *core.Snapshot { - h.RLock() - - ret := core.NewSnapshot("Harvester") - ret.AddEntry("Speed (Lps)", h.line_speed) - ret.AddEntry("Speed (Bps)", h.byte_speed) - ret.AddEntry("Processed lines", h.line_count) - ret.AddEntry("Current offset", h.offset) - if h.last_eof_off == nil { - ret.AddEntry("Last EOF Offset", "Never") - } else { - ret.AddEntry("Last EOF Offset", h.last_eof_off) - } - if h.last_eof == nil { - ret.AddEntry("Status", "Alive") - } else { - ret.AddEntry("Status", "Idle") - if age := time.Since(*h.last_eof); age < h.fileconfig.DeadTime { - ret.AddEntry("Dead timer", h.fileconfig.DeadTime-age) - } else { - ret.AddEntry("Dead timer", "0s") - } - } - - if sub_snap := h.codec.Snapshot(); sub_snap != nil { - ret.AddSub(sub_snap) - } - - h.RUnlock() - - return ret + h.RLock() + + ret := core.NewSnapshot("Harvester") + ret.AddEntry("Speed (Lps)", h.line_speed) + ret.AddEntry("Speed (Bps)", h.byte_speed) + ret.AddEntry("Processed lines", h.line_count) + ret.AddEntry("Current offset", h.offset) + if h.last_eof_off == nil { + ret.AddEntry("Last EOF Offset", "Never") + } else { + ret.AddEntry("Last EOF Offset", h.last_eof_off) + } + if h.last_eof == nil { + ret.AddEntry("Status", "Alive") + } else { + ret.AddEntry("Status", "Idle") + if age := time.Since(*h.last_eof); age < h.stream_config.DeadTime { + ret.AddEntry("Dead timer", h.stream_config.DeadTime-age) + } else { + ret.AddEntry("Dead timer", "0s") + } + } + + if sub_snap := h.codec.Snapshot(); sub_snap != nil { + ret.AddSub(sub_snap) + } + + h.RUnlock() + + return ret } diff --git a/src/lc-lib/harvester/harvester_other.go b/src/lc-lib/harvester/harvester_other.go index c8f95d85..7a7468dc 100644 --- a/src/lc-lib/harvester/harvester_other.go +++ b/src/lc-lib/harvester/harvester_other.go @@ -19,9 +19,9 @@ package harvester import ( - "os" + "os" ) func (h *Harvester) openFile(path string) (*os.File, error) { - return os.Open(path) + return os.Open(path) } diff --git a/src/lc-lib/harvester/harvester_windows.go b/src/lc-lib/harvester/harvester_windows.go index 9f224da0..da44af27 100644 --- a/src/lc-lib/harvester/harvester_windows.go +++ b/src/lc-lib/harvester/harvester_windows.go @@ -17,26 +17,26 @@ package harvester import ( - "os" - "syscall" + "os" + "syscall" ) func (h *Harvester) openFile(path string) (*os.File, error) { - // We will call CreateFile directly so we can pass in FILE_SHARE_DELETE - // This ensures that a program can still rotate the file even though we have it open - pathp, err := syscall.UTF16PtrFromString(path) - if err != nil { - return nil, err - } + // We will call CreateFile directly so we can pass in FILE_SHARE_DELETE + // This ensures that a program can still rotate the file even though we have it open + pathp, err := syscall.UTF16PtrFromString(path) + if err != nil { + return nil, err + } - var sa *syscall.SecurityAttributes + var sa *syscall.SecurityAttributes - handle, err := syscall.CreateFile( - pathp, syscall.GENERIC_READ, syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, - sa, syscall.OPEN_EXISTING, syscall.FILE_ATTRIBUTE_NORMAL, 0) - if err != nil { - return nil, err - } + handle, err := syscall.CreateFile( + pathp, syscall.GENERIC_READ, syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, + sa, syscall.OPEN_EXISTING, syscall.FILE_ATTRIBUTE_NORMAL, 0) + if err != nil { + return nil, err + } - return os.NewFile(uintptr(handle), path), nil + return os.NewFile(uintptr(handle), path), nil } diff --git a/src/lc-lib/harvester/linereader.go b/src/lc-lib/harvester/linereader.go index d068ede9..d729a05d 100644 --- a/src/lc-lib/harvester/linereader.go +++ b/src/lc-lib/harvester/linereader.go @@ -12,107 +12,107 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. -*/ + */ package harvester import ( - "bytes" - "errors" - "io" + "bytes" + "errors" + "io" ) var ( - ErrLineTooLong = errors.New("LineReader: line too long") + ErrLineTooLong = errors.New("LineReader: line too long") ) // A read interface to tail type LineReader struct { - rd io.Reader - buf []byte - overflow [][]byte - size int - max_line int - cur_max int - start int - end int - err error + rd io.Reader + buf []byte + overflow [][]byte + size int + max_line int + cur_max int + start int + end int + err error } func NewLineReader(rd io.Reader, size int, max_line int) *LineReader { - lr := &LineReader{ - rd: rd, - buf: make([]byte, size), - size: size, - max_line: max_line, - cur_max: max_line, - } - - return lr + lr := &LineReader{ + rd: rd, + buf: make([]byte, size), + size: size, + max_line: max_line, + cur_max: max_line, + } + + return lr } func (lr *LineReader) Reset() { - lr.start = 0 + lr.start = 0 } func (lr *LineReader) ReadSlice() ([]byte, error) { - var err error - var line []byte - - if lr.end == 0 { - err = lr.fill() - } - - for { - if n := bytes.IndexByte(lr.buf[lr.start:lr.end], '\n'); n >= 0 && n < lr.cur_max { - line = lr.buf[lr.start:lr.start+n+1] - lr.start += n + 1 - err = nil - break - } - - if err != nil { - return nil, err - } - - if lr.end - lr.start >= lr.cur_max { - line = lr.buf[lr.start:lr.start+lr.cur_max] - lr.start += lr.cur_max - err = ErrLineTooLong - break - } - - if lr.end - lr.start >= len(lr.buf) { - lr.start, lr.end = 0, 0 - if lr.overflow == nil { - lr.overflow = make([][]byte, 0, 1) - } - lr.overflow = append(lr.overflow, lr.buf) - lr.cur_max -= len(lr.buf) - lr.buf = make([]byte, lr.size) - } - - err = lr.fill() - } - - if lr.overflow != nil { - lr.overflow = append(lr.overflow, line) - line = bytes.Join(lr.overflow, []byte{}) - lr.overflow = nil - lr.cur_max = lr.max_line - } - return line, err + var err error + var line []byte + + if lr.end == 0 { + err = lr.fill() + } + + for { + if n := bytes.IndexByte(lr.buf[lr.start:lr.end], '\n'); n >= 0 && n < lr.cur_max { + line = lr.buf[lr.start : lr.start+n+1] + lr.start += n + 1 + err = nil + break + } + + if err != nil { + return nil, err + } + + if lr.end-lr.start >= lr.cur_max { + line = lr.buf[lr.start : lr.start+lr.cur_max] + lr.start += lr.cur_max + err = ErrLineTooLong + break + } + + if lr.end-lr.start >= len(lr.buf) { + lr.start, lr.end = 0, 0 + if lr.overflow == nil { + lr.overflow = make([][]byte, 0, 1) + } + lr.overflow = append(lr.overflow, lr.buf) + lr.cur_max -= len(lr.buf) + lr.buf = make([]byte, lr.size) + } + + err = lr.fill() + } + + if lr.overflow != nil { + lr.overflow = append(lr.overflow, line) + line = bytes.Join(lr.overflow, []byte{}) + lr.overflow = nil + lr.cur_max = lr.max_line + } + return line, err } func (lr *LineReader) fill() error { - if lr.start != 0 { - copy(lr.buf, lr.buf[lr.start:lr.end]) - lr.end -= lr.start - lr.start = 0 - } + if lr.start != 0 { + copy(lr.buf, lr.buf[lr.start:lr.end]) + lr.end -= lr.start + lr.start = 0 + } - n, err := lr.rd.Read(lr.buf[lr.end:]) - lr.end += n + n, err := lr.rd.Read(lr.buf[lr.end:]) + lr.end += n - return err + return err } diff --git a/src/lc-lib/harvester/linereader_test.go b/src/lc-lib/harvester/linereader_test.go index 59903c9c..ed899072 100644 --- a/src/lc-lib/harvester/linereader_test.go +++ b/src/lc-lib/harvester/linereader_test.go @@ -12,127 +12,126 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. -*/ + */ package harvester import ( - "bytes" - "testing" + "bytes" + "testing" ) func checkLine(t *testing.T, reader *LineReader, expected []byte) { - line, err := reader.ReadSlice() - if line == nil { - t.Log("No line returned") - t.FailNow() - } - if !bytes.Equal(line, expected) { - t.Logf("Line data incorrect: [% X]", line) - t.FailNow() - } - if err != nil { - t.Logf("Unexpected error: %s", err) - t.FailNow() - } + line, err := reader.ReadSlice() + if line == nil { + t.Log("No line returned") + t.FailNow() + } + if !bytes.Equal(line, expected) { + t.Logf("Line data incorrect: [% X]", line) + t.FailNow() + } + if err != nil { + t.Logf("Unexpected error: %s", err) + t.FailNow() + } } func checkLineTooLong(t *testing.T, reader *LineReader, expected []byte) { - line, err := reader.ReadSlice() - if line == nil { - t.Log("No line returned") - t.FailNow() - } - if !bytes.Equal(line, expected) { - t.Logf("Line data incorrect: [% X]", line) - t.FailNow() - } - if err != ErrLineTooLong { - t.Logf("Unexpected error: %s", err) - t.FailNow() - } + line, err := reader.ReadSlice() + if line == nil { + t.Log("No line returned") + t.FailNow() + } + if !bytes.Equal(line, expected) { + t.Logf("Line data incorrect: [% X]", line) + t.FailNow() + } + if err != ErrLineTooLong { + t.Logf("Unexpected error: %s", err) + t.FailNow() + } } func checkEnd(t *testing.T, reader *LineReader) { - line, err := reader.ReadSlice() - if line != nil { - t.Log("Unexpected line returned") - t.FailNow() - } - if err == nil { - t.Log("Expected error") - t.FailNow() - } + line, err := reader.ReadSlice() + if line != nil { + t.Log("Unexpected line returned") + t.FailNow() + } + if err == nil { + t.Log("Expected error") + t.FailNow() + } } func TestLineRead(t *testing.T) { - data := bytes.NewBufferString("12345678901234567890\n12345678901234567890\n") + data := bytes.NewBufferString("12345678901234567890\n12345678901234567890\n") - // New line read with 100 bytes, enough for the above - reader := NewLineReader(data, 100, 100) + // New line read with 100 bytes, enough for the above + reader := NewLineReader(data, 100, 100) - checkLine(t, reader, []byte("12345678901234567890\n")) - checkLine(t, reader, []byte("12345678901234567890\n")) - checkEnd(t, reader) + checkLine(t, reader, []byte("12345678901234567890\n")) + checkLine(t, reader, []byte("12345678901234567890\n")) + checkEnd(t, reader) } func TestLineReadEmpty(t *testing.T) { - data := bytes.NewBufferString("\n12345678901234567890\n") + data := bytes.NewBufferString("\n12345678901234567890\n") - // New line read with 100 bytes, enough for the above - reader := NewLineReader(data, 100, 100) + // New line read with 100 bytes, enough for the above + reader := NewLineReader(data, 100, 100) - checkLine(t, reader, []byte("\n")) - checkLine(t, reader, []byte("12345678901234567890\n")) - checkEnd(t, reader) + checkLine(t, reader, []byte("\n")) + checkLine(t, reader, []byte("12345678901234567890\n")) + checkEnd(t, reader) } func TestLineReadIncomplete(t *testing.T) { - data := bytes.NewBufferString("\n12345678901234567890\n123456") + data := bytes.NewBufferString("\n12345678901234567890\n123456") - // New line read with 100 bytes, enough for the above - reader := NewLineReader(data, 100, 100) + // New line read with 100 bytes, enough for the above + reader := NewLineReader(data, 100, 100) - checkLine(t, reader, []byte("\n")) - checkLine(t, reader, []byte("12345678901234567890\n")) - checkEnd(t, reader) + checkLine(t, reader, []byte("\n")) + checkLine(t, reader, []byte("12345678901234567890\n")) + checkEnd(t, reader) } func TestLineReadOverflow(t *testing.T) { - data := bytes.NewBufferString("12345678901234567890\n123456789012345678901234567890\n12345678901234567890\n") + data := bytes.NewBufferString("12345678901234567890\n123456789012345678901234567890\n12345678901234567890\n") - // New line read with 21 bytes buffer but 100 max line to trigger overflow - reader := NewLineReader(data, 21, 100) + // New line read with 21 bytes buffer but 100 max line to trigger overflow + reader := NewLineReader(data, 21, 100) - checkLine(t, reader, []byte("12345678901234567890\n")) - checkLine(t, reader, []byte("123456789012345678901234567890\n")) - checkLine(t, reader, []byte("12345678901234567890\n")) - checkEnd(t, reader) + checkLine(t, reader, []byte("12345678901234567890\n")) + checkLine(t, reader, []byte("123456789012345678901234567890\n")) + checkLine(t, reader, []byte("12345678901234567890\n")) + checkEnd(t, reader) } - func TestLineReadOverflowTooLong(t *testing.T) { - data := bytes.NewBufferString("12345678901234567890\n123456789012345678901234567890\n12345678901234567890\n") + data := bytes.NewBufferString("12345678901234567890\n123456789012345678901234567890\n12345678901234567890\n") - // New line read with 10 bytes buffer and 21 max line length - reader := NewLineReader(data, 10, 21) + // New line read with 10 bytes buffer and 21 max line length + reader := NewLineReader(data, 10, 21) - checkLine(t, reader, []byte("12345678901234567890\n")) - checkLineTooLong(t, reader, []byte("123456789012345678901")) - checkLine(t, reader, []byte("234567890\n")) - checkLine(t, reader, []byte("12345678901234567890\n")) - checkEnd(t, reader) + checkLine(t, reader, []byte("12345678901234567890\n")) + checkLineTooLong(t, reader, []byte("123456789012345678901")) + checkLine(t, reader, []byte("234567890\n")) + checkLine(t, reader, []byte("12345678901234567890\n")) + checkEnd(t, reader) } func TestLineReadTooLong(t *testing.T) { - data := bytes.NewBufferString("12345678901234567890\n123456789012345678901234567890\n12345678901234567890\n") + data := bytes.NewBufferString("12345678901234567890\n123456789012345678901234567890\n12345678901234567890\n") - // New line read with ample buffer and 21 max line length - reader := NewLineReader(data, 100, 21) + // New line read with ample buffer and 21 max line length + reader := NewLineReader(data, 100, 21) - checkLine(t, reader, []byte("12345678901234567890\n")) - checkLineTooLong(t, reader, []byte("123456789012345678901")) - checkLine(t, reader, []byte("234567890\n")) - checkLine(t, reader, []byte("12345678901234567890\n")) - checkEnd(t, reader) + checkLine(t, reader, []byte("12345678901234567890\n")) + checkLineTooLong(t, reader, []byte("123456789012345678901")) + checkLine(t, reader, []byte("234567890\n")) + checkLine(t, reader, []byte("12345678901234567890\n")) + checkEnd(t, reader) } diff --git a/src/lc-lib/harvester/logging.go b/src/lc-lib/harvester/logging.go index 3a363141..df6f8a86 100644 --- a/src/lc-lib/harvester/logging.go +++ b/src/lc-lib/harvester/logging.go @@ -21,5 +21,5 @@ import "github.com/op/go-logging" var log *logging.Logger func init() { - log = logging.MustGetLogger("harvester") + log = logging.MustGetLogger("harvester") } diff --git a/src/lc-lib/prospector/errors.go b/src/lc-lib/prospector/errors.go index a2c01a5e..495f80d2 100644 --- a/src/lc-lib/prospector/errors.go +++ b/src/lc-lib/prospector/errors.go @@ -12,18 +12,18 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. -*/ + */ package prospector type ProspectorSkipError struct { - message string + message string } func newProspectorSkipError(message string) *ProspectorSkipError { - return &ProspectorSkipError{message: message} + return &ProspectorSkipError{message: message} } func (e *ProspectorSkipError) Error() string { - return e.message + return e.message } diff --git a/src/lc-lib/prospector/info.go b/src/lc-lib/prospector/info.go index 52ea34fc..98ac6eaf 100644 --- a/src/lc-lib/prospector/info.go +++ b/src/lc-lib/prospector/info.go @@ -17,123 +17,119 @@ package prospector import ( - "lc-lib/core" - "lc-lib/harvester" - "lc-lib/registrar" - "os" + "github.com/driskell/log-courier/src/lc-lib/core" + "github.com/driskell/log-courier/src/lc-lib/harvester" + "github.com/driskell/log-courier/src/lc-lib/registrar" + "os" ) const ( - Status_Ok = iota - Status_Resume - Status_Failed - Status_Invalid + Status_Ok = iota + Status_Resume + Status_Failed + Status_Invalid ) const ( - Orphaned_No = iota - Orphaned_Maybe - Orphaned_Yes + Orphaned_No = iota + Orphaned_Maybe + Orphaned_Yes ) type prospectorInfo struct { - file string - identity registrar.FileIdentity - last_seen uint32 - status int - running bool - orphaned int - finish_offset int64 - harvester *harvester.Harvester - err error + file string + identity registrar.FileIdentity + last_seen uint32 + status int + running bool + orphaned int + finish_offset int64 + harvester *harvester.Harvester + err error } func newProspectorInfoFromFileState(file string, filestate *registrar.FileState) *prospectorInfo { - return &prospectorInfo{ - file: file, - identity: filestate, - status: Status_Resume, - finish_offset: filestate.Offset, - } + return &prospectorInfo{ + file: file, + identity: filestate, + status: Status_Resume, + finish_offset: filestate.Offset, + } } func newProspectorInfoFromFileInfo(file string, fileinfo os.FileInfo) *prospectorInfo { - return &prospectorInfo{ - file: file, - identity: registrar.NewFileInfo(fileinfo), // fileinfo is nil for stdin - } + return &prospectorInfo{ + file: file, + identity: registrar.NewFileInfo(fileinfo), // fileinfo is nil for stdin + } } func newProspectorInfoInvalid(file string, err error) *prospectorInfo { - return &prospectorInfo{ - file: file, - err: err, - status: Status_Invalid, - } + return &prospectorInfo{ + file: file, + err: err, + status: Status_Invalid, + } } func (pi *prospectorInfo) Info() (string, os.FileInfo) { - return pi.file, pi.identity.Stat() + return pi.file, pi.identity.Stat() } func (pi *prospectorInfo) isRunning() bool { - if !pi.running { - return false - } + if !pi.running { + return false + } - select { - case status := <-pi.harvester.OnFinish(): - pi.setHarvesterStopped(status) - default: - } + select { + case status := <-pi.harvester.OnFinish(): + pi.setHarvesterStopped(status) + default: + } - return pi.running + return pi.running } -/*func (pi *prospectorInfo) ShutdownSignal() <-chan interface{} { - return pi.harvester_stop -}*/ - func (pi *prospectorInfo) stop() { - if !pi.running { - return - } - if pi.file == "-" { - // Just in case someone started us outside a pipeline with stdin - // to stop confusion at why we don't exit after Ctrl+C - // There's no deadline on Stdin reads :-( - log.Notice("Waiting for Stdin to close (EOF or Ctrl+D)") - } - pi.harvester.Stop() + if !pi.running { + return + } + pi.harvester.Stop() } func (pi *prospectorInfo) wait() { - if !pi.running { - return - } - status := <-pi.harvester.OnFinish() - pi.setHarvesterStopped(status) + if !pi.running { + return + } + status := <-pi.harvester.OnFinish() + pi.setHarvesterStopped(status) } func (pi *prospectorInfo) getSnapshot() *core.Snapshot { - return pi.harvester.Snapshot() + return pi.harvester.Snapshot() } func (pi *prospectorInfo) setHarvesterStopped(status *harvester.HarvesterFinish) { - pi.running = false - pi.finish_offset = status.Last_Offset - if status.Error != nil { - pi.status = Status_Failed - pi.err = status.Error - } - pi.harvester = nil + pi.running = false + // Resume harvesting from the last event offset, not the last read, to allow codec to read from the last event + // This ensures multiline codec populates correctly on resume + pi.finish_offset = status.Last_Event_Offset + if status.Error != nil { + pi.status = Status_Failed + pi.err = status.Error + } + if status.Last_Stat != nil { + // Keep the last stat the harvester ran so we compare timestamps for potential resume + pi.identity.Update(status.Last_Stat, &pi.identity) + } + pi.harvester = nil } func (pi *prospectorInfo) update(fileinfo os.FileInfo, iteration uint32) { - if fileinfo != nil { - // Allow identity to replace itself with a new identity (this allows a FileState to promote itself to a FileInfo) - pi.identity.Update(fileinfo, &pi.identity) - } + if fileinfo != nil { + // Allow identity to replace itself with a new identity (this allows a FileState to promote itself to a FileInfo) + pi.identity.Update(fileinfo, &pi.identity) + } - pi.last_seen = iteration + pi.last_seen = iteration } diff --git a/src/lc-lib/prospector/logging.go b/src/lc-lib/prospector/logging.go index 8688c0e8..29213c62 100644 --- a/src/lc-lib/prospector/logging.go +++ b/src/lc-lib/prospector/logging.go @@ -12,7 +12,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. -*/ + */ package prospector @@ -21,5 +21,5 @@ import "github.com/op/go-logging" var log *logging.Logger func init() { - log = logging.MustGetLogger("prospector") + log = logging.MustGetLogger("prospector") } diff --git a/src/lc-lib/prospector/prospector.go b/src/lc-lib/prospector/prospector.go index 7e0c6a0d..00e30720 100644 --- a/src/lc-lib/prospector/prospector.go +++ b/src/lc-lib/prospector/prospector.go @@ -20,504 +20,466 @@ package prospector import ( - "fmt" - "lc-lib/core" - "lc-lib/harvester" - "lc-lib/registrar" - "lc-lib/spooler" - "os" - "path/filepath" - "time" + "fmt" + "github.com/driskell/log-courier/src/lc-lib/core" + "github.com/driskell/log-courier/src/lc-lib/harvester" + "github.com/driskell/log-courier/src/lc-lib/registrar" + "github.com/driskell/log-courier/src/lc-lib/spooler" + "os" + "path/filepath" + "time" ) type Prospector struct { - core.PipelineSegment - core.PipelineConfigReceiver - core.PipelineSnapshotProvider - - config *core.Config - prospectorindex map[string]*prospectorInfo - prospectors map[*prospectorInfo]*prospectorInfo - from_beginning bool - iteration uint32 - lastscan time.Time - registrar *registrar.Registrar - registrar_spool *registrar.RegistrarEventSpool - snapshot_chan chan interface{} - snapshot_sink chan []*core.Snapshot - - output chan<- *core.EventDescriptor + core.PipelineSegment + core.PipelineConfigReceiver + core.PipelineSnapshotProvider + + config *core.Config + prospectorindex map[string]*prospectorInfo + prospectors map[*prospectorInfo]*prospectorInfo + from_beginning bool + iteration uint32 + lastscan time.Time + registrar registrar.Registrator + registrar_spool registrar.EventSpooler + snapshot_chan chan interface{} + snapshot_sink chan []*core.Snapshot + + output chan<- *core.EventDescriptor } -func NewProspector(pipeline *core.Pipeline, config *core.Config, from_beginning bool, registrar_imp *registrar.Registrar, spooler_imp *spooler.Spooler) (*Prospector, error) { - ret := &Prospector{ - config: config, - prospectorindex: make(map[string]*prospectorInfo), - prospectors: make(map[*prospectorInfo]*prospectorInfo), - from_beginning: from_beginning, - registrar: registrar_imp, - registrar_spool: registrar_imp.Connect(), - snapshot_chan: make(chan interface{}), - snapshot_sink: make(chan []*core.Snapshot), - output: spooler_imp.Connect(), - } - - if err := ret.init(); err != nil { - return nil, err - } - - pipeline.Register(ret) - - return ret, nil +func NewProspector(pipeline *core.Pipeline, config *core.Config, from_beginning bool, registrar_imp registrar.Registrator, spooler_imp *spooler.Spooler) (*Prospector, error) { + ret := &Prospector{ + config: config, + prospectorindex: make(map[string]*prospectorInfo), + prospectors: make(map[*prospectorInfo]*prospectorInfo), + from_beginning: from_beginning, + registrar: registrar_imp, + registrar_spool: registrar_imp.Connect(), + snapshot_chan: make(chan interface{}), + snapshot_sink: make(chan []*core.Snapshot), + output: spooler_imp.Connect(), + } + + if err := ret.init(); err != nil { + return nil, err + } + + pipeline.Register(ret) + + return ret, nil } func (p *Prospector) init() (err error) { - var have_previous bool - if have_previous, err = p.registrar.LoadPrevious(p.loadCallback); err != nil { - return - } - - if have_previous { - // -from-beginning=false flag should only affect the very first run (no previous state) - p.from_beginning = true - - // Pre-populate prospectors with what we had previously - for _, v := range p.prospectorindex { - p.prospectors[v] = v - } - } - - return + var have_previous bool + if have_previous, err = p.registrar.LoadPrevious(p.loadCallback); err != nil { + return + } + + if have_previous { + // -from-beginning=false flag should only affect the very first run (no previous state) + p.from_beginning = true + + // Pre-populate prospectors with what we had previously + for _, v := range p.prospectorindex { + p.prospectors[v] = v + } + } + + return } func (p *Prospector) loadCallback(file string, state *registrar.FileState) (core.Stream, error) { - p.prospectorindex[file] = newProspectorInfoFromFileState(file, state) - return p.prospectorindex[file], nil + p.prospectorindex[file] = newProspectorInfoFromFileState(file, state) + return p.prospectorindex[file], nil } func (p *Prospector) Run() { - defer func() { - p.Done() - }() - - // Handle any "-" (stdin) paths - but only once - stdin_started := false - for config_k, config := range p.config.Files { - for i, path := range config.Paths { - if path == "-" { - if !stdin_started { - // We need to check err - we cannot allow a nil stat - stat, err := os.Stdin.Stat() - if err != nil { - log.Error("stat(Stdin) failed: %s", err) - continue - } - - // Stdin is implicitly an orphaned fileinfo - info := newProspectorInfoFromFileInfo("-", stat) - info.orphaned = Orphaned_Yes - - // Store the reference so we can shut it down later - p.prospectors[info] = info - - // Start the harvester - p.startHarvesterWithOffset(info, &p.config.Files[config_k], 0) - - stdin_started = true - } - - // Remove it from the file list - config.Paths = append(config.Paths[:i], config.Paths[i+1:]...) - } - } - } + defer func() { + p.Done() + }() ProspectLoop: - for { - newlastscan := time.Now() - p.iteration++ // Overflow is allowed - - for config_k, config := range p.config.Files { - for _, path := range config.Paths { - // Scan - flag false so new files always start at beginning - p.scan(path, &p.config.Files[config_k]) - } - } - - // We only obey *from_beginning (which is stored in this) on startup, - // afterwards we force from beginning - p.from_beginning = true - - // Clean up the prospector collections - for _, info := range p.prospectors { - if info.orphaned >= Orphaned_Maybe { - if !info.isRunning() { - delete(p.prospectors, info) - } - } else { - if info.last_seen >= p.iteration { - continue - } - delete(p.prospectorindex, info.file) - info.orphaned = Orphaned_Maybe - } - if info.orphaned == Orphaned_Maybe { - info.orphaned = Orphaned_Yes - p.registrar_spool.Add(registrar.NewDeletedEvent(info)) - } - } - - // Flush the accumulated registrar events - p.registrar_spool.Send() - - p.lastscan = newlastscan - - // Defer next scan for a bit - now := time.Now() - scan_deadline := now.Add(p.config.General.ProspectInterval) - - DelayLoop: - for { - select { - case <-time.After(scan_deadline.Sub(now)): - break DelayLoop - case <-p.OnShutdown(): - break ProspectLoop - case <-p.snapshot_chan: - p.handleSnapshot() - case config := <-p.OnConfig(): - p.config = config - } - - now = time.Now() - if now.After(scan_deadline) { - break - } - } - } - - // Send stop signal to all harvesters, then wait for them, for quick shutdown - for _, info := range p.prospectors { - info.stop() - } - for _, info := range p.prospectors { - info.wait() - } - - // Disconnect from the registrar - p.registrar_spool.Close() - - log.Info("Prospector exiting") + for { + newlastscan := time.Now() + p.iteration++ // Overflow is allowed + + for config_k, config := range p.config.Files { + for _, path := range config.Paths { + // Scan - flag false so new files always start at beginning + p.scan(path, &p.config.Files[config_k]) + } + } + + // We only obey *from_beginning (which is stored in this) on startup, + // afterwards we force from beginning + p.from_beginning = true + + // Clean up the prospector collections + for _, info := range p.prospectors { + if info.orphaned >= Orphaned_Maybe { + if !info.isRunning() { + delete(p.prospectors, info) + } + } else { + if info.last_seen >= p.iteration { + continue + } + delete(p.prospectorindex, info.file) + info.orphaned = Orphaned_Maybe + } + if info.orphaned == Orphaned_Maybe { + info.orphaned = Orphaned_Yes + p.registrar_spool.Add(registrar.NewDeletedEvent(info)) + } + } + + // Flush the accumulated registrar events + p.registrar_spool.Send() + + p.lastscan = newlastscan + + // Defer next scan for a bit + now := time.Now() + scan_deadline := now.Add(p.config.General.ProspectInterval) + + DelayLoop: + for { + select { + case <-time.After(scan_deadline.Sub(now)): + break DelayLoop + case <-p.OnShutdown(): + break ProspectLoop + case <-p.snapshot_chan: + p.handleSnapshot() + case config := <-p.OnConfig(): + p.config = config + } + + now = time.Now() + if now.After(scan_deadline) { + break + } + } + } + + // Send stop signal to all harvesters, then wait for them, for quick shutdown + for _, info := range p.prospectors { + info.stop() + } + for _, info := range p.prospectors { + info.wait() + } + + // Disconnect from the registrar + p.registrar_spool.Close() + + log.Info("Prospector exiting") } func (p *Prospector) scan(path string, config *core.FileConfig) { - // Evaluate the path as a wildcards/shell glob - matches, err := filepath.Glob(path) - if err != nil { - log.Error("glob(%s) failed: %v", path, err) - return - } - - // Check any matched files to see if we need to start a harvester - for _, file := range matches { - // Check the current info against our index - info, is_known := p.prospectorindex[file] - - // Stat the file, following any symlinks - // TODO: Low priority. Trigger loadFileId here for Windows instead of - // waiting for Harvester or Registrar to do it - fileinfo, err := os.Stat(file) - if err == nil { - if fileinfo.IsDir() { - err = newProspectorSkipError("Directory") - } - } - - if err != nil { - // Do we know this entry? - if is_known { - if info.status != Status_Invalid { - // The current entry is not an error, orphan it so we can log one - info.orphaned = Orphaned_Yes - } else if info.err != err { - // The error is different, remove this entry we'll log a new one - delete(p.prospectors, info) - } else { - // The same error occurred - don't log it again - info.update(nil, p.iteration) - continue - } - } - - // This is a new error - info = newProspectorInfoInvalid(file, err) - info.update(nil, p.iteration) - - // Print a friendly log message - if _, ok := err.(*ProspectorSkipError); ok { - log.Info("Skipping %s: %s", file, err) - } else { - log.Error("Error prospecting %s: %s", file, err) - } - - p.prospectors[info] = info - p.prospectorindex[file] = info - continue - } else if is_known && info.status == Status_Invalid { - // We have an error stub and we've just successfully got fileinfo - // Mark is_known so we treat as a new file still - is_known = false - } - - // Conditions for starting a new harvester: - // - file path hasn't been seen before - // - the file's inode or device changed - if !is_known { - // Check for dead time, but only if the file modification time is before the last scan started - // This ensures we don't skip genuine creations with dead times less than 10s - if previous, previousinfo := p.lookupFileIds(file, fileinfo); previous != "" { - // Symlinks could mean we see the same file twice - skip if we have - if previousinfo == nil { - p.flagDuplicateError(file, info) - continue - } - - // This file was simply renamed (known inode+dev) - link the same harvester channel as the old file - log.Info("File rename was detected: %s -> %s", previous, file) - info = previousinfo - info.file = file - - p.registrar_spool.Add(registrar.NewRenamedEvent(info, file)) - } else { - // This is a new entry - info = newProspectorInfoFromFileInfo(file, fileinfo) - - if fileinfo.ModTime().Before(p.lastscan) && time.Since(fileinfo.ModTime()) > config.DeadTime { - // Old file, skip it, but push offset of file size so we start from the end if this file changes and needs picking up - log.Info("Skipping file (older than dead time of %v): %s", config.DeadTime, file) - - // Store the offset that we should resume from if we notice a modification - info.finish_offset = fileinfo.Size() - p.registrar_spool.Add(registrar.NewDiscoverEvent(info, file, fileinfo.Size(), fileinfo)) - } else { - // Process new file - log.Info("Launching harvester on new file: %s", file) - p.startHarvester(info, config) - } - } - - // Store the new entry - p.prospectors[info] = info - } else { - if !info.identity.SameAs(fileinfo) { - // Keep the old file in case we find it again shortly - info.orphaned = Orphaned_Maybe - - if previous, previousinfo := p.lookupFileIds(file, fileinfo); previous != "" { - // Symlinks could mean we see the same file twice - skip if we have - if previousinfo == nil { - p.flagDuplicateError(file, nil) - continue - } - - // This file was renamed from another file we know - link the same harvester channel as the old file - log.Info("File rename was detected: %s -> %s", previous, file) - info = previousinfo - info.file = file - - p.registrar_spool.Add(registrar.NewRenamedEvent(info, file)) - } else { - // File is not the same file we saw previously, it must have rotated and is a new file - log.Info("Launching harvester on rotated file: %s", file) - - // Forget about the previous harvester and let it continue on the old file - so start a new channel to use with the new harvester - info = newProspectorInfoFromFileInfo(file, fileinfo) - - // Process new file - p.startHarvester(info, config) - } - - // Store it - p.prospectors[info] = info - } - } - - // Resume stopped harvesters - resume := !info.isRunning() - if resume { - if info.status == Status_Resume { - // This is a filestate that was saved, resume the harvester - log.Info("Resuming harvester on a previously harvested file: %s", file) - } else if info.status == Status_Failed { - // Last attempt we failed to start, try again - log.Info("Attempting to restart failed harvester: %s", file) - } else if info.identity.Stat().ModTime() != fileinfo.ModTime() { - // Resume harvesting of an old file we've stopped harvesting from - log.Info("Resuming harvester on an old file that was just modified: %s", file) - } else { - resume = false - } - } - - info.update(fileinfo, p.iteration) - - if resume { - p.startHarvesterWithOffset(info, config, info.finish_offset) - } - - p.prospectorindex[file] = info - } // for each file matched by the glob + // Evaluate the path as a wildcards/shell glob + matches, err := filepath.Glob(path) + if err != nil { + log.Error("glob(%s) failed: %v", path, err) + return + } + + // Check any matched files to see if we need to start a harvester + for _, file := range matches { + // Check the current info against our index + info, is_known := p.prospectorindex[file] + + // Stat the file, following any symlinks + // TODO: Low priority. Trigger loadFileId here for Windows instead of + // waiting for Harvester or Registrar to do it + fileinfo, err := os.Stat(file) + if err == nil { + if fileinfo.IsDir() { + err = newProspectorSkipError("Directory") + } + } + + if err != nil { + // Do we know this entry? + if is_known { + if info.status != Status_Invalid { + // The current entry is not an error, orphan it so we can log one + info.orphaned = Orphaned_Maybe + } else if info.err.Error() == err.Error() { + // The same error occurred - don't log it again + info.update(nil, p.iteration) + continue + } + } + + // This is a new error + info = newProspectorInfoInvalid(file, err) + info.update(nil, p.iteration) + + // Print a friendly log message + if _, ok := err.(*ProspectorSkipError); ok { + log.Info("Skipping %s: %s", file, err) + } else { + log.Error("Error prospecting %s: %s", file, err) + } + + p.prospectors[info] = info + p.prospectorindex[file] = info + continue + } else if is_known && info.status == Status_Invalid { + // We have an error stub and we've just successfully got fileinfo + // Mark is_known so we treat as a new file still + is_known = false + } + + // Conditions for starting a new harvester: + // - file path hasn't been seen before + // - the file's inode or device changed + if !is_known { + // Check for dead time, but only if the file modification time is before the last scan started + // This ensures we don't skip genuine creations with dead times less than 10s + if previous, previousinfo := p.lookupFileIds(file, fileinfo); previous != "" { + // Symlinks could mean we see the same file twice - skip if we have + if previousinfo == nil { + p.flagDuplicateError(file, info) + continue + } + + // This file was simply renamed (known inode+dev) - link the same harvester channel as the old file + log.Info("File rename was detected: %s -> %s", previous, file) + info = previousinfo + info.file = file + + p.registrar_spool.Add(registrar.NewRenamedEvent(info, file)) + } else { + // This is a new entry + info = newProspectorInfoFromFileInfo(file, fileinfo) + + if fileinfo.ModTime().Before(p.lastscan) && time.Since(fileinfo.ModTime()) > config.DeadTime { + // Old file, skip it, but push offset of file size so we start from the end if this file changes and needs picking up + log.Info("Skipping file (older than dead time of %v): %s", config.DeadTime, file) + + // Store the offset that we should resume from if we notice a modification + info.finish_offset = fileinfo.Size() + p.registrar_spool.Add(registrar.NewDiscoverEvent(info, file, fileinfo.Size(), fileinfo)) + } else { + // Process new file + log.Info("Launching harvester on new file: %s", file) + p.startHarvester(info, config) + } + } + + // Store the new entry + p.prospectors[info] = info + } else { + if !info.identity.SameAs(fileinfo) { + // Keep the old file in case we find it again shortly + info.orphaned = Orphaned_Maybe + + if previous, previousinfo := p.lookupFileIds(file, fileinfo); previous != "" { + // Symlinks could mean we see the same file twice - skip if we have + if previousinfo == nil { + p.flagDuplicateError(file, nil) + continue + } + + // This file was renamed from another file we know - link the same harvester channel as the old file + log.Info("File rename was detected: %s -> %s", previous, file) + info = previousinfo + info.file = file + + p.registrar_spool.Add(registrar.NewRenamedEvent(info, file)) + } else { + // File is not the same file we saw previously, it must have rotated and is a new file + log.Info("Launching harvester on rotated file: %s", file) + + // Forget about the previous harvester and let it continue on the old file - so start a new channel to use with the new harvester + info = newProspectorInfoFromFileInfo(file, fileinfo) + + // Process new file + p.startHarvester(info, config) + } + + // Store it + p.prospectors[info] = info + } + } + + // Resume stopped harvesters + resume := !info.isRunning() + if resume { + if info.status == Status_Resume { + // This is a filestate that was saved, resume the harvester + log.Info("Resuming harvester on a previously harvested file: %s", file) + } else if info.status == Status_Failed { + // Last attempt we failed to start, try again + log.Info("Attempting to restart failed harvester: %s", file) + } else if info.identity.Stat().ModTime() != fileinfo.ModTime() { + // Resume harvesting of an old file we've stopped harvesting from + log.Info("Resuming harvester on an old file that was just modified: %s", file) + } else { + resume = false + } + } + + info.update(fileinfo, p.iteration) + + if resume { + p.startHarvesterWithOffset(info, config, info.finish_offset) + } + + p.prospectorindex[file] = info + } // for each file matched by the glob } func (p *Prospector) flagDuplicateError(file string, info *prospectorInfo) { - // Have we already logged this error? - if info != nil { - if info.status == Status_Invalid { - if skip_err, ok := info.err.(*ProspectorSkipError); ok && skip_err.message == "Duplicate" { - return - } - } - - // Remove the old info - delete(p.prospectors, info) - } - - // Flag duplicate error and save it - info = newProspectorInfoInvalid(file, newProspectorSkipError("Duplicate")) - info.update(nil, p.iteration) - p.prospectors[info] = info - p.prospectorindex[file] = info + // Have we already logged this error? + if info != nil { + if info.status == Status_Invalid { + if skip_err, ok := info.err.(*ProspectorSkipError); ok && skip_err.message == "Duplicate" { + return + } + } + } + + // Flag duplicate error and save it + info = newProspectorInfoInvalid(file, newProspectorSkipError("Duplicate")) + info.update(nil, p.iteration) + p.prospectors[info] = info + p.prospectorindex[file] = info } func (p *Prospector) startHarvester(info *prospectorInfo, fileconfig *core.FileConfig) { - var offset int64 + var offset int64 - if p.from_beginning { - offset = 0 - } else { - offset = info.identity.Stat().Size() - } + if p.from_beginning { + offset = 0 + } else { + offset = info.identity.Stat().Size() + } - // Send a new file event to allow registrar to begin persisting for this harvester - p.registrar_spool.Add(registrar.NewDiscoverEvent(info, info.file, offset, info.identity.Stat())) + // Send a new file event to allow registrar to begin persisting for this harvester + p.registrar_spool.Add(registrar.NewDiscoverEvent(info, info.file, offset, info.identity.Stat())) - p.startHarvesterWithOffset(info, fileconfig, offset) + p.startHarvesterWithOffset(info, fileconfig, offset) } func (p *Prospector) startHarvesterWithOffset(info *prospectorInfo, fileconfig *core.FileConfig, offset int64) { - // TODO - hook in a shutdown channel - info.harvester = harvester.NewHarvester(info, p.config, fileconfig, offset) - info.running = true - info.status = Status_Ok - info.harvester.Start(p.output) + // TODO - hook in a shutdown channel + info.harvester = harvester.NewHarvester(info, p.config, &fileconfig.StreamConfig, offset) + info.running = true + info.status = Status_Ok + info.harvester.Start(p.output) } func (p *Prospector) lookupFileIds(file string, info os.FileInfo) (string, *prospectorInfo) { - for _, ki := range p.prospectors { - if ki.status == Status_Invalid { - // Don't consider error placeholders - continue - } - if ki.orphaned == Orphaned_No && ki.file == file { - // We already know the prospector info for this file doesn't match, so don't check again - continue - } - if ki.identity.SameAs(info) { - // Already seen? - if ki.last_seen == p.iteration { - return ki.file, nil - } - - // Found previous information, remove it and return it (it will be added again) - delete(p.prospectors, ki) - if ki.orphaned == Orphaned_No { - delete(p.prospectorindex, ki.file) - } else { - ki.orphaned = Orphaned_No - } - return ki.file, ki - } - } - - return "", nil + for _, ki := range p.prospectors { + if ki.status == Status_Invalid { + // Don't consider error placeholders + continue + } + if ki.orphaned == Orphaned_No && ki.file == file { + // We already know the prospector info for this file doesn't match, so don't check again + continue + } + if ki.identity.SameAs(info) { + // Already seen? + if ki.last_seen == p.iteration { + return ki.file, nil + } + + // Found previous information, remove it and return it (it will be added again) + delete(p.prospectors, ki) + if ki.orphaned == Orphaned_No { + delete(p.prospectorindex, ki.file) + } else { + ki.orphaned = Orphaned_No + } + return ki.file, ki + } + } + + return "", nil } func (p *Prospector) Snapshot() []*core.Snapshot { - select { - case p.snapshot_chan <- 1: - // Timeout after 5 seconds - case <-time.After(5 * time.Second): - ret := core.NewSnapshot("Prospector") - ret.AddEntry("Error", "Timeout") - return []*core.Snapshot{ret} - } - - return <-p.snapshot_sink + select { + case p.snapshot_chan <- 1: + // Timeout after 5 seconds + case <-time.After(5 * time.Second): + ret := core.NewSnapshot("Prospector") + ret.AddEntry("Error", "Timeout") + return []*core.Snapshot{ret} + } + + return <-p.snapshot_sink } func (p *Prospector) handleSnapshot() { - snapshots := make([]*core.Snapshot, 1) + snapshots := make([]*core.Snapshot, 1) - snapshots[0] = core.NewSnapshot("Prospector") - snapshots[0].AddEntry("Watched files", len(p.prospectorindex)) - snapshots[0].AddEntry("Active states", len(p.prospectors)) + snapshots[0] = core.NewSnapshot("Prospector") + snapshots[0].AddEntry("Watched files", len(p.prospectorindex)) + snapshots[0].AddEntry("Active states", len(p.prospectors)) - for _, info := range p.prospectorindex { - snapshots = append(snapshots, p.snapshotInfo(info)) - } + for _, info := range p.prospectorindex { + snapshots = append(snapshots, p.snapshotInfo(info)) + } - for _, info := range p.prospectors { - if info.orphaned == Orphaned_No { - continue - } - snapshots = append(snapshots, p.snapshotInfo(info)) - } + for _, info := range p.prospectors { + if info.orphaned == Orphaned_No { + continue + } + snapshots = append(snapshots, p.snapshotInfo(info)) + } - p.snapshot_sink <- snapshots + p.snapshot_sink <- snapshots } func (p *Prospector) snapshotInfo(info *prospectorInfo) *core.Snapshot { - var extra string - var status string - - if info.file == "-" { - extra = "Stdin / " - } else { - switch (info.orphaned) { - case Orphaned_Maybe: - extra = "Orphan? / " - case Orphaned_Yes: - extra = "Orphan / " - } - } - - switch (info.status) { - default: - if info.running { - status = "Running" - } else { - status = "Dead" - } - case Status_Resume: - status = "Resuming" - case Status_Failed: - status = fmt.Sprintf("Failed: %s", info.err) - case Status_Invalid: - if _, ok := info.err.(*ProspectorSkipError); ok { - status = fmt.Sprintf("Skipped (%s)", info.err) - } else { - status = fmt.Sprintf("Error: %s", info.err) - } - } - - snap := core.NewSnapshot(fmt.Sprintf("\"State: %s (%s%p)\"", info.file, extra, info)) - snap.AddEntry("Status", status) - - if info.running { - if sub_snap := info.getSnapshot(); sub_snap != nil { - snap.AddSub(sub_snap) - } - } - - return snap + var extra string + var status string + + if info.file == "-" { + extra = "Stdin / " + } else { + switch info.orphaned { + case Orphaned_Maybe: + extra = "Orphan? / " + case Orphaned_Yes: + extra = "Orphan / " + } + } + + switch info.status { + default: + if info.running { + status = "Running" + } else { + status = "Dead" + } + case Status_Resume: + status = "Resuming" + case Status_Failed: + status = fmt.Sprintf("Failed: %s", info.err) + case Status_Invalid: + if _, ok := info.err.(*ProspectorSkipError); ok { + status = fmt.Sprintf("Skipped (%s)", info.err) + } else { + status = fmt.Sprintf("Error: %s", info.err) + } + } + + snap := core.NewSnapshot(fmt.Sprintf("\"State: %s (%s%p)\"", info.file, extra, info)) + snap.AddEntry("Status", status) + + if info.running { + if sub_snap := info.getSnapshot(); sub_snap != nil { + snap.AddSub(sub_snap) + } + } + + return snap } diff --git a/src/lc-lib/prospector/snapshot.go b/src/lc-lib/prospector/snapshot.go index d6f11b02..2cc2b151 100644 --- a/src/lc-lib/prospector/snapshot.go +++ b/src/lc-lib/prospector/snapshot.go @@ -12,7 +12,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. -*/ + */ package prospector diff --git a/src/lc-lib/publisher/logging.go b/src/lc-lib/publisher/logging.go index c96d9ea6..12c857b3 100644 --- a/src/lc-lib/publisher/logging.go +++ b/src/lc-lib/publisher/logging.go @@ -21,5 +21,5 @@ import "github.com/op/go-logging" var log *logging.Logger func init() { - log = logging.MustGetLogger("publisher") + log = logging.MustGetLogger("publisher") } diff --git a/src/lc-lib/publisher/pending_payload.go b/src/lc-lib/publisher/pending_payload.go index bc545cfa..a01f8fa5 100644 --- a/src/lc-lib/publisher/pending_payload.go +++ b/src/lc-lib/publisher/pending_payload.go @@ -17,113 +17,113 @@ package publisher import ( - "bytes" - "compress/zlib" - "encoding/binary" - "errors" - "lc-lib/core" - "time" + "bytes" + "compress/zlib" + "encoding/binary" + "errors" + "github.com/driskell/log-courier/src/lc-lib/core" + "time" ) var ( - ErrPayloadCorrupt = errors.New("Payload is corrupt") + ErrPayloadCorrupt = errors.New("Payload is corrupt") ) type pendingPayload struct { - next *pendingPayload - nonce string - events []*core.EventDescriptor - last_sequence int - sequence_len int - ack_events int - processed int - payload []byte - timeout time.Time + next *pendingPayload + nonce string + events []*core.EventDescriptor + last_sequence int + sequence_len int + ack_events int + processed int + payload []byte + timeout time.Time } func newPendingPayload(events []*core.EventDescriptor, nonce string, timeout time.Duration) (*pendingPayload, error) { - payload := &pendingPayload{ - events: events, - nonce: nonce, - timeout: time.Now().Add(timeout), - } + payload := &pendingPayload{ + events: events, + nonce: nonce, + timeout: time.Now().Add(timeout), + } - if err := payload.Generate(); err != nil { - return nil, err - } + if err := payload.Generate(); err != nil { + return nil, err + } - return payload, nil + return payload, nil } func (pp *pendingPayload) Generate() (err error) { - var buffer bytes.Buffer - - // Assertion - if len(pp.events) == 0 { - return ErrPayloadCorrupt - } - - // Begin with the nonce - if _, err = buffer.Write([]byte(pp.nonce)[0:16]); err != nil { - return - } - - var compressor *zlib.Writer - if compressor, err = zlib.NewWriterLevel(&buffer, 3); err != nil { - return - } - - // Append all the events - for _, event := range pp.events[pp.ack_events:] { - if err = binary.Write(compressor, binary.BigEndian, uint32(len(event.Event))); err != nil { - return - } - if _, err = compressor.Write(event.Event); err != nil { - return - } - } - - compressor.Close() - - pp.payload = buffer.Bytes() - pp.last_sequence = 0 - pp.sequence_len = len(pp.events) - pp.ack_events - - return + var buffer bytes.Buffer + + // Assertion + if len(pp.events) == 0 { + return ErrPayloadCorrupt + } + + // Begin with the nonce + if _, err = buffer.Write([]byte(pp.nonce)[0:16]); err != nil { + return + } + + var compressor *zlib.Writer + if compressor, err = zlib.NewWriterLevel(&buffer, 3); err != nil { + return + } + + // Append all the events + for _, event := range pp.events[pp.ack_events:] { + if err = binary.Write(compressor, binary.BigEndian, uint32(len(event.Event))); err != nil { + return + } + if _, err = compressor.Write(event.Event); err != nil { + return + } + } + + compressor.Close() + + pp.payload = buffer.Bytes() + pp.last_sequence = 0 + pp.sequence_len = len(pp.events) - pp.ack_events + + return } func (pp *pendingPayload) Ack(sequence int) (int, bool) { - if sequence <= pp.last_sequence { - // No change - return 0, false - } else if sequence >= pp.sequence_len { - // Full ACK - lines := pp.sequence_len - pp.last_sequence - pp.ack_events = len(pp.events) - pp.last_sequence = sequence - pp.payload = nil - return lines, true - } - - lines := sequence - pp.last_sequence - pp.ack_events += lines - pp.last_sequence = sequence - pp.payload = nil - return lines, false + if sequence <= pp.last_sequence { + // No change + return 0, false + } else if sequence >= pp.sequence_len { + // Full ACK + lines := pp.sequence_len - pp.last_sequence + pp.ack_events = len(pp.events) + pp.last_sequence = sequence + pp.payload = nil + return lines, true + } + + lines := sequence - pp.last_sequence + pp.ack_events += lines + pp.last_sequence = sequence + pp.payload = nil + return lines, false } func (pp *pendingPayload) HasAck() bool { - return pp.ack_events != 0 + return pp.ack_events != 0 } func (pp *pendingPayload) Complete() bool { - return len(pp.events) == 0 + return len(pp.events) == 0 } func (pp *pendingPayload) Rollup() []*core.EventDescriptor { - pp.processed += pp.ack_events - rollup := pp.events[:pp.ack_events] - pp.events = pp.events[pp.ack_events:] - pp.ack_events = 0 - return rollup + pp.processed += pp.ack_events + rollup := pp.events[:pp.ack_events] + pp.events = pp.events[pp.ack_events:] + pp.ack_events = 0 + return rollup } diff --git a/src/lc-lib/publisher/pending_payload_test.go b/src/lc-lib/publisher/pending_payload_test.go index f7ddba37..6a56eaaa 100644 --- a/src/lc-lib/publisher/pending_payload_test.go +++ b/src/lc-lib/publisher/pending_payload_test.go @@ -12,14 +12,14 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. -*/ + */ package publisher import ( - "lc-lib/core" - "time" + "github.com/driskell/log-courier/src/lc-lib/core" "testing" + "time" ) const ( diff --git a/src/lc-lib/publisher/publisher.go b/src/lc-lib/publisher/publisher.go index 84f276ea..469f97d0 100644 --- a/src/lc-lib/publisher/publisher.go +++ b/src/lc-lib/publisher/publisher.go @@ -20,629 +20,652 @@ package publisher import ( - "bytes" - "encoding/binary" - "errors" - "fmt" - "lc-lib/core" - "lc-lib/registrar" - "math/rand" - "sync" - "time" + "bytes" + "encoding/binary" + "errors" + "fmt" + "github.com/driskell/log-courier/src/lc-lib/core" + "github.com/driskell/log-courier/src/lc-lib/registrar" + "math/rand" + "sync" + "time" ) var ( - ErrNetworkTimeout = errors.New("Server did not respond within network timeout") - ErrNetworkPing = errors.New("Server did not respond to PING") + ErrNetworkTimeout = errors.New("Server did not respond within network timeout") + ErrNetworkPing = errors.New("Server did not respond to PING") ) const ( - // TODO(driskell): Make the idle timeout configurable like the network timeout is? - keepalive_timeout time.Duration = 900 * time.Second + // TODO(driskell): Make the idle timeout configurable like the network timeout is? + keepalive_timeout time.Duration = 900 * time.Second ) const ( - Status_Disconnected = iota - Status_Connected - Status_Reconnecting + Status_Disconnected = iota + Status_Connected + Status_Reconnecting ) +type NullEventSpool struct { +} + +func newNullEventSpool() *NullEventSpool { + return &NullEventSpool{} +} + +func (s *NullEventSpool) Close() { +} + +func (s *NullEventSpool) Add(event registrar.EventProcessor) { +} + +func (s *NullEventSpool) Send() { +} + type Publisher struct { - core.PipelineSegment - core.PipelineConfigReceiver - core.PipelineSnapshotProvider - - sync.RWMutex - - config *core.NetworkConfig - transport core.Transport - status int - can_send <-chan int - pending_ping bool - pending_payloads map[string]*pendingPayload - first_payload *pendingPayload - last_payload *pendingPayload - num_payloads int64 - out_of_sync int - input chan []*core.EventDescriptor - registrar_spool *registrar.RegistrarEventSpool - shutdown bool - line_count int64 - retry_count int64 - seconds_no_ack int - - timeout_count int64 - line_speed float64 - last_line_count int64 - last_retry_count int64 - last_measurement time.Time + core.PipelineSegment + core.PipelineConfigReceiver + core.PipelineSnapshotProvider + + sync.RWMutex + + config *core.NetworkConfig + transport core.Transport + status int + can_send <-chan int + pending_ping bool + pending_payloads map[string]*pendingPayload + first_payload *pendingPayload + last_payload *pendingPayload + num_payloads int64 + out_of_sync int + input chan []*core.EventDescriptor + registrar_spool registrar.EventSpooler + shutdown bool + line_count int64 + retry_count int64 + seconds_no_ack int + + timeout_count int64 + line_speed float64 + last_line_count int64 + last_retry_count int64 + last_measurement time.Time } -func NewPublisher(pipeline *core.Pipeline, config *core.NetworkConfig, registrar_imp *registrar.Registrar) (*Publisher, error) { - ret := &Publisher{ - config: config, - input: make(chan []*core.EventDescriptor, 1), - registrar_spool: registrar_imp.Connect(), - } +func NewPublisher(pipeline *core.Pipeline, config *core.NetworkConfig, registrar registrar.Registrator) (*Publisher, error) { + ret := &Publisher{ + config: config, + input: make(chan []*core.EventDescriptor, 1), + } + + if registrar == nil { + ret.registrar_spool = newNullEventSpool() + } else { + ret.registrar_spool = registrar.Connect() + } - if err := ret.init(); err != nil { - return nil, err - } + if err := ret.init(); err != nil { + return nil, err + } - pipeline.Register(ret) + pipeline.Register(ret) - return ret, nil + return ret, nil } func (p *Publisher) init() error { - var err error + var err error - p.pending_payloads = make(map[string]*pendingPayload) + p.pending_payloads = make(map[string]*pendingPayload) - // Set up the selected transport - if err = p.loadTransport(); err != nil { - return err - } + // Set up the selected transport + if err = p.loadTransport(); err != nil { + return err + } - return nil + return nil } func (p *Publisher) loadTransport() error { - transport, err := p.config.TransportFactory.NewTransport(p.config) - if err != nil { - return err - } + transport, err := p.config.TransportFactory.NewTransport(p.config) + if err != nil { + return err + } - p.transport = transport + p.transport = transport - return nil + return nil } func (p *Publisher) Connect() chan<- []*core.EventDescriptor { - return p.input + return p.input } func (p *Publisher) Run() { - defer func() { - p.Done() - }() - - var input_toggle <-chan []*core.EventDescriptor - var retry_payload *pendingPayload - var err error - var reload int - - timer := time.NewTimer(keepalive_timeout) - stats_timer := time.NewTimer(time.Second) - - control_signal := p.OnShutdown() - delay_shutdown := func() { - // Flag shutdown for when we finish pending payloads - // TODO: Persist pending payloads and resume? Quicker shutdown - log.Warning("Delaying shutdown to wait for pending responses from the server") - control_signal = nil - p.shutdown = true - p.can_send = nil - input_toggle = nil - } + defer func() { + p.Done() + }() + + var input_toggle <-chan []*core.EventDescriptor + var retry_payload *pendingPayload + var err error + var reload int + + timer := time.NewTimer(keepalive_timeout) + stats_timer := time.NewTimer(time.Second) + + control_signal := p.OnShutdown() + delay_shutdown := func() { + // Flag shutdown for when we finish pending payloads + // TODO: Persist pending payloads and resume? Quicker shutdown + log.Warning("Delaying shutdown to wait for pending responses from the server") + control_signal = nil + p.shutdown = true + p.can_send = nil + input_toggle = nil + } PublishLoop: - for { - // Do we need to reload transport? - if reload == core.Reload_Transport { - // Shutdown and reload transport - p.transport.Shutdown() - - if err = p.loadTransport(); err != nil { - log.Error("The new transport configuration failed to apply: %s", err) - } - - reload = core.Reload_None - } else if reload != core.Reload_None { - reload = core.Reload_None - } - - if err = p.transport.Init(); err != nil { - log.Error("Transport init failed: %s", err) - - now := time.Now() - reconnect_due := now.Add(p.config.Reconnect) - - ReconnectTimeLoop: - for { - - select { - case <-time.After(reconnect_due.Sub(now)): - break ReconnectTimeLoop - case <-control_signal: - // TODO: Persist pending payloads and resume? Quicker shutdown - if p.num_payloads == 0 { - break PublishLoop - } - - delay_shutdown() - case config := <-p.OnConfig(): - // Apply and check for changes - reload = p.reloadConfig(&config.Network) - - // If a change and no pending payloads, process immediately - if reload != core.Reload_None && p.num_payloads == 0 { - break ReconnectTimeLoop - } - } - - now = time.Now() - if now.After(reconnect_due) { - break - } - } - - continue - } - - p.Lock() - p.status = Status_Connected - p.Unlock() - - timer.Reset(keepalive_timeout) - stats_timer.Reset(time.Second) - - p.pending_ping = false - input_toggle = nil - p.can_send = p.transport.CanSend() - - SelectLoop: - for { - select { - case <-p.can_send: - // Resend payloads from full retry first - if retry_payload != nil { - // Do we need to regenerate the payload? - if retry_payload.payload == nil { - if err = retry_payload.Generate(); err != nil { - break SelectLoop - } - } - - // Reset timeout - retry_payload.timeout = time.Now().Add(p.config.Timeout) - - log.Debug("Send now open: Retrying next payload") - - // Send the payload again - if err = p.transport.Write("JDAT", retry_payload.payload); err != nil { - break SelectLoop - } - - // Expect an ACK within network timeout if this is the first of the retries - if p.first_payload == retry_payload { - timer.Reset(p.config.Timeout) - } - - // Move to next non-empty payload - for { - retry_payload = retry_payload.next - if retry_payload == nil || retry_payload.ack_events != len(retry_payload.events) { - break - } - } - - break - } else if p.out_of_sync != 0 { - var resent bool - if resent, err = p.checkResend(); err != nil { - break SelectLoop - } else if resent { - log.Debug("Send now open: Resent a timed out payload") - // Expect an ACK within network timeout - timer.Reset(p.config.Timeout) - break - } - } - - // No pending payloads, are we shutting down? Skip if so - if p.shutdown { - break - } - - log.Debug("Send now open: Awaiting events for new payload") - - // Enable event wait - input_toggle = p.input - case events := <-input_toggle: - log.Debug("Sending new payload of %d events", len(events)) - - // Send - if err = p.sendNewPayload(events); err != nil { - break SelectLoop - } - - // Wait for send signal again - input_toggle = nil - - if p.num_payloads >= p.config.MaxPendingPayloads { - // Too many pending payloads, disable send temporarily - p.can_send = nil - log.Debug("Pending payload limit reached") - } - - // Expect an ACK within network timeout if this is first payload after idle - // Otherwise leave the previous timer - if p.num_payloads == 1 { - timer.Reset(p.config.Timeout) - } - case data := <-p.transport.Read(): - var signature, message []byte - - // Error? Or data? - switch data.(type) { - case error: - err = data.(error) - break SelectLoop - default: - signature = data.([][]byte)[0] - message = data.([][]byte)[1] - } - - switch { - case bytes.Compare(signature, []byte("PONG")) == 0: - if err = p.processPong(message); err != nil { - break SelectLoop - } - case bytes.Compare(signature, []byte("ACKN")) == 0: - if err = p.processAck(message); err != nil { - break SelectLoop - } - default: - err = fmt.Errorf("Unknown message received: % X", signature) - break SelectLoop - } - - // If no more pending payloads, set keepalive, otherwise reset to network timeout - if p.num_payloads == 0 { - // Handle shutdown - if p.shutdown { - break PublishLoop - } else if reload != core.Reload_None { - break SelectLoop - } - log.Debug("No more pending payloads, entering idle") - timer.Reset(keepalive_timeout) - } else { - log.Debug("%d payloads still pending, resetting timeout", p.num_payloads) - timer.Reset(p.config.Timeout) - } - case <-timer.C: - // If we have pending payloads, we should've received something by now - if p.num_payloads != 0 { - err = ErrNetworkTimeout - break SelectLoop - } - - // If we haven't received a PONG yet this is a timeout - if p.pending_ping { - err = ErrNetworkPing - break SelectLoop - } - - log.Debug("Idle timeout: sending PING") - - // Send a ping and expect a pong back (eventually) - // If we receive an ACK first, that's fine we'll reset timer - // But after those ACKs we should get a PONG - if err = p.transport.Write("PING", nil); err != nil { - break SelectLoop - } - - p.pending_ping = true - - // We may have just filled the send buffer - input_toggle = nil - - // Allow network timeout to receive something - timer.Reset(p.config.Timeout) - case <-control_signal: - // If no pending payloads, simply end - if p.num_payloads == 0 { - break PublishLoop - } - - delay_shutdown() - case config := <-p.OnConfig(): - // Apply and check for changes - reload = p.reloadConfig(&config.Network) - - // If a change and no pending payloads, process immediately - if reload != core.Reload_None && p.num_payloads == 0 { - break SelectLoop - } - - p.can_send = nil - case <-stats_timer.C: - p.updateStatistics(Status_Connected, nil) - stats_timer.Reset(time.Second) - } - } - - if err != nil { - // If we're shutting down and we hit a timeout and aren't out of sync - // We can then quit - as we'd be resending payloads anyway - if p.shutdown && p.out_of_sync == 0 { - log.Error("Transport error: %s", err) - break PublishLoop - } - - p.updateStatistics(Status_Reconnecting, err) - - // An error occurred, reconnect after timeout - log.Error("Transport error, will try again: %s", err) - time.Sleep(p.config.Reconnect) - } else { - log.Info("Reconnecting transport") - - p.updateStatistics(Status_Reconnecting, nil) - } - - retry_payload = p.first_payload - } - - p.transport.Shutdown() - - // Disconnect from registrar - p.registrar_spool.Close() - - log.Info("Publisher exiting") + for { + // Do we need to reload transport? + if reload == core.Reload_Transport { + // Shutdown and reload transport + p.transport.Shutdown() + + if err = p.loadTransport(); err != nil { + log.Error("The new transport configuration failed to apply: %s", err) + } + + reload = core.Reload_None + } else if reload != core.Reload_None { + reload = core.Reload_None + } + + if err = p.transport.Init(); err != nil { + log.Error("Transport init failed: %s", err) + + now := time.Now() + reconnect_due := now.Add(p.config.Reconnect) + + ReconnectTimeLoop: + for { + + select { + case <-time.After(reconnect_due.Sub(now)): + break ReconnectTimeLoop + case <-control_signal: + // TODO: Persist pending payloads and resume? Quicker shutdown + if p.num_payloads == 0 { + break PublishLoop + } + + delay_shutdown() + case config := <-p.OnConfig(): + // Apply and check for changes + reload = p.reloadConfig(&config.Network) + + // If a change and no pending payloads, process immediately + if reload != core.Reload_None && p.num_payloads == 0 { + break ReconnectTimeLoop + } + } + + now = time.Now() + if now.After(reconnect_due) { + break + } + } + + continue + } + + p.Lock() + p.status = Status_Connected + p.Unlock() + + timer.Reset(keepalive_timeout) + stats_timer.Reset(time.Second) + + p.pending_ping = false + input_toggle = nil + p.can_send = p.transport.CanSend() + + SelectLoop: + for { + select { + case <-p.can_send: + // Resend payloads from full retry first + if retry_payload != nil { + // Do we need to regenerate the payload? + if retry_payload.payload == nil { + if err = retry_payload.Generate(); err != nil { + break SelectLoop + } + } + + // Reset timeout + retry_payload.timeout = time.Now().Add(p.config.Timeout) + + log.Debug("Send now open: Retrying next payload") + + // Send the payload again + if err = p.transport.Write("JDAT", retry_payload.payload); err != nil { + break SelectLoop + } + + // Expect an ACK within network timeout if this is the first of the retries + if p.first_payload == retry_payload { + timer.Reset(p.config.Timeout) + } + + // Move to next non-empty payload + for { + retry_payload = retry_payload.next + if retry_payload == nil || retry_payload.ack_events != len(retry_payload.events) { + break + } + } + + break + } else if p.out_of_sync != 0 { + var resent bool + if resent, err = p.checkResend(); err != nil { + break SelectLoop + } else if resent { + log.Debug("Send now open: Resent a timed out payload") + // Expect an ACK within network timeout + timer.Reset(p.config.Timeout) + break + } + } + + // No pending payloads, are we shutting down? Skip if so + if p.shutdown { + break + } + + log.Debug("Send now open: Awaiting events for new payload") + + // Enable event wait + input_toggle = p.input + case events := <-input_toggle: + log.Debug("Sending new payload of %d events", len(events)) + + // Send + if err = p.sendNewPayload(events); err != nil { + break SelectLoop + } + + // Wait for send signal again + input_toggle = nil + + if p.num_payloads >= p.config.MaxPendingPayloads { + // Too many pending payloads, disable send temporarily + p.can_send = nil + log.Debug("Pending payload limit of %d reached", p.config.MaxPendingPayloads) + } else { + log.Debug("%d/%d pending payloads now in transit", p.num_payloads, p.config.MaxPendingPayloads) + } + + // Expect an ACK within network timeout if this is first payload after idle + // Otherwise leave the previous timer + if p.num_payloads == 1 { + timer.Reset(p.config.Timeout) + } + case data := <-p.transport.Read(): + var signature, message []byte + + // Error? Or data? + switch data.(type) { + case error: + err = data.(error) + break SelectLoop + default: + signature = data.([][]byte)[0] + message = data.([][]byte)[1] + } + + switch { + case bytes.Compare(signature, []byte("PONG")) == 0: + if err = p.processPong(message); err != nil { + break SelectLoop + } + case bytes.Compare(signature, []byte("ACKN")) == 0: + if err = p.processAck(message); err != nil { + break SelectLoop + } + default: + err = fmt.Errorf("Unknown message received: % X", signature) + break SelectLoop + } + + // If no more pending payloads, set keepalive, otherwise reset to network timeout + if p.num_payloads == 0 { + // Handle shutdown + if p.shutdown { + break PublishLoop + } else if reload != core.Reload_None { + break SelectLoop + } + log.Debug("No more pending payloads, entering idle") + timer.Reset(keepalive_timeout) + } else { + log.Debug("%d payloads still pending, resetting timeout", p.num_payloads) + timer.Reset(p.config.Timeout) + } + case <-timer.C: + // If we have pending payloads, we should've received something by now + if p.num_payloads != 0 { + err = ErrNetworkTimeout + break SelectLoop + } + + // If we haven't received a PONG yet this is a timeout + if p.pending_ping { + err = ErrNetworkPing + break SelectLoop + } + + log.Debug("Idle timeout: sending PING") + + // Send a ping and expect a pong back (eventually) + // If we receive an ACK first, that's fine we'll reset timer + // But after those ACKs we should get a PONG + if err = p.transport.Write("PING", nil); err != nil { + break SelectLoop + } + + p.pending_ping = true + + // We may have just filled the send buffer + input_toggle = nil + + // Allow network timeout to receive something + timer.Reset(p.config.Timeout) + case <-control_signal: + // If no pending payloads, simply end + if p.num_payloads == 0 { + break PublishLoop + } + + delay_shutdown() + case config := <-p.OnConfig(): + // Apply and check for changes + reload = p.reloadConfig(&config.Network) + + // If a change and no pending payloads, process immediately + if reload != core.Reload_None && p.num_payloads == 0 { + break SelectLoop + } + + p.can_send = nil + case <-stats_timer.C: + p.updateStatistics(Status_Connected, nil) + stats_timer.Reset(time.Second) + } + } + + if err != nil { + // If we're shutting down and we hit a timeout and aren't out of sync + // We can then quit - as we'd be resending payloads anyway + if p.shutdown && p.out_of_sync == 0 { + log.Error("Transport error: %s", err) + break PublishLoop + } + + p.updateStatistics(Status_Reconnecting, err) + + // An error occurred, reconnect after timeout + log.Error("Transport error, will try again: %s", err) + time.Sleep(p.config.Reconnect) + } else { + log.Info("Reconnecting transport") + + p.updateStatistics(Status_Reconnecting, nil) + } + + retry_payload = p.first_payload + } + + p.transport.Shutdown() + + // Disconnect from registrar + p.registrar_spool.Close() + + log.Info("Publisher exiting") } func (p *Publisher) reloadConfig(new_config *core.NetworkConfig) int { - old_config := p.config - p.config = new_config - - // Transport reload will return whether we need a full reload or not - reload := p.transport.ReloadConfig(new_config) - if reload == core.Reload_Transport { - return core.Reload_Transport - } - - // Same servers? - if len(new_config.Servers) != len(old_config.Servers) { - return core.Reload_Servers - } - - for i := range new_config.Servers { - if new_config.Servers[i] != old_config.Servers[i] { - return core.Reload_Servers - } - } - - return reload + old_config := p.config + p.config = new_config + + // Transport reload will return whether we need a full reload or not + reload := p.transport.ReloadConfig(new_config) + if reload == core.Reload_Transport { + return core.Reload_Transport + } + + // Same servers? + if len(new_config.Servers) != len(old_config.Servers) { + return core.Reload_Servers + } + + for i := range new_config.Servers { + if new_config.Servers[i] != old_config.Servers[i] { + return core.Reload_Servers + } + } + + return reload } func (p *Publisher) updateStatistics(status int, err error) { - p.Lock() + p.Lock() - p.status = status + p.status = status - p.line_speed = core.CalculateSpeed(time.Since(p.last_measurement), p.line_speed, float64(p.line_count - p.last_line_count), &p.seconds_no_ack) + p.line_speed = core.CalculateSpeed(time.Since(p.last_measurement), p.line_speed, float64(p.line_count-p.last_line_count), &p.seconds_no_ack) - p.last_line_count = p.line_count - p.last_retry_count = p.retry_count - p.last_measurement = time.Now() + p.last_line_count = p.line_count + p.last_retry_count = p.retry_count + p.last_measurement = time.Now() - if err == ErrNetworkTimeout || err == ErrNetworkPing { - p.timeout_count++ - } + if err == ErrNetworkTimeout || err == ErrNetworkPing { + p.timeout_count++ + } - p.Unlock() + p.Unlock() } func (p *Publisher) checkResend() (bool, error) { - // We're out of sync (received ACKs for later payloads but not earlier ones) - // Check timeouts of earlier payloads and resend if necessary - if payload := p.first_payload; payload.timeout.Before(time.Now()) { - p.retry_count++ - - // Do we need to regenerate the payload? - if payload.payload == nil { - if err := payload.Generate(); err != nil { - return false, err - } - } - - // Update timeout - payload.timeout = time.Now().Add(p.config.Timeout) - - // Requeue the payload - p.first_payload = payload.next - payload.next = nil - p.last_payload.next = payload - p.last_payload = payload - - // Send the payload again - if err := p.transport.Write("JDAT", payload.payload); err != nil { - return false, err - } - - return true, nil - } - - return false, nil + // We're out of sync (received ACKs for later payloads but not earlier ones) + // Check timeouts of earlier payloads and resend if necessary + if payload := p.first_payload; payload.timeout.Before(time.Now()) { + p.retry_count++ + + // Do we need to regenerate the payload? + if payload.payload == nil { + if err := payload.Generate(); err != nil { + return false, err + } + } + + // Update timeout + payload.timeout = time.Now().Add(p.config.Timeout) + + // Requeue the payload + p.first_payload = payload.next + payload.next = nil + p.last_payload.next = payload + p.last_payload = payload + + // Send the payload again + if err := p.transport.Write("JDAT", payload.payload); err != nil { + return false, err + } + + return true, nil + } + + return false, nil } func (p *Publisher) generateNonce() string { - // This could maybe be made a bit more efficient - nonce := make([]byte, 16) - for i := 0; i < 16; i++ { - nonce[i] = byte(rand.Intn(255)) - } - return string(nonce) + // This could maybe be made a bit more efficient + nonce := make([]byte, 16) + for i := 0; i < 16; i++ { + nonce[i] = byte(rand.Intn(255)) + } + return string(nonce) } func (p *Publisher) sendNewPayload(events []*core.EventDescriptor) (err error) { - // Calculate a nonce - nonce := p.generateNonce() - for { - if _, found := p.pending_payloads[nonce]; !found { - break - } - // Collision - generate again - should be extremely rare - nonce = p.generateNonce() - } - - var payload *pendingPayload - if payload, err = newPendingPayload(events, nonce, p.config.Timeout); err != nil { - return - } - - // Save pending payload until we receive ack, and discard buffer - p.pending_payloads[nonce] = payload - if p.first_payload == nil { - p.first_payload = payload - } else { - p.last_payload.next = payload - } - p.last_payload = payload - - p.Lock() - p.num_payloads++ - p.Unlock() - - return p.transport.Write("JDAT", payload.payload) + // Calculate a nonce + nonce := p.generateNonce() + for { + if _, found := p.pending_payloads[nonce]; !found { + break + } + // Collision - generate again - should be extremely rare + nonce = p.generateNonce() + } + + var payload *pendingPayload + if payload, err = newPendingPayload(events, nonce, p.config.Timeout); err != nil { + return + } + + // Save pending payload until we receive ack, and discard buffer + p.pending_payloads[nonce] = payload + if p.first_payload == nil { + p.first_payload = payload + } else { + p.last_payload.next = payload + } + p.last_payload = payload + + p.Lock() + p.num_payloads++ + p.Unlock() + + return p.transport.Write("JDAT", payload.payload) } func (p *Publisher) processPong(message []byte) error { - if len(message) != 0 { - return fmt.Errorf("PONG message overflow (%d)", len(message)) - } + if len(message) != 0 { + return fmt.Errorf("PONG message overflow (%d)", len(message)) + } - // Were we pending a ping? - if !p.pending_ping { - return errors.New("Unexpected PONG received") - } + // Were we pending a ping? + if !p.pending_ping { + return errors.New("Unexpected PONG received") + } - log.Debug("PONG message received") + log.Debug("PONG message received") - p.pending_ping = false - return nil + p.pending_ping = false + return nil } func (p *Publisher) processAck(message []byte) (err error) { - if len(message) != 20 { - err = fmt.Errorf("ACKN message corruption (%d)", len(message)) - return - } - - // Read the nonce and sequence number acked - nonce, sequence := string(message[:16]), binary.BigEndian.Uint32(message[16:20]) - - log.Debug("ACKN message received for payload %x sequence %d", nonce, sequence) - - // Grab the payload the ACK corresponds to by using nonce - payload, found := p.pending_payloads[nonce] - if !found { - // Don't fail here in case we had temporary issues and resend a payload, only for us to receive duplicate ACKN - return - } - - ack_events := payload.ack_events - - // Process ACK - lines, complete := payload.Ack(int(sequence)) - p.line_count += int64(lines) - - if complete { - // No more events left for this payload, remove from pending list - delete(p.pending_payloads, nonce) - } - - // We potentially receive out-of-order ACKs due to payloads distributed across servers - // This is where we enforce ordering again to ensure registrar receives ACK in order - if payload == p.first_payload { - // The out of sync count we have will never include the first payload, so - // take the value +1 - out_of_sync := p.out_of_sync + 1 - - // For each full payload we mark off, we decrease this count, the first we - // mark off will always be the first payload - thus the +1. Subsequent - // payloads are the out of sync ones - so if we mark them off we decrease - // the out of sync count - for payload.HasAck() { - p.registrar_spool.Add(registrar.NewAckEvent(payload.Rollup())) - - if !payload.Complete() { - break - } - - payload = payload.next - p.first_payload = payload - out_of_sync-- - p.out_of_sync = out_of_sync - - p.Lock() - p.num_payloads-- - p.Unlock() - - // Resume sending if we stopped due to excessive pending payload count - if !p.shutdown && p.can_send == nil { - p.can_send = p.transport.CanSend() - } - - if payload == nil { - break - } - } - - p.registrar_spool.Send() - } else if ack_events == 0 { - // If this is NOT the first payload, and this is the first acknowledgement - // for this payload, then increase out of sync payload count - p.out_of_sync++ - } - - return + if len(message) != 20 { + err = fmt.Errorf("ACKN message corruption (%d)", len(message)) + return + } + + // Read the nonce and sequence number acked + nonce, sequence := string(message[:16]), binary.BigEndian.Uint32(message[16:20]) + + log.Debug("ACKN message received for payload %x sequence %d", nonce, sequence) + + // Grab the payload the ACK corresponds to by using nonce + payload, found := p.pending_payloads[nonce] + if !found { + // Don't fail here in case we had temporary issues and resend a payload, only for us to receive duplicate ACKN + return + } + + ack_events := payload.ack_events + + // Process ACK + lines, complete := payload.Ack(int(sequence)) + p.line_count += int64(lines) + + if complete { + // No more events left for this payload, remove from pending list + delete(p.pending_payloads, nonce) + } + + // We potentially receive out-of-order ACKs due to payloads distributed across servers + // This is where we enforce ordering again to ensure registrar receives ACK in order + if payload == p.first_payload { + // The out of sync count we have will never include the first payload, so + // take the value +1 + out_of_sync := p.out_of_sync + 1 + + // For each full payload we mark off, we decrease this count, the first we + // mark off will always be the first payload - thus the +1. Subsequent + // payloads are the out of sync ones - so if we mark them off we decrease + // the out of sync count + for payload.HasAck() { + p.registrar_spool.Add(registrar.NewAckEvent(payload.Rollup())) + + if !payload.Complete() { + break + } + + payload = payload.next + p.first_payload = payload + out_of_sync-- + p.out_of_sync = out_of_sync + + p.Lock() + p.num_payloads-- + p.Unlock() + + // Resume sending if we stopped due to excessive pending payload count + if !p.shutdown && p.can_send == nil { + p.can_send = p.transport.CanSend() + } + + if payload == nil { + break + } + } + + p.registrar_spool.Send() + } else if ack_events == 0 { + // If this is NOT the first payload, and this is the first acknowledgement + // for this payload, then increase out of sync payload count + p.out_of_sync++ + } + + return } func (p *Publisher) Snapshot() []*core.Snapshot { - p.RLock() + p.RLock() - snapshot := core.NewSnapshot("Publisher") + snapshot := core.NewSnapshot("Publisher") - switch p.status { - case Status_Connected: - snapshot.AddEntry("Status", "Connected") - case Status_Reconnecting: - snapshot.AddEntry("Status", "Reconnecting...") - default: - snapshot.AddEntry("Status", "Disconnected") - } + switch p.status { + case Status_Connected: + snapshot.AddEntry("Status", "Connected") + case Status_Reconnecting: + snapshot.AddEntry("Status", "Reconnecting...") + default: + snapshot.AddEntry("Status", "Disconnected") + } - snapshot.AddEntry("Speed (Lps)", p.line_speed) - snapshot.AddEntry("Published lines", p.last_line_count) - snapshot.AddEntry("Pending Payloads", p.num_payloads) - snapshot.AddEntry("Timeouts", p.timeout_count) - snapshot.AddEntry("Retransmissions", p.last_retry_count) + snapshot.AddEntry("Speed (Lps)", p.line_speed) + snapshot.AddEntry("Published lines", p.last_line_count) + snapshot.AddEntry("Pending Payloads", p.num_payloads) + snapshot.AddEntry("Timeouts", p.timeout_count) + snapshot.AddEntry("Retransmissions", p.last_retry_count) - p.RUnlock() + p.RUnlock() - return []*core.Snapshot{snapshot} + return []*core.Snapshot{snapshot} } diff --git a/src/lc-lib/registrar/event_ack.go b/src/lc-lib/registrar/event_ack.go index 11d2169f..70b0c546 100644 --- a/src/lc-lib/registrar/event_ack.go +++ b/src/lc-lib/registrar/event_ack.go @@ -17,33 +17,33 @@ package registrar import ( - "lc-lib/core" + "github.com/driskell/log-courier/src/lc-lib/core" ) type AckEvent struct { - events []*core.EventDescriptor + events []*core.EventDescriptor } func NewAckEvent(events []*core.EventDescriptor) *AckEvent { - return &AckEvent{ - events: events, - } + return &AckEvent{ + events: events, + } } func (e *AckEvent) Process(state map[core.Stream]*FileState) { - if len(e.events) == 1 { - log.Debug("Registrar received offsets for %d log entries", len(e.events)) - } else { - log.Debug("Registrar received offsets for %d log entries", len(e.events)) - } + if len(e.events) == 1 { + log.Debug("Registrar received offsets for %d log entries", len(e.events)) + } else { + log.Debug("Registrar received offsets for %d log entries", len(e.events)) + } - for _, event := range e.events { - _, is_found := state[event.Stream] - if !is_found { - // This is probably stdin then or a deleted file we can't resume - continue - } + for _, event := range e.events { + _, is_found := state[event.Stream] + if !is_found { + // This is probably stdin then or a deleted file we can't resume + continue + } - state[event.Stream].Offset = event.Offset - } + state[event.Stream].Offset = event.Offset + } } diff --git a/src/lc-lib/registrar/event_deleted.go b/src/lc-lib/registrar/event_deleted.go index a713fce1..473556d8 100644 --- a/src/lc-lib/registrar/event_deleted.go +++ b/src/lc-lib/registrar/event_deleted.go @@ -17,27 +17,27 @@ package registrar import ( - "lc-lib/core" + "github.com/driskell/log-courier/src/lc-lib/core" ) type DeletedEvent struct { - stream core.Stream + stream core.Stream } func NewDeletedEvent(stream core.Stream) *DeletedEvent { - return &DeletedEvent{ - stream: stream, - } + return &DeletedEvent{ + stream: stream, + } } func (e *DeletedEvent) Process(state map[core.Stream]*FileState) { - if _, ok := state[e.stream]; ok { - log.Debug("Registrar received a deletion event for %s", *state[e.stream].Source) - } else { - log.Warning("Registrar received a deletion event for UNKNOWN (%p)", e.stream) - } + if _, ok := state[e.stream]; ok { + log.Debug("Registrar received a deletion event for %s", *state[e.stream].Source) + } else { + log.Warning("Registrar received a deletion event for UNKNOWN (%p)", e.stream) + } - // Purge the registrar entry - means the file is deleted so we can't resume - // This keeps the state clean so it doesn't build up after thousands of log files - delete(state, e.stream) + // Purge the registrar entry - means the file is deleted so we can't resume + // This keeps the state clean so it doesn't build up after thousands of log files + delete(state, e.stream) } diff --git a/src/lc-lib/registrar/event_discover.go b/src/lc-lib/registrar/event_discover.go index c0ff8b0c..2bac27b2 100644 --- a/src/lc-lib/registrar/event_discover.go +++ b/src/lc-lib/registrar/event_discover.go @@ -17,33 +17,33 @@ package registrar import ( - "lc-lib/core" - "os" + "github.com/driskell/log-courier/src/lc-lib/core" + "os" ) type DiscoverEvent struct { - stream core.Stream - source string - offset int64 - fileinfo os.FileInfo + stream core.Stream + source string + offset int64 + fileinfo os.FileInfo } func NewDiscoverEvent(stream core.Stream, source string, offset int64, fileinfo os.FileInfo) *DiscoverEvent { - return &DiscoverEvent{ - stream: stream, - source: source, - offset: offset, - fileinfo: fileinfo, - } + return &DiscoverEvent{ + stream: stream, + source: source, + offset: offset, + fileinfo: fileinfo, + } } func (e *DiscoverEvent) Process(state map[core.Stream]*FileState) { - log.Debug("Registrar received a new file event for %s", e.source) + log.Debug("Registrar received a new file event for %s", e.source) - // A new file we need to save offset information for so we can resume - state[e.stream] = &FileState{ - Source: &e.source, - Offset: e.offset, - } - state[e.stream].PopulateFileIds(e.fileinfo) + // A new file we need to save offset information for so we can resume + state[e.stream] = &FileState{ + Source: &e.source, + Offset: e.offset, + } + state[e.stream].PopulateFileIds(e.fileinfo) } diff --git a/src/lc-lib/registrar/event_renamed.go b/src/lc-lib/registrar/event_renamed.go index 0ab29327..6c55a7e7 100644 --- a/src/lc-lib/registrar/event_renamed.go +++ b/src/lc-lib/registrar/event_renamed.go @@ -17,30 +17,30 @@ package registrar import ( - "lc-lib/core" + "github.com/driskell/log-courier/src/lc-lib/core" ) type RenamedEvent struct { - stream core.Stream - source string + stream core.Stream + source string } func NewRenamedEvent(stream core.Stream, source string) *RenamedEvent { - return &RenamedEvent{ - stream: stream, - source: source, - } + return &RenamedEvent{ + stream: stream, + source: source, + } } func (e *RenamedEvent) Process(state map[core.Stream]*FileState) { - _, is_found := state[e.stream] - if !is_found { - // This is probably stdin or a deleted file we can't resume - return - } + _, is_found := state[e.stream] + if !is_found { + // This is probably stdin or a deleted file we can't resume + return + } - log.Debug("Registrar received a rename event for %s -> %s", state[e.stream].Source, e.source) + log.Debug("Registrar received a rename event for %s -> %s", state[e.stream].Source, e.source) - // Update the stored file name - state[e.stream].Source = &e.source + // Update the stored file name + state[e.stream].Source = &e.source } diff --git a/src/lc-lib/registrar/eventspool.go b/src/lc-lib/registrar/eventspool.go index 0579c4c2..e432446f 100644 --- a/src/lc-lib/registrar/eventspool.go +++ b/src/lc-lib/registrar/eventspool.go @@ -17,41 +17,48 @@ package registrar import ( - "lc-lib/core" + "github.com/driskell/log-courier/src/lc-lib/core" ) -type RegistrarEvent interface { - Process(state map[core.Stream]*FileState) +type EventProcessor interface { + Process(state map[core.Stream]*FileState) } -type RegistrarEventSpool struct { - registrar *Registrar - events []RegistrarEvent +type EventSpooler interface { + Close() + Add(EventProcessor) + Send() } -func newRegistrarEventSpool(r *Registrar) *RegistrarEventSpool { - ret := &RegistrarEventSpool{ - registrar: r, - } - ret.reset() - return ret +type EventSpool struct { + registrar *Registrar + events []EventProcessor } -func (r *RegistrarEventSpool) Close() { - r.registrar.dereferenceSpooler() +func newEventSpool(r *Registrar) *EventSpool { + ret := &EventSpool{ + registrar: r, + } + ret.reset() + return ret } -func (r *RegistrarEventSpool) Add(event RegistrarEvent) { - r.events = append(r.events, event) +func (r *EventSpool) Close() { + r.registrar.dereferenceSpooler() + r.registrar = nil } -func (r *RegistrarEventSpool) Send() { - if len(r.events) != 0 { - r.registrar.registrar_chan <- r.events - r.reset() - } +func (r *EventSpool) Add(event EventProcessor) { + r.events = append(r.events, event) } -func (r *RegistrarEventSpool) reset() { - r.events = make([]RegistrarEvent, 0, 0) +func (r *EventSpool) Send() { + if len(r.events) != 0 { + r.registrar.registrar_chan <- r.events + r.reset() + } +} + +func (r *EventSpool) reset() { + r.events = make([]EventProcessor, 0, 0) } diff --git a/src/lc-lib/registrar/filestate.go b/src/lc-lib/registrar/filestate.go index b6e6b40c..2432d675 100644 --- a/src/lc-lib/registrar/filestate.go +++ b/src/lc-lib/registrar/filestate.go @@ -20,48 +20,48 @@ package registrar import ( - "os" + "os" ) type FileState struct { - FileStateOS - Source *string `json:"source,omitempty"` - Offset int64 `json:"offset,omitempty"` + FileStateOS + Source *string `json:"source,omitempty"` + Offset int64 `json:"offset,omitempty"` } type FileInfo struct { - fileinfo os.FileInfo + fileinfo os.FileInfo } func NewFileInfo(fileinfo os.FileInfo) *FileInfo { - return &FileInfo{ - fileinfo: fileinfo, - } + return &FileInfo{ + fileinfo: fileinfo, + } } func (fs *FileInfo) SameAs(info os.FileInfo) bool { - return os.SameFile(info, fs.fileinfo) + return os.SameFile(info, fs.fileinfo) } func (fs *FileInfo) Stat() os.FileInfo { - return fs.fileinfo + return fs.fileinfo } func (fs *FileInfo) Update(fileinfo os.FileInfo, identity *FileIdentity) { - fs.fileinfo = fileinfo + fs.fileinfo = fileinfo } func (fs *FileState) Stat() os.FileInfo { - return nil + return nil } func (fs *FileState) Update(fileinfo os.FileInfo, identity *FileIdentity) { - // Promote to a FileInfo - (*identity) = NewFileInfo(fileinfo) + // Promote to a FileInfo + (*identity) = NewFileInfo(fileinfo) } type FileIdentity interface { - SameAs(os.FileInfo) bool - Stat() os.FileInfo - Update(os.FileInfo, *FileIdentity) + SameAs(os.FileInfo) bool + Stat() os.FileInfo + Update(os.FileInfo, *FileIdentity) } diff --git a/src/lc-lib/registrar/filestateos_darwin.go b/src/lc-lib/registrar/filestateos_darwin.go index f35c9bd8..7f316747 100644 --- a/src/lc-lib/registrar/filestateos_darwin.go +++ b/src/lc-lib/registrar/filestateos_darwin.go @@ -20,23 +20,23 @@ package registrar import ( - "os" - "syscall" + "os" + "syscall" ) type FileStateOS struct { - Inode uint64 `json:"inode,omitempty"` - Device int32 `json:"device,omitempty"` + Inode uint64 `json:"inode,omitempty"` + Device int32 `json:"device,omitempty"` } func (fs *FileStateOS) PopulateFileIds(info os.FileInfo) { - fstat := info.Sys().(*syscall.Stat_t) - fs.Inode = fstat.Ino - fs.Device = fstat.Dev + fstat := info.Sys().(*syscall.Stat_t) + fs.Inode = fstat.Ino + fs.Device = fstat.Dev } func (fs *FileStateOS) SameAs(info os.FileInfo) bool { - state := &FileStateOS{} - state.PopulateFileIds(info) - return (fs.Inode == state.Inode && fs.Device == state.Device) + state := &FileStateOS{} + state.PopulateFileIds(info) + return (fs.Inode == state.Inode && fs.Device == state.Device) } diff --git a/src/lc-lib/registrar/filestateos_freebsd.go b/src/lc-lib/registrar/filestateos_freebsd.go index 8ff3f614..666fe29a 100644 --- a/src/lc-lib/registrar/filestateos_freebsd.go +++ b/src/lc-lib/registrar/filestateos_freebsd.go @@ -20,23 +20,23 @@ package registrar import ( - "os" - "syscall" + "os" + "syscall" ) type FileStateOS struct { - Inode uint32 `json:"inode,omitempty"` - Device uint32 `json:"device,omitempty"` + Inode uint32 `json:"inode,omitempty"` + Device uint32 `json:"device,omitempty"` } func (fs *FileStateOS) PopulateFileIds(info os.FileInfo) { - fstat := info.Sys().(*syscall.Stat_t) - fs.Inode = fstat.Ino - fs.Device = fstat.Dev + fstat := info.Sys().(*syscall.Stat_t) + fs.Inode = fstat.Ino + fs.Device = fstat.Dev } func (fs *FileStateOS) SameAs(info os.FileInfo) bool { - state := &FileStateOS{} - state.PopulateFileIds(info) - return (fs.Inode == state.Inode && fs.Device == state.Device) + state := &FileStateOS{} + state.PopulateFileIds(info) + return (fs.Inode == state.Inode && fs.Device == state.Device) } diff --git a/src/lc-lib/registrar/filestateos_linux.go b/src/lc-lib/registrar/filestateos_linux.go index 9f030c29..26bc36a2 100644 --- a/src/lc-lib/registrar/filestateos_linux.go +++ b/src/lc-lib/registrar/filestateos_linux.go @@ -20,23 +20,23 @@ package registrar import ( - "os" - "syscall" + "os" + "syscall" ) type FileStateOS struct { - Inode uint64 `json:"inode,omitempty"` - Device uint64 `json:"device,omitempty"` + Inode uint64 `json:"inode,omitempty"` + Device uint64 `json:"device,omitempty"` } func (fs *FileStateOS) PopulateFileIds(info os.FileInfo) { - fstat := info.Sys().(*syscall.Stat_t) - fs.Inode = fstat.Ino - fs.Device = fstat.Dev + fstat := info.Sys().(*syscall.Stat_t) + fs.Inode = fstat.Ino + fs.Device = fstat.Dev } func (fs *FileStateOS) SameAs(info os.FileInfo) bool { - state := &FileStateOS{} - state.PopulateFileIds(info) - return (fs.Inode == state.Inode && fs.Device == state.Device) + state := &FileStateOS{} + state.PopulateFileIds(info) + return (fs.Inode == state.Inode && fs.Device == state.Device) } diff --git a/src/lc-lib/registrar/filestateos_openbsd.go b/src/lc-lib/registrar/filestateos_openbsd.go index f35c9bd8..7f316747 100644 --- a/src/lc-lib/registrar/filestateos_openbsd.go +++ b/src/lc-lib/registrar/filestateos_openbsd.go @@ -20,23 +20,23 @@ package registrar import ( - "os" - "syscall" + "os" + "syscall" ) type FileStateOS struct { - Inode uint64 `json:"inode,omitempty"` - Device int32 `json:"device,omitempty"` + Inode uint64 `json:"inode,omitempty"` + Device int32 `json:"device,omitempty"` } func (fs *FileStateOS) PopulateFileIds(info os.FileInfo) { - fstat := info.Sys().(*syscall.Stat_t) - fs.Inode = fstat.Ino - fs.Device = fstat.Dev + fstat := info.Sys().(*syscall.Stat_t) + fs.Inode = fstat.Ino + fs.Device = fstat.Dev } func (fs *FileStateOS) SameAs(info os.FileInfo) bool { - state := &FileStateOS{} - state.PopulateFileIds(info) - return (fs.Inode == state.Inode && fs.Device == state.Device) + state := &FileStateOS{} + state.PopulateFileIds(info) + return (fs.Inode == state.Inode && fs.Device == state.Device) } diff --git a/src/lc-lib/registrar/filestateos_windows.go b/src/lc-lib/registrar/filestateos_windows.go index 5c15ebcb..9be98d73 100644 --- a/src/lc-lib/registrar/filestateos_windows.go +++ b/src/lc-lib/registrar/filestateos_windows.go @@ -20,55 +20,55 @@ package registrar import ( - "os" - "reflect" + "os" + "reflect" ) type FileStateOS struct { - Vol uint32 `json:"vol,omitempty"` - IdxHi uint32 `json:"idxhi,omitempty"` - IdxLo uint32 `json:"idxlo,omitempty"` + Vol uint32 `json:"vol,omitempty"` + IdxHi uint32 `json:"idxhi,omitempty"` + IdxLo uint32 `json:"idxlo,omitempty"` } func (fs *FileStateOS) PopulateFileIds(info os.FileInfo) { - // For information on the following, see Go source: src/pkg/os/types_windows.go - // This is the only way we can get at the idxhi and idxlo - // Unix it is much easier as syscall.Stat_t is exposed and os.FileInfo interface has a Sys() method to get a syscall.Stat_t - // Unfortunately, the relevant Windows information is in a private struct so we have to dig inside + // For information on the following, see Go source: src/pkg/os/types_windows.go + // This is the only way we can get at the idxhi and idxlo + // Unix it is much easier as syscall.Stat_t is exposed and os.FileInfo interface has a Sys() method to get a syscall.Stat_t + // Unfortunately, the relevant Windows information is in a private struct so we have to dig inside - // NOTE: This WILL be prone to break if Go source changes, but I'd rather just fix it if it does or make it fail gracefully + // NOTE: This WILL be prone to break if Go source changes, but I'd rather just fix it if it does or make it fail gracefully - // info is os.FileInfo which is an interface to a - // - *os.fileStat (holding methods) which is a pointer to a - // - os.fileStat (holding data) - // ValueOf will pick up the interface contents immediately, so we need a single Elem() + // info is os.FileInfo which is an interface to a + // - *os.fileStat (holding methods) which is a pointer to a + // - os.fileStat (holding data) + // ValueOf will pick up the interface contents immediately, so we need a single Elem() - // Ensure that the numbers are loaded by calling os.SameFile - // os.SameFile will call sameFile (types_windows.go) which will call *os.fileStat's loadFileId - // Reflection panics if we try to call loadFileId directly as its a hidden method; regardless this is much safer and more reliable - os.SameFile(info, info) + // Ensure that the numbers are loaded by calling os.SameFile + // os.SameFile will call sameFile (types_windows.go) which will call *os.fileStat's loadFileId + // Reflection panics if we try to call loadFileId directly as its a hidden method; regardless this is much safer and more reliable + os.SameFile(info, info) - // If any of the following fails, report the library has changed and recover and return 0s - defer func() { - if r := recover(); r != nil { - log.Error("BUG: File rotations that occur while Log Courier is not running will NOT be detected due to an incompatible change to the Go library used for compiling.") - fs.Vol = 0 - fs.IdxHi = 0 - fs.IdxLo = 0 - } - }() + // If any of the following fails, report the library has changed and recover and return 0s + defer func() { + if r := recover(); r != nil { + log.Error("BUG: File rotations that occur while Log Courier is not running will NOT be detected due to an incompatible change to the Go library used for compiling.") + fs.Vol = 0 + fs.IdxHi = 0 + fs.IdxLo = 0 + } + }() - // Following makes fstat hold os.fileStat - fstat := reflect.ValueOf(info).Elem() + // Following makes fstat hold os.fileStat + fstat := reflect.ValueOf(info).Elem() - // To get the data, we need the os.fileStat that fstat points to, so one more Elem() - fs.Vol = uint32(fstat.FieldByName("vol").Uint()) - fs.IdxHi = uint32(fstat.FieldByName("idxhi").Uint()) - fs.IdxLo = uint32(fstat.FieldByName("idxlo").Uint()) + // To get the data, we need the os.fileStat that fstat points to, so one more Elem() + fs.Vol = uint32(fstat.FieldByName("vol").Uint()) + fs.IdxHi = uint32(fstat.FieldByName("idxhi").Uint()) + fs.IdxLo = uint32(fstat.FieldByName("idxlo").Uint()) } func (fs *FileStateOS) SameAs(info os.FileInfo) bool { - state := &FileStateOS{} - state.PopulateFileIds(info) - return (fs.Vol == state.Vol && fs.IdxHi == state.IdxHi && fs.IdxLo == state.IdxLo) + state := &FileStateOS{} + state.PopulateFileIds(info) + return (fs.Vol == state.Vol && fs.IdxHi == state.IdxHi && fs.IdxLo == state.IdxLo) } diff --git a/src/lc-lib/registrar/logging.go b/src/lc-lib/registrar/logging.go index 5bfa8f79..eabf045d 100644 --- a/src/lc-lib/registrar/logging.go +++ b/src/lc-lib/registrar/logging.go @@ -21,5 +21,5 @@ import "github.com/op/go-logging" var log *logging.Logger func init() { - log = logging.MustGetLogger("registrar") + log = logging.MustGetLogger("registrar") } diff --git a/src/lc-lib/registrar/registrar.go b/src/lc-lib/registrar/registrar.go index 6dd21537..f14570e6 100644 --- a/src/lc-lib/registrar/registrar.go +++ b/src/lc-lib/registrar/registrar.go @@ -20,148 +20,152 @@ package registrar import ( - "encoding/json" - "fmt" - "lc-lib/core" - "os" - "sync" + "encoding/json" + "fmt" + "github.com/driskell/log-courier/src/lc-lib/core" + "os" + "sync" ) type LoadPreviousFunc func(string, *FileState) (core.Stream, error) +type Registrator interface { + Connect() EventSpooler + LoadPrevious(LoadPreviousFunc) (bool, error) +} + type Registrar struct { - core.PipelineSegment + core.PipelineSegment - sync.Mutex + sync.Mutex - registrar_chan chan []RegistrarEvent - references int - persistdir string - statefile string - state map[core.Stream]*FileState + registrar_chan chan []EventProcessor + references int + persistdir string + statefile string + state map[core.Stream]*FileState } func NewRegistrar(pipeline *core.Pipeline, persistdir string) *Registrar { - ret := &Registrar{ - registrar_chan: make(chan []RegistrarEvent, 16), // TODO: Make configurable? - persistdir: persistdir, - statefile: ".log-courier", - state: make(map[core.Stream]*FileState), - } + ret := &Registrar{ + registrar_chan: make(chan []EventProcessor, 16), // TODO: Make configurable? + persistdir: persistdir, + statefile: ".log-courier", + state: make(map[core.Stream]*FileState), + } - pipeline.Register(ret) + pipeline.Register(ret) - return ret + return ret } func (r *Registrar) LoadPrevious(callback_func LoadPreviousFunc) (have_previous bool, err error) { - data := make(map[string]*FileState) - - // Load the previous state - opening RDWR ensures we can write too and fail early - // c_filename is what we will use to test create capability - filename := r.persistdir + string(os.PathSeparator) + ".log-courier" - c_filename := r.persistdir + string(os.PathSeparator) + ".log-courier.new" - - var f *os.File - f, err = os.OpenFile(filename, os.O_RDWR, 0600) - if err != nil { - // Fail immediately if this is not a path not found error - if !os.IsNotExist(err) { - return - } - - // Try the .new file - maybe we failed mid-move - filename, c_filename = c_filename, filename - f, err = os.OpenFile(filename, os.O_RDWR, 0600) - } - - if err != nil { - // Did we fail, or did it just not exist? - if !os.IsNotExist(err) { - return - } - return false, nil - } - - // Parse the data - log.Notice("Loading registrar data from %s", filename) - have_previous = true - - decoder := json.NewDecoder(f) - decoder.Decode(&data) - f.Close() - - r.state = make(map[core.Stream]*FileState, len(data)) - - var stream core.Stream - for file, state := range data { - if stream, err = callback_func(file, state); err != nil { - return - } - r.state[stream] = state - } - - // Test we can successfully save new states by attempting to save now - if err = r.writeRegistry(); err != nil { - return false, fmt.Errorf("Registry write failed: %s", err) - } - - return + data := make(map[string]*FileState) + + // Load the previous state - opening RDWR ensures we can write too and fail early + // c_filename is what we will use to test create capability + filename := r.persistdir + string(os.PathSeparator) + ".log-courier" + c_filename := r.persistdir + string(os.PathSeparator) + ".log-courier.new" + + var f *os.File + f, err = os.OpenFile(filename, os.O_RDWR, 0600) + if err != nil { + // Fail immediately if this is not a path not found error + if !os.IsNotExist(err) { + return + } + + // Try the .new file - maybe we failed mid-move + filename, c_filename = c_filename, filename + f, err = os.OpenFile(filename, os.O_RDWR, 0600) + } + + if err != nil { + // Did we fail, or did it just not exist? + if !os.IsNotExist(err) { + return + } + return false, nil + } + + // Parse the data + log.Notice("Loading registrar data from %s", filename) + have_previous = true + + decoder := json.NewDecoder(f) + decoder.Decode(&data) + f.Close() + + r.state = make(map[core.Stream]*FileState, len(data)) + + var stream core.Stream + for file, state := range data { + if stream, err = callback_func(file, state); err != nil { + return + } + r.state[stream] = state + } + + // Test we can successfully save new states by attempting to save now + if err = r.writeRegistry(); err != nil { + return false, fmt.Errorf("Registry write failed: %s", err) + } + + return } -func (r *Registrar) Connect() *RegistrarEventSpool { - r.Lock() - ret := newRegistrarEventSpool(r) - r.references++ - r.Unlock() - return ret +func (r *Registrar) Connect() EventSpooler { + r.Lock() + defer r.Unlock() + r.references++ + return newEventSpool(r) } func (r *Registrar) dereferenceSpooler() { - r.Lock() - r.references-- - if r.references == 0 { - // Shutdown registrar, all references are closed - close(r.registrar_chan) - } - r.Unlock() + r.Lock() + defer r.Unlock() + r.references-- + if r.references == 0 { + // Shutdown registrar, all references are closed + close(r.registrar_chan) + } } func (r *Registrar) toCanonical() (canonical map[string]*FileState) { - canonical = make(map[string]*FileState, len(r.state)) - for _, value := range r.state { - if _, ok := canonical[*value.Source]; ok { - // We should never allow this - report an error - log.Error("BUG: Unexpected registrar conflict detected for %s", *value.Source) - } - canonical[*value.Source] = value - } - return + canonical = make(map[string]*FileState, len(r.state)) + for _, value := range r.state { + if _, ok := canonical[*value.Source]; ok { + // We should never allow this - report an error + log.Error("BUG: Unexpected registrar conflict detected for %s", *value.Source) + } + canonical[*value.Source] = value + } + return } func (r *Registrar) Run() { - defer func() { - r.Done() - }() + defer func() { + r.Done() + }() RegistrarLoop: - for { - // Ignore shutdown channel - wait for registrar to close - select { - case spool := <-r.registrar_chan: - if spool == nil { - break RegistrarLoop - } - - for _, event := range spool { - event.Process(r.state) - } - - if err := r.writeRegistry(); err != nil { - log.Error("Registry write failed: %s", err) - } - } - } - - log.Info("Registrar exiting") + for { + // Ignore shutdown channel - wait for registrar to close + select { + case spool := <-r.registrar_chan: + if spool == nil { + break RegistrarLoop + } + + for _, event := range spool { + event.Process(r.state) + } + + if err := r.writeRegistry(); err != nil { + log.Error("Registry write failed: %s", err) + } + } + } + + log.Info("Registrar exiting") } diff --git a/src/lc-lib/registrar/registrar_other.go b/src/lc-lib/registrar/registrar_other.go index 9bebe108..7cbaacc5 100644 --- a/src/lc-lib/registrar/registrar_other.go +++ b/src/lc-lib/registrar/registrar_other.go @@ -22,22 +22,22 @@ package registrar import ( - "encoding/json" - "os" + "encoding/json" + "os" ) func (r *Registrar) writeRegistry() error { - // Open tmp file, write, flush, rename - fname := r.persistdir + string(os.PathSeparator) + r.statefile - tname := fname + ".new" - file, err := os.Create(tname) - if err != nil { - return err - } - defer file.Close() + // Open tmp file, write, flush, rename + fname := r.persistdir + string(os.PathSeparator) + r.statefile + tname := fname + ".new" + file, err := os.Create(tname) + if err != nil { + return err + } + defer file.Close() - encoder := json.NewEncoder(file) - encoder.Encode(r.toCanonical()) + encoder := json.NewEncoder(file) + encoder.Encode(r.toCanonical()) - return os.Rename(tname, fname) + return os.Rename(tname, fname) } diff --git a/src/lc-lib/registrar/registrar_windows.go b/src/lc-lib/registrar/registrar_windows.go index fdcce814..da9ace19 100644 --- a/src/lc-lib/registrar/registrar_windows.go +++ b/src/lc-lib/registrar/registrar_windows.go @@ -20,32 +20,32 @@ package registrar import ( - "encoding/json" - "fmt" - "os" + "encoding/json" + "fmt" + "os" ) func (r *Registrar) writeRegistry() error { - fname := r.persistdir + string(os.PathSeparator) + r.statefile - tname := fname + ".new" - file, err := os.Create(tname) - if err != nil { - return err - } + fname := r.persistdir + string(os.PathSeparator) + r.statefile + tname := fname + ".new" + file, err := os.Create(tname) + if err != nil { + return err + } - encoder := json.NewEncoder(file) - encoder.Encode(r.toCanonical()) - file.Close() + encoder := json.NewEncoder(file) + encoder.Encode(r.toCanonical()) + file.Close() - var d_err error - if _, err = os.Stat(fname); err == nil || !os.IsNotExist(err) { - d_err = os.Remove(fname) - } + var d_err error + if _, err = os.Stat(fname); err == nil || !os.IsNotExist(err) { + d_err = os.Remove(fname) + } - err = os.Rename(tname, fname) - if err != nil { - return fmt.Errorf("%s -> %s", d_err, err) - } + err = os.Rename(tname, fname) + if err != nil { + return fmt.Errorf("%s -> %s", d_err, err) + } - return nil + return nil } diff --git a/src/lc-lib/spooler/logging.go b/src/lc-lib/spooler/logging.go index 44667972..a86c3077 100644 --- a/src/lc-lib/spooler/logging.go +++ b/src/lc-lib/spooler/logging.go @@ -21,5 +21,5 @@ import "github.com/op/go-logging" var log *logging.Logger func init() { - log = logging.MustGetLogger("spooler") + log = logging.MustGetLogger("spooler") } diff --git a/src/lc-lib/spooler/spooler.go b/src/lc-lib/spooler/spooler.go index 709520d4..5adc8b86 100644 --- a/src/lc-lib/spooler/spooler.go +++ b/src/lc-lib/spooler/spooler.go @@ -20,152 +20,169 @@ package spooler import ( - "lc-lib/core" - "lc-lib/publisher" - "time" + "github.com/driskell/log-courier/src/lc-lib/core" + "github.com/driskell/log-courier/src/lc-lib/publisher" + "time" ) const ( - // Event header is just uint32 at the moment - event_header_size = 4 + // Event header is just uint32 at the moment + event_header_size = 4 ) type Spooler struct { - core.PipelineSegment - core.PipelineConfigReceiver - - config *core.GeneralConfig - spool []*core.EventDescriptor - spool_size int - input chan *core.EventDescriptor - output chan<- []*core.EventDescriptor - timer_start time.Time - timer *time.Timer + core.PipelineSegment + core.PipelineConfigReceiver + + config *core.GeneralConfig + spool []*core.EventDescriptor + spool_size int + input chan *core.EventDescriptor + output chan<- []*core.EventDescriptor + timer_start time.Time + timer *time.Timer } func NewSpooler(pipeline *core.Pipeline, config *core.GeneralConfig, publisher_imp *publisher.Publisher) *Spooler { - ret := &Spooler{ - config: config, - spool: make([]*core.EventDescriptor, 0, config.SpoolSize), - input: make(chan *core.EventDescriptor, 16), // TODO: Make configurable? - output: publisher_imp.Connect(), - } + ret := &Spooler{ + config: config, + spool: make([]*core.EventDescriptor, 0, config.SpoolSize), + input: make(chan *core.EventDescriptor, 16), // TODO: Make configurable? + output: publisher_imp.Connect(), + } - pipeline.Register(ret) + pipeline.Register(ret) - return ret + return ret } func (s *Spooler) Connect() chan<- *core.EventDescriptor { - return s.input + return s.input +} + +func (s *Spooler) Flush() { + s.input <- nil } func (s *Spooler) Run() { - defer func() { - s.Done() - }() + defer func() { + s.Done() + }() - s.timer_start = time.Now() - s.timer = time.NewTimer(s.config.SpoolTimeout) + s.timer_start = time.Now() + s.timer = time.NewTimer(s.config.SpoolTimeout) SpoolerLoop: - for { - select { - case event := <-s.input: - if len(s.spool) > 0 && int64(s.spool_size) + int64(len(event.Event)) + event_header_size >= s.config.SpoolMaxBytes { - log.Debug("Spooler flushing %d events due to spool max bytes (%d/%d - next is %d)", len(s.spool), s.spool_size, s.config.SpoolMaxBytes, len(event.Event) + 4) - - // Can't fit this event in the spool - flush and then queue - if !s.sendSpool() { - break SpoolerLoop - } - - s.resetTimer() - s.spool_size += len(event.Event) + event_header_size - s.spool = append(s.spool, event) - - continue - } - - s.spool_size += len(event.Event) + event_header_size - s.spool = append(s.spool, event) - - // Flush if full - if len(s.spool) >= cap(s.spool) { - log.Debug("Spooler flushing %d events due to spool size reached", len(s.spool)) - - if !s.sendSpool() { - break SpoolerLoop - } - - s.resetTimer() - } - case <-s.timer.C: - // Flush what we have, if anything - if len(s.spool) > 0 { - log.Debug("Spooler flushing %d events due to spool timeout exceeded", len(s.spool)) - - if !s.sendSpool() { - break SpoolerLoop - } - } - - s.resetTimer() - case <-s.OnShutdown(): - break SpoolerLoop - case config := <-s.OnConfig(): - if !s.reloadConfig(config) { - break SpoolerLoop - } - } - } - - log.Info("Spooler exiting") + for { + select { + case event := <-s.input: + // Nil event means flush + if event == nil { + if len(s.spool) > 0 { + log.Debug("Spooler flushing %d events due to flush event", len(s.spool)) + + if !s.sendSpool() { + break SpoolerLoop + } + } + + continue + } + + if len(s.spool) > 0 && int64(s.spool_size)+int64(len(event.Event))+event_header_size >= s.config.SpoolMaxBytes { + log.Debug("Spooler flushing %d events due to spool max bytes (%d/%d - next is %d)", len(s.spool), s.spool_size, s.config.SpoolMaxBytes, len(event.Event)+4) + + // Can't fit this event in the spool - flush and then queue + if !s.sendSpool() { + break SpoolerLoop + } + + s.resetTimer() + s.spool_size += len(event.Event) + event_header_size + s.spool = append(s.spool, event) + + continue + } + + s.spool_size += len(event.Event) + event_header_size + s.spool = append(s.spool, event) + + // Flush if full + if len(s.spool) >= cap(s.spool) { + log.Debug("Spooler flushing %d events due to spool size reached", len(s.spool)) + + if !s.sendSpool() { + break SpoolerLoop + } + + s.resetTimer() + } + case <-s.timer.C: + // Flush what we have, if anything + if len(s.spool) > 0 { + log.Debug("Spooler flushing %d events due to spool timeout exceeded", len(s.spool)) + + if !s.sendSpool() { + break SpoolerLoop + } + } + + s.resetTimer() + case <-s.OnShutdown(): + break SpoolerLoop + case config := <-s.OnConfig(): + if !s.reloadConfig(config) { + break SpoolerLoop + } + } + } + + log.Info("Spooler exiting") } func (s *Spooler) sendSpool() bool { - select { - case <-s.OnShutdown(): - return false - case config := <-s.OnConfig(): - if !s.reloadConfig(config) { - return false - } - case s.output <- s.spool: - } - - s.spool = make([]*core.EventDescriptor, 0, s.config.SpoolSize) - s.spool_size = 0 - - return true + select { + case <-s.OnShutdown(): + return false + case config := <-s.OnConfig(): + if !s.reloadConfig(config) { + return false + } + case s.output <- s.spool: + } + + s.spool = make([]*core.EventDescriptor, 0, s.config.SpoolSize) + s.spool_size = 0 + + return true } func (s *Spooler) resetTimer() { - s.timer_start = time.Now() - - // Stop the timer, and ensure the channel is empty before restarting it - s.timer.Stop() - select { - case <-s.timer.C: - default: - } - s.timer.Reset(s.config.SpoolTimeout) + s.timer_start = time.Now() + + // Stop the timer, and ensure the channel is empty before restarting it + s.timer.Stop() + select { + case <-s.timer.C: + default: + } + s.timer.Reset(s.config.SpoolTimeout) } func (s *Spooler) reloadConfig(config *core.Config) bool { - s.config = &config.General - - // Immediate flush? - passed := time.Now().Sub(s.timer_start) - if passed >= s.config.SpoolTimeout || len(s.spool) >= int(s.config.SpoolSize) { - if !s.sendSpool() { - return false - } - s.timer_start = time.Now() - s.timer.Reset(s.config.SpoolTimeout) - } else { - s.timer.Reset(passed - s.config.SpoolTimeout) - } - - return true + s.config = &config.General + + // Immediate flush? + passed := time.Now().Sub(s.timer_start) + if passed >= s.config.SpoolTimeout || len(s.spool) >= int(s.config.SpoolSize) { + if !s.sendSpool() { + return false + } + s.timer_start = time.Now() + s.timer.Reset(s.config.SpoolTimeout) + } else { + s.timer.Reset(passed - s.config.SpoolTimeout) + } + + return true } diff --git a/src/lc-lib/transports/address_pool.go b/src/lc-lib/transports/address_pool.go new file mode 100644 index 00000000..719a5467 --- /dev/null +++ b/src/lc-lib/transports/address_pool.go @@ -0,0 +1,210 @@ +/* + * Copyright 2014 Jason Woods. + * + * This file is a modification of code from Logstash Forwarder. + * Copyright 2012-2013 Jordan Sissel and contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package transports + +import ( + "fmt" + "net" + "math/rand" + "strconv" + "time" +) + +type AddressPool struct { + servers []string + rfc2782 bool + rfc2782Service string + roundrobin int + host_is_ip bool + host string + addresses []*net.TCPAddr +} + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +func NewAddressPool(servers []string) *AddressPool { + ret := &AddressPool{ + servers: servers, + } + + // Randomise the initial host - after this it will round robin + // Round robin after initial attempt ensures we don't retry same host twice, + // and also ensures we try all hosts one by one + rnd := rand.Intn(len(servers)) + if rnd != 0 { + ret.servers = append(append(make([]string, 0), servers[rnd:]...), servers[:rnd]...) + } + + return ret +} + +func (p *AddressPool) SetRfc2782(enabled bool, service string) { + p.rfc2782 = enabled + p.rfc2782Service = service +} + +func (p *AddressPool) IsLast() bool { + return p.addresses == nil +} + +func (p *AddressPool) IsLastServer() bool { + return p.roundrobin%len(p.servers) == 0 +} + +func (p *AddressPool) Next() (*net.TCPAddr, string, error) { + // Have we exhausted the address list we had? + if p.addresses == nil { + p.addresses = make([]*net.TCPAddr, 0) + if err := p.populateAddresses(); err != nil { + p.addresses = nil + return nil, "", err + } + } + + next := p.addresses[0] + if len(p.addresses) > 1 { + p.addresses = p.addresses[1:] + } else { + p.addresses = nil + } + + var desc string + if p.host_is_ip { + desc = fmt.Sprintf("%s", next) + } else { + desc = fmt.Sprintf("%s (%s)", next, p.host) + } + + return next, desc, nil +} + +func (p *AddressPool) NextServer() (string, error) { + // Round robin to the next server + selected := p.servers[p.roundrobin%len(p.servers)] + p.roundrobin++ + + // @hostname means SRV record where the host and port are in the record + if len(selected) > 0 && selected[0] == '@' { + srvs, err := p.processSrv(selected[1:]) + if err != nil { + return "", err + } + return net.JoinHostPort(srvs[0].Target, strconv.FormatUint(uint64(srvs[0].Port), 10)), nil + } + + return selected, nil +} + +func (p *AddressPool) Host() string { + return p.host +} + +func (p *AddressPool) populateAddresses() (error) { + // Round robin to the next server + selected := p.servers[p.roundrobin%len(p.servers)] + p.roundrobin++ + + // @hostname means SRV record where the host and port are in the record + if len(selected) > 0 && selected[0] == '@' { + srvs, err := p.processSrv(selected[1:]) + if err != nil { + return err + } + + for _, srv := range srvs { + if _, err := p.populateLookup(srv.Target, int(srv.Port)); err != nil { + return err + } + } + + return nil + } + + // Standard host:port declaration + var port_str string + var port uint64 + var err error + if p.host, port_str, err = net.SplitHostPort(selected); err != nil { + return fmt.Errorf("Invalid hostport given: %s", selected) + } + + if port, err = strconv.ParseUint(port_str, 10, 16); err != nil { + return fmt.Errorf("Invalid port given: %s", port_str) + } + + if p.host_is_ip, err = p.populateLookup(p.host, int(port)); err != nil { + return err + } + + return nil +} + +func (p *AddressPool) processSrv(server string) ([]*net.SRV, error) { + var service, protocol string + + p.host = server + p.host_is_ip = false + + if p.rfc2782 { + service, protocol = p.rfc2782Service, "tcp" + } else { + service, protocol = "", "" + } + + _, srvs, err := net.LookupSRV(service, protocol, p.host) + if err != nil { + return nil, fmt.Errorf("DNS SRV lookup failure \"%s\": %s", p.host, err) + } else if len(srvs) == 0 { + return nil, fmt.Errorf("DNS SRV lookup failure \"%s\": No targets found", p.host) + } + + return srvs, nil +} + +func (p *AddressPool) populateLookup(host string, port int) (bool, error) { + if ip := net.ParseIP(host); ip != nil { + // IP address + p.addresses = append(p.addresses, &net.TCPAddr{ + IP: ip, + Port: port, + }) + + return true, nil + } + + // Lookup the hostname in DNS + ips, err := net.LookupIP(host) + if err != nil { + return false, fmt.Errorf("DNS lookup failure \"%s\": %s", host, err) + } else if len(ips) == 0 { + return false, fmt.Errorf("DNS lookup failure \"%s\": No addresses found", host) + } + + for _, ip := range ips { + p.addresses = append(p.addresses, &net.TCPAddr{ + IP: ip, + Port: port, + }) + } + + return false, nil +} diff --git a/src/lc-lib/transports/address_pool_test.go b/src/lc-lib/transports/address_pool_test.go new file mode 100644 index 00000000..5f8fcebc --- /dev/null +++ b/src/lc-lib/transports/address_pool_test.go @@ -0,0 +1,231 @@ +/* + * Copyright 2014 Jason Woods. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package transports + +import ( + "testing" +) + +func TestAddressPoolIP(t *testing.T) { + // Test failures when parsing + pool := NewAddressPool([]string{"127.0.0.1:1234"}) + + addr, desc, err := pool.Next() + + // Should have succeeeded + if err != nil { + t.Error("Address pool did not parse IP correctly: ", err) + } else if addr == nil { + t.Error("Address pool did not returned nil addr") + } else if desc != "127.0.0.1:1234" { + t.Error("Address pool did not return correct desc: ", desc) + } else if addr.String() != "127.0.0.1:1234" { + t.Error("Address pool did not return correct addr: ", addr.String()) + } +} + +func TestAddressPoolHost(t *testing.T) { + // Test failures when parsing + pool := NewAddressPool([]string{"google-public-dns-a.google.com:555"}) + + addr, desc, err := pool.Next() + + if err != nil { + t.Error("Address pool did not parse Host correctly: ", err) + } else if addr == nil { + t.Error("Address pool did not returned nil addr") + } else if desc != "8.8.8.8:555 (google-public-dns-a.google.com)" && desc != "[2001:4860:4860::8888]:555 (google-public-dns-a.google.com)" { + t.Error("Address pool did not return correct desc: ", desc) + } else if addr.String() != "8.8.8.8:555" && addr.String() != "[2001:4860:4860::8888]:555" { + t.Error("Address pool did not return correct addr: ", addr.String()) + } +} + +func TestAddressPoolHostMultiple(t *testing.T) { + // Test failures when parsing + pool := NewAddressPool([]string{"google.com:555"}) + + for i := 0; i < 2; i++ { + addr, _, err := pool.Next() + + // Should have succeeeded + if err != nil { + t.Error("Address pool did not parse Host correctly: ", err) + } else if addr == nil { + t.Error("Address pool did not returned nil addr") + } + + if i == 0 { + if pool.IsLast() { + t.Error("Address pool did not return multiple addresses") + } + } + } +} + +func TestAddressPoolSrv(t *testing.T) { + // Test failures when parsing + pool := NewAddressPool([]string{"@_xmpp-server._tcp.google.com"}) + + addr, _, err := pool.Next() + + // Should have succeeeded + if err != nil { + t.Error("Address pool did not parse SRV correctly: ", err) + } else if addr == nil { + t.Error("Address pool did not returned nil addr") + } +} + +func TestAddressPoolSrvRfc(t *testing.T) { + // Test failures when parsing + pool := NewAddressPool([]string{"@google.com"}) + pool.SetRfc2782(true, "xmpp-server") + + addr, _, err := pool.Next() + + // Should have succeeeded + if err != nil { + t.Error("Address pool did not parse RFC SRV correctly: ", err) + } else if addr == nil { + t.Error("Address pool did not returned nil addr") + } +} + +func TestAddressPoolInvalid(t *testing.T) { + // Test failures when parsing + pool := NewAddressPool([]string{"127.0..0:1234"}) + + _, _, err := pool.Next() + + // Should have failed + if err == nil { + t.Logf("Address pool did not return failure correctly") + t.FailNow() + } +} + +func TestAddressPoolHostFailure(t *testing.T) { + // Test failures when parsing + pool := NewAddressPool([]string{"google-public-dns-not-exist.google.com:1234"}) + + _, _, err := pool.Next() + + // Should have failed + if err == nil { + t.Logf("Address pool did not return failure correctly") + t.FailNow() + } +} + +func TestAddressPoolIsLast(t *testing.T) { + // Test that IsLastServer works correctly + pool := NewAddressPool([]string{"outlook.com:1234"}) + + // Should report as last + if !pool.IsLast() { + t.Error("Address pool IsLast did not return correctly") + } + + for i := 0; i <= 42; i++ { + _, _, err := pool.Next() + + // Should succeed + if err != nil { + t.Error("Address pool did not parse Host correctly") + } + + if i <= 1 { + // Should not report as last + if pool.IsLast() { + t.Error("Address pool IsLast did not return correctly") + } + + continue + } + + // Wait until last + if pool.IsLast() { + return + } + } + + // Hit 42 servers without hitting last + t.Error("Address pool IsLast did not return correctly") +} + +func TestAddressPoolIsLastServer(t *testing.T) { + // Test that IsLastServer works correctly + pool := NewAddressPool([]string{"127.0.0.1:1234", "127.0.0.1:1234", "127.0.0.1:1234"}) + + // Should report as last server + if !pool.IsLastServer() { + t.Error("Address pool IsLastServer did not return correctly") + } + + for i := 0; i < 3; i++ { + _, _, err := pool.Next() + + // Should succeed + if err != nil { + t.Error("Address pool did not parse IP correctly") + } + + if i < 2 { + // Should not report as last server + if pool.IsLastServer() { + t.Error("Address pool IsLastServer did not return correctly") + } + + continue + } + } + + // Should report as last server + if !pool.IsLastServer() { + t.Error("Address pool IsLastServer did not return correctly") + } +} + +func TestAddressPoolNextServer(t *testing.T) { + // Test that IsLastServer works correctly + pool := NewAddressPool([]string{"google.com:1234", "google.com:1234"}) + + cnt := 0 + for i := 0; i < 42; i++ { + addr, err := pool.NextServer() + + // Should succeed + if err != nil { + t.Error("Address pool did not parse IP correctly") + } else if addr != "google.com:1234" { + t.Error("Address pool returned incorrect address: ", addr) + } + + cnt++ + + // Break on last server + if pool.IsLastServer() { + break + } + } + + // Should have stopped at 2 servers + if cnt != 2 { + t.Error("Address pool NextServer failed") + } +} diff --git a/src/lc-lib/transports/logging.go b/src/lc-lib/transports/logging.go index d2ba8d90..b6c13486 100644 --- a/src/lc-lib/transports/logging.go +++ b/src/lc-lib/transports/logging.go @@ -21,5 +21,5 @@ import "github.com/op/go-logging" var log *logging.Logger func init() { - log = logging.MustGetLogger("transports") + log = logging.MustGetLogger("transports") } diff --git a/src/lc-lib/transports/tcp.go b/src/lc-lib/transports/tcp.go index fb8d9ec6..23639890 100644 --- a/src/lc-lib/transports/tcp.go +++ b/src/lc-lib/transports/tcp.go @@ -20,20 +20,18 @@ package transports import ( - "bytes" - "crypto/tls" - "crypto/x509" - "encoding/binary" - "encoding/pem" - "errors" - "fmt" - "io/ioutil" - "lc-lib/core" - "math/rand" - "net" - "regexp" - "sync" - "time" + "bytes" + "crypto/tls" + "crypto/x509" + "encoding/binary" + "encoding/pem" + "fmt" + "github.com/driskell/log-courier/src/lc-lib/core" + "io/ioutil" + "net" + "regexp" + "sync" + "time" ) // Support for newer SSL signature algorithms @@ -41,383 +39,363 @@ import _ "crypto/sha256" import _ "crypto/sha512" const ( - // Essentially, this is how often we should check for disconnect/shutdown during socket reads - socket_interval_seconds = 1 + // Essentially, this is how often we should check for disconnect/shutdown during socket reads + socket_interval_seconds = 1 ) type TransportTcpRegistrar struct { } type TransportTcpFactory struct { - transport string + transport string - SSLCertificate string `config:"ssl certificate"` - SSLKey string `config:"ssl key"` - SSLCA string `config:"ssl ca"` + SSLCertificate string `config:"ssl certificate"` + SSLKey string `config:"ssl key"` + SSLCA string `config:"ssl ca"` - hostport_re *regexp.Regexp - tls_config tls.Config + hostport_re *regexp.Regexp + tls_config tls.Config } type TransportTcp struct { - config *TransportTcpFactory - net_config *core.NetworkConfig - socket net.Conn - tlssocket *tls.Conn + config *TransportTcpFactory + net_config *core.NetworkConfig + socket net.Conn + tlssocket *tls.Conn - wait sync.WaitGroup - shutdown chan interface{} + wait sync.WaitGroup + shutdown chan interface{} - send_chan chan []byte - recv_chan chan interface{} + send_chan chan []byte + recv_chan chan interface{} - can_send chan int + can_send chan int - roundrobin int - host_is_ip bool - host string - port string - addresses []net.IP + addressPool *AddressPool } func NewTcpTransportFactory(config *core.Config, config_path string, unused map[string]interface{}, name string) (core.TransportFactory, error) { - var err error - - ret := &TransportTcpFactory{ - transport: name, - hostport_re: regexp.MustCompile(`^\[?([^]]+)\]?:([0-9]+)$`), - } - - // Only allow SSL configurations if this is "tls" - if name == "tls" { - if err = config.PopulateConfig(ret, config_path, unused); err != nil { - return nil, err - } - - if len(ret.SSLCertificate) > 0 && len(ret.SSLKey) > 0 { - cert, err := tls.LoadX509KeyPair(ret.SSLCertificate, ret.SSLKey) - if err != nil { - return nil, fmt.Errorf("Failed loading client ssl certificate: %s", err) - } - - ret.tls_config.Certificates = []tls.Certificate{cert} - } - - if len(ret.SSLCA) > 0 { - ret.tls_config.RootCAs = x509.NewCertPool() - - pemdata, err := ioutil.ReadFile(ret.SSLCA) - if err != nil { - return nil, fmt.Errorf("Failure reading CA certificate: %s", err) - } - - block, _ := pem.Decode(pemdata) - if block == nil { - return nil, errors.New("Failed to decode CA certificate data") - } - if block.Type != "CERTIFICATE" { - return nil, fmt.Errorf("Specified CA certificate is not a certificate: %s", ret.SSLCA) - } - - cert, err := x509.ParseCertificate(block.Bytes) - if err != nil { - return nil, fmt.Errorf("Failed to parse CA certificate: %s", err) - } - - ret.tls_config.RootCAs.AddCert(cert) - } - } else { - if err := config.ReportUnusedConfig(config_path, unused); err != nil { - return nil, err - } - } - - return ret, nil + var err error + + ret := &TransportTcpFactory{ + transport: name, + hostport_re: regexp.MustCompile(`^\[?([^]]+)\]?:([0-9]+)$`), + } + + // Only allow SSL configurations if this is "tls" + if name == "tls" { + if err = config.PopulateConfig(ret, config_path, unused); err != nil { + return nil, err + } + + if len(ret.SSLCertificate) > 0 && len(ret.SSLKey) > 0 { + cert, err := tls.LoadX509KeyPair(ret.SSLCertificate, ret.SSLKey) + if err != nil { + return nil, fmt.Errorf("Failed loading client ssl certificate: %s", err) + } + + ret.tls_config.Certificates = []tls.Certificate{cert} + } + + if len(ret.SSLCA) > 0 { + ret.tls_config.RootCAs = x509.NewCertPool() + pemdata, err := ioutil.ReadFile(ret.SSLCA) + if err != nil { + return nil, fmt.Errorf("Failure reading CA certificate: %s\n", err) + } + rest := pemdata + var block *pem.Block + var pemBlockNum = 1 + for { + block, rest = pem.Decode(rest) + if block != nil { + if block.Type != "CERTIFICATE" { + return nil, fmt.Errorf("Block %d does not contain a certificate: %s\n", pemBlockNum, ret.SSLCA) + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, fmt.Errorf("Failed to parse CA certificate in block %d: %s\n", pemBlockNum, ret.SSLCA) + } + ret.tls_config.RootCAs.AddCert(cert) + pemBlockNum += 1 + } else { + break + } + } + } + } else { + if err := config.ReportUnusedConfig(config_path, unused); err != nil { + return nil, err + } + } + + return ret, nil } func (f *TransportTcpFactory) NewTransport(config *core.NetworkConfig) (core.Transport, error) { - return &TransportTcp{config: f, net_config: config}, nil -} - -func (t *TransportTcp) ReloadConfig(new_net_config *core.NetworkConfig) int { - // Check we can grab new TCP config to compare, if not force transport reinit - new_config, ok := new_net_config.TransportFactory.(*TransportTcpFactory) - if !ok { - return core.Reload_Transport - } + ret := &TransportTcp{ + config: f, + net_config: config, + addressPool: NewAddressPool(config.Servers), + } - // TODO - This does not catch changes to the underlying certificate file! - if new_config.SSLCertificate != t.config.SSLCertificate || new_config.SSLKey != t.config.SSLKey || new_config.SSLCA != t.config.SSLCA { - return core.Reload_Transport - } + if ret.net_config.Rfc2782Srv { + ret.addressPool.SetRfc2782(true, ret.net_config.Rfc2782Service) + } - // Publisher handles changes to net_config, but ensure we store the latest in case it asks for a reconnect - t.net_config = new_net_config + return ret, nil +} - return core.Reload_None +func (t *TransportTcp) ReloadConfig(new_net_config *core.NetworkConfig) int { + // Check we can grab new TCP config to compare, if not force transport reinit + new_config, ok := new_net_config.TransportFactory.(*TransportTcpFactory) + if !ok { + return core.Reload_Transport + } + + // TODO - This does not catch changes to the underlying certificate file! + if new_config.SSLCertificate != t.config.SSLCertificate || new_config.SSLKey != t.config.SSLKey || new_config.SSLCA != t.config.SSLCA { + return core.Reload_Transport + } + + // Publisher handles changes to net_config, but ensure we store the latest in case it asks for a reconnect + t.net_config = new_net_config + t.addressPool = NewAddressPool(t.net_config.Servers) + + if t.net_config.Rfc2782Srv { + t.addressPool.SetRfc2782(true, t.net_config.Rfc2782Service) + } + + return core.Reload_None } func (t *TransportTcp) Init() error { - if t.shutdown != nil { - t.disconnect() - } - - // Have we exhausted the address list we had? - if t.addresses == nil { - var err error - - // Round robin to the next server - selected := t.net_config.Servers[t.roundrobin%len(t.net_config.Servers)] - t.roundrobin++ - - t.host, t.port, err = net.SplitHostPort(selected) - if err != nil { - return fmt.Errorf("Invalid hostport given: %s", selected) - } - - // Are we an IP? - if ip := net.ParseIP(t.host); ip != nil { - t.host_is_ip = true - t.addresses = []net.IP{ip} - } else { - // Lookup the server in DNS - t.host_is_ip = false - t.addresses, err = net.LookupIP(t.host) - if err != nil { - return fmt.Errorf("DNS lookup failure \"%s\": %s", t.host, err) - } - } - } - - // Try next address and drop it from our list - addressport := net.JoinHostPort(t.addresses[0].String(), t.port) - if len(t.addresses) > 1 { - t.addresses = t.addresses[1:] - } else { - t.addresses = nil - } - - var desc string - if t.host_is_ip { - desc = fmt.Sprintf("%s", addressport) - } else { - desc = fmt.Sprintf("%s (%s)", addressport, t.host) - } - - log.Info("Attempting to connect to %s", desc) - - tcpsocket, err := net.DialTimeout("tcp", addressport, t.net_config.Timeout) - if err != nil { - return fmt.Errorf("Failed to connect to %s: %s", desc, err) - } - - // Now wrap in TLS if this is the "tls" transport - if t.config.transport == "tls" { - // Set the tlsconfig server name for server validation (required since Go 1.3) - t.config.tls_config.ServerName = t.host - - t.tlssocket = tls.Client(&transportTcpWrap{transport: t, tcpsocket: tcpsocket}, &t.config.tls_config) - t.tlssocket.SetDeadline(time.Now().Add(t.net_config.Timeout)) - err = t.tlssocket.Handshake() - if err != nil { - t.tlssocket.Close() - tcpsocket.Close() - return fmt.Errorf("TLS Handshake failure with %s: %s", desc, err) - } - - t.socket = t.tlssocket - } else { - t.socket = tcpsocket - } - - log.Info("Connected to %s", desc) - - // Signal channels - t.shutdown = make(chan interface{}, 1) - t.send_chan = make(chan []byte, 1) - // Buffer of two for recv_chan since both routines may send an error to it - // First error we get back initiates disconnect, thus we must not block routines - t.recv_chan = make(chan interface{}, 2) - t.can_send = make(chan int, 1) - - // Start with a send - t.can_send <- 1 - - t.wait.Add(2) - - // Start separate sender and receiver so we can asynchronously send and receive for max performance - // They have to be different routines too because we don't have cross-platform poll, so they will need to block - // Of course, we'll time out and check shutdown on occasion - go t.sender() - go t.receiver() - - return nil + if t.shutdown != nil { + t.disconnect() + } + + // Try next address + addressport, desc, err := t.addressPool.Next() + if err != nil { + return err + } + + log.Info("Attempting to connect to %s", desc) + + tcpsocket, err := net.DialTimeout("tcp", addressport.String(), t.net_config.Timeout) + if err != nil { + return fmt.Errorf("Failed to connect to %s: %s", desc, err) + } + + // Now wrap in TLS if this is the "tls" transport + if t.config.transport == "tls" { + // Disable SSLv3 (mitigate POODLE vulnerability) + t.config.tls_config.MinVersion = tls.VersionTLS10 + + // Set the tlsconfig server name for server validation (required since Go 1.3) + t.config.tls_config.ServerName = t.addressPool.Host() + + t.tlssocket = tls.Client(&transportTcpWrap{transport: t, tcpsocket: tcpsocket}, &t.config.tls_config) + t.tlssocket.SetDeadline(time.Now().Add(t.net_config.Timeout)) + err = t.tlssocket.Handshake() + if err != nil { + t.tlssocket.Close() + tcpsocket.Close() + return fmt.Errorf("TLS Handshake failure with %s: %s", desc, err) + } + + t.socket = t.tlssocket + } else { + t.socket = tcpsocket + } + + log.Info("Connected to %s", desc) + + // Signal channels + t.shutdown = make(chan interface{}, 1) + t.send_chan = make(chan []byte, 1) + // Buffer of two for recv_chan since both routines may send an error to it + // First error we get back initiates disconnect, thus we must not block routines + t.recv_chan = make(chan interface{}, 2) + t.can_send = make(chan int, 1) + + // Start with a send + t.can_send <- 1 + + t.wait.Add(2) + + // Start separate sender and receiver so we can asynchronously send and receive for max performance + // They have to be different routines too because we don't have cross-platform poll, so they will need to block + // Of course, we'll time out and check shutdown on occasion + go t.sender() + go t.receiver() + + return nil } func (t *TransportTcp) disconnect() { - if t.shutdown == nil { - return - } + if t.shutdown == nil { + return + } - // Send shutdown request - close(t.shutdown) - t.wait.Wait() - t.shutdown = nil + // Send shutdown request + close(t.shutdown) + t.wait.Wait() + t.shutdown = nil - // If tls, shutdown tls socket first - if t.config.transport == "tls" { - t.tlssocket.Close() - } + // If tls, shutdown tls socket first + if t.config.transport == "tls" { + t.tlssocket.Close() + } - t.socket.Close() + t.socket.Close() } func (t *TransportTcp) sender() { SendLoop: - for { - select { - case <-t.shutdown: - // Shutdown - break SendLoop - case msg := <-t.send_chan: - // Ask for more while we send this - t.setChan(t.can_send) - // Write deadline is managed by our net.Conn wrapper that tls will call into - _, err := t.socket.Write(msg) - if err != nil { - if net_err, ok := err.(net.Error); ok && net_err.Timeout() { - // Shutdown will have been received by the wrapper - break SendLoop - } else { - // Pass error back - t.recv_chan <- err - } - } - } - } - - t.wait.Done() + for { + select { + case <-t.shutdown: + // Shutdown + break SendLoop + case msg := <-t.send_chan: + // Ask for more while we send this + t.setChan(t.can_send) + // Write deadline is managed by our net.Conn wrapper that tls will call into + _, err := t.socket.Write(msg) + if err != nil { + if net_err, ok := err.(net.Error); ok && net_err.Timeout() { + // Shutdown will have been received by the wrapper + break SendLoop + } else { + // Pass the error back and abort + t.recv_chan <- err + break SendLoop + } + } + } + } + + t.wait.Done() } func (t *TransportTcp) receiver() { - var err error - var shutdown bool - header := make([]byte, 8) - - for { - if err, shutdown = t.receiverRead(header); err != nil || shutdown { - break - } - - // Grab length of message - length := binary.BigEndian.Uint32(header[4:8]) - - // Sanity - if length > 1048576 { - err = fmt.Errorf("Data too large (%d)", length) - break - } - - // Allocate for full message - message := make([]byte, length) - - if err, shutdown = t.receiverRead(message); err != nil || shutdown { - break - } - - // Pass back the message - select { - case <-t.shutdown: - break - case t.recv_chan <- [][]byte{header[0:4], message}: - } - } /* loop until shutdown */ - - if err != nil { - // Pass the error back and abort - select { - case <-t.shutdown: - case t.recv_chan <- err: - } - } - - t.wait.Done() + var err error + var shutdown bool + header := make([]byte, 8) + + for { + if err, shutdown = t.receiverRead(header); err != nil || shutdown { + break + } + + // Grab length of message + length := binary.BigEndian.Uint32(header[4:8]) + + // Sanity + if length > 1048576 { + err = fmt.Errorf("Data too large (%d)", length) + break + } + + // Allocate for full message + message := make([]byte, length) + + if err, shutdown = t.receiverRead(message); err != nil || shutdown { + break + } + + // Pass back the message + select { + case <-t.shutdown: + break + case t.recv_chan <- [][]byte{header[0:4], message}: + } + } /* loop until shutdown */ + + if err != nil { + // Pass the error back and abort + select { + case <-t.shutdown: + case t.recv_chan <- err: + } + } + + t.wait.Done() } func (t *TransportTcp) receiverRead(data []byte) (error, bool) { - received := 0 + received := 0 RecvLoop: - for { - select { - case <-t.shutdown: - // Shutdown - break RecvLoop - default: - // Timeout after socket_interval_seconds, check for shutdown, and try again - t.socket.SetReadDeadline(time.Now().Add(socket_interval_seconds * time.Second)) - - length, err := t.socket.Read(data[received:]) - received += length - if err == nil || received >= len(data) { - // Success - return nil, false - } else if net_err, ok := err.(net.Error); ok && net_err.Timeout() { - // Keep trying - continue - } else { - // Pass an error back - return err, false - } - } /* select */ - } /* loop until required amount receive or shutdown */ - - return nil, true + for { + select { + case <-t.shutdown: + // Shutdown + break RecvLoop + default: + // Timeout after socket_interval_seconds, check for shutdown, and try again + t.socket.SetReadDeadline(time.Now().Add(socket_interval_seconds * time.Second)) + + length, err := t.socket.Read(data[received:]) + received += length + if err == nil || received >= len(data) { + // Success + return nil, false + } else if net_err, ok := err.(net.Error); ok && net_err.Timeout() { + // Keep trying + continue + } else { + // Pass an error back + return err, false + } + } /* select */ + } /* loop until required amount receive or shutdown */ + + return nil, true } func (t *TransportTcp) setChan(set chan<- int) { - select { - case set <- 1: - default: - } + select { + case set <- 1: + default: + } } func (t *TransportTcp) CanSend() <-chan int { - return t.can_send + return t.can_send } func (t *TransportTcp) Write(signature string, message []byte) (err error) { - var write_buffer *bytes.Buffer - write_buffer = bytes.NewBuffer(make([]byte, 0, len(signature)+4+len(message))) - - if _, err = write_buffer.Write([]byte(signature)); err != nil { - return - } - if err = binary.Write(write_buffer, binary.BigEndian, uint32(len(message))); err != nil { - return - } - if len(message) != 0 { - if _, err = write_buffer.Write(message); err != nil { - return - } - } - - t.send_chan <- write_buffer.Bytes() - return nil + var write_buffer *bytes.Buffer + write_buffer = bytes.NewBuffer(make([]byte, 0, len(signature)+4+len(message))) + + if _, err = write_buffer.Write([]byte(signature)); err != nil { + return + } + if err = binary.Write(write_buffer, binary.BigEndian, uint32(len(message))); err != nil { + return + } + if len(message) != 0 { + if _, err = write_buffer.Write(message); err != nil { + return + } + } + + t.send_chan <- write_buffer.Bytes() + return nil } func (t *TransportTcp) Read() <-chan interface{} { - return t.recv_chan + return t.recv_chan } func (t *TransportTcp) Shutdown() { - t.disconnect() + t.disconnect() } // Register the transports func init() { - rand.Seed(time.Now().UnixNano()) - - core.RegisterTransport("tcp", NewTcpTransportFactory) - core.RegisterTransport("tls", NewTcpTransportFactory) + core.RegisterTransport("tcp", NewTcpTransportFactory) + core.RegisterTransport("tls", NewTcpTransportFactory) } diff --git a/src/lc-lib/transports/tcp_wrap.go b/src/lc-lib/transports/tcp_wrap.go index c78f4e3d..811e2a63 100644 --- a/src/lc-lib/transports/tcp_wrap.go +++ b/src/lc-lib/transports/tcp_wrap.go @@ -17,71 +17,71 @@ package transports import ( - "net" - "time" + "net" + "time" ) // If tls.Conn.Write ever times out it will permanently break, so we cannot use SetWriteDeadline with it directly // So we wrap the given tcpsocket and handle the SetWriteDeadline there and check shutdown signal and loop // Inside tls.Conn the Write blocks until it finishes and everyone is happy type transportTcpWrap struct { - transport *TransportTcp - tcpsocket net.Conn + transport *TransportTcp + tcpsocket net.Conn - net.Conn + net.Conn } func (w *transportTcpWrap) Read(b []byte) (int, error) { - return w.tcpsocket.Read(b) + return w.tcpsocket.Read(b) } func (w *transportTcpWrap) Write(b []byte) (n int, err error) { - length := 0 + length := 0 RetrySend: - for { - // Timeout after socket_interval_seconds, check for shutdown, and try again - w.tcpsocket.SetWriteDeadline(time.Now().Add(socket_interval_seconds * time.Second)) + for { + // Timeout after socket_interval_seconds, check for shutdown, and try again + w.tcpsocket.SetWriteDeadline(time.Now().Add(socket_interval_seconds * time.Second)) - n, err = w.tcpsocket.Write(b[length:]) - length += n - if err == nil { - return length, err - } else if net_err, ok := err.(net.Error); ok && net_err.Timeout() { - // Check for shutdown, then try again - select { - case <-w.transport.shutdown: - // Shutdown - return length, err - default: - goto RetrySend - } - } else { - return length, err - } - } /* loop forever */ + n, err = w.tcpsocket.Write(b[length:]) + length += n + if err == nil { + return length, err + } else if net_err, ok := err.(net.Error); ok && net_err.Timeout() { + // Check for shutdown, then try again + select { + case <-w.transport.shutdown: + // Shutdown + return length, err + default: + goto RetrySend + } + } else { + return length, err + } + } /* loop forever */ } func (w *transportTcpWrap) Close() error { - return w.tcpsocket.Close() + return w.tcpsocket.Close() } func (w *transportTcpWrap) LocalAddr() net.Addr { - return w.tcpsocket.LocalAddr() + return w.tcpsocket.LocalAddr() } func (w *transportTcpWrap) RemoteAddr() net.Addr { - return w.tcpsocket.RemoteAddr() + return w.tcpsocket.RemoteAddr() } func (w *transportTcpWrap) SetDeadline(t time.Time) error { - return w.tcpsocket.SetDeadline(t) + return w.tcpsocket.SetDeadline(t) } func (w *transportTcpWrap) SetReadDeadline(t time.Time) error { - return w.tcpsocket.SetReadDeadline(t) + return w.tcpsocket.SetReadDeadline(t) } func (w *transportTcpWrap) SetWriteDeadline(t time.Time) error { - return w.tcpsocket.SetWriteDeadline(t) + return w.tcpsocket.SetWriteDeadline(t) } diff --git a/src/lc-lib/transports/zmq.go b/src/lc-lib/transports/zmq.go index f3d071fb..2426a2b0 100644 --- a/src/lc-lib/transports/zmq.go +++ b/src/lc-lib/transports/zmq.go @@ -19,727 +19,722 @@ package transports import ( - "bytes" - "encoding/binary" - "errors" - "fmt" - zmq "github.com/alecthomas/gozmq" - "lc-lib/core" - "net" - "regexp" - "runtime" - "sync" - "syscall" + "bytes" + "encoding/binary" + "errors" + "fmt" + zmq "github.com/alecthomas/gozmq" + "github.com/driskell/log-courier/src/lc-lib/core" + "regexp" + "runtime" + "sync" + "syscall" ) const ( - zmq_signal_output = "O" - zmq_signal_input = "I" - zmq_signal_shutdown = "S" + zmq_signal_output = "O" + zmq_signal_input = "I" + zmq_signal_shutdown = "S" ) const ( - Monitor_Part_Header = iota - Monitor_Part_Data - Monitor_Part_Extraneous + Monitor_Part_Header = iota + Monitor_Part_Data + Monitor_Part_Extraneous ) const ( - default_NetworkConfig_PeerSendQueue int64 = 2 + default_NetworkConfig_PeerSendQueue int64 = 2 ) type TransportZmqFactory struct { - transport string + transport string - CurveServerkey string `config:"curve server key"` - CurvePublickey string `config:"curve public key"` - CurveSecretkey string `config:"curve secret key"` + CurveServerkey string `config:"curve server key"` + CurvePublickey string `config:"curve public key"` + CurveSecretkey string `config:"curve secret key"` - PeerSendQueue int64 + PeerSendQueue int64 - hostport_re *regexp.Regexp + hostport_re *regexp.Regexp } type TransportZmq struct { - config *TransportZmqFactory - net_config *core.NetworkConfig - context *zmq.Context - dealer *zmq.Socket - monitor *zmq.Socket - poll_items []zmq.PollItem - send_buff *ZMQMessage - recv_buff [][]byte - recv_body bool - event ZMQEvent - ready bool - - wait sync.WaitGroup - - bridge_chan chan []byte - - send_chan chan *ZMQMessage - recv_chan chan interface{} - recv_bridge_chan chan interface{} - - can_send chan int + config *TransportZmqFactory + net_config *core.NetworkConfig + context *zmq.Context + dealer *zmq.Socket + monitor *zmq.Socket + poll_items []zmq.PollItem + send_buff *ZMQMessage + recv_buff [][]byte + recv_body bool + event ZMQEvent + ready bool + + wait sync.WaitGroup + + bridge_chan chan []byte + + send_chan chan *ZMQMessage + recv_chan chan interface{} + recv_bridge_chan chan interface{} + + can_send chan int } type ZMQMessage struct { - part []byte - final bool + part []byte + final bool } type ZMQEvent struct { - part int - event zmq.Event - val int32 - data string + part int + event zmq.Event + val int32 + data string } func (e *ZMQEvent) Log() { - switch e.event { - case zmq.EVENT_CONNECTED: - if e.data == "" { - log.Info("Connected") - } else { - log.Info("Connected to %s", e.data) - } - case zmq.EVENT_CONNECT_DELAYED: - // Don't log anything for this - case zmq.EVENT_CONNECT_RETRIED: - if e.data == "" { - log.Info("Attempting to connect") - } else { - log.Info("Attempting to connect to %s", e.data) - } - case zmq.EVENT_CLOSED: - if e.data == "" { - log.Error("Connection closed") - } else { - log.Error("Connection to %s closed", e.data) - } - case zmq.EVENT_DISCONNECTED: - if e.data == "" { - log.Error("Lost connection") - } else { - log.Error("Lost connection to %s", e.data) - } - default: - log.Debug("Unknown monitor message (event:%d, val:%d, data:[% X])", e.event, e.val, e.data) - } + switch e.event { + case zmq.EVENT_CONNECTED: + if e.data == "" { + log.Info("Connected") + } else { + log.Info("Connected to %s", e.data) + } + case zmq.EVENT_CONNECT_DELAYED: + // Don't log anything for this + case zmq.EVENT_CONNECT_RETRIED: + if e.data == "" { + log.Info("Attempting to connect") + } else { + log.Info("Attempting to connect to %s", e.data) + } + case zmq.EVENT_CLOSED: + if e.data == "" { + log.Error("Connection closed") + } else { + log.Error("Connection to %s closed", e.data) + } + case zmq.EVENT_DISCONNECTED: + if e.data == "" { + log.Error("Lost connection") + } else { + log.Error("Lost connection to %s", e.data) + } + default: + log.Debug("Unknown monitor message (event:%d, val:%d, data:[% X])", e.event, e.val, e.data) + } } type TransportZmqRegistrar struct { } func NewZmqTransportFactory(config *core.Config, config_path string, unused map[string]interface{}, name string) (core.TransportFactory, error) { - var err error - - ret := &TransportZmqFactory{ - transport: name, - hostport_re: regexp.MustCompile(`^\[?([^]]+)\]?:([0-9]+)$`), - } - - if name == "zmq" { - if err = config.PopulateConfig(ret, config_path, unused); err != nil { - return nil, err - } - - if err := ret.processConfig(config_path); err != nil { - return nil, err - } - - return ret, nil - } - - // Don't allow curve settings - if _, ok := unused["CurveServerkey"]; ok { - goto CheckUnused - } - if _, ok := unused["CurvePublickey"]; ok { - goto CheckUnused - } - if _, ok := unused["CurveSecretkey"]; ok { - goto CheckUnused - } - - if err = config.PopulateConfig(ret, config_path, unused); err != nil { - return nil, err - } - - if ret.PeerSendQueue == 0 { - ret.PeerSendQueue = default_NetworkConfig_PeerSendQueue - } + var err error + + ret := &TransportZmqFactory{ + transport: name, + hostport_re: regexp.MustCompile(`^\[?([^]]+)\]?:([0-9]+)$`), + } + + if name == "zmq" { + if err = config.PopulateConfig(ret, config_path, unused); err != nil { + return nil, err + } + + if err := ret.processConfig(config_path); err != nil { + return nil, err + } + + return ret, nil + } + + // Don't allow curve settings + if _, ok := unused["CurveServerkey"]; ok { + goto CheckUnused + } + if _, ok := unused["CurvePublickey"]; ok { + goto CheckUnused + } + if _, ok := unused["CurveSecretkey"]; ok { + goto CheckUnused + } + + if err = config.PopulateConfig(ret, config_path, unused); err != nil { + return nil, err + } + + if ret.PeerSendQueue == 0 { + ret.PeerSendQueue = default_NetworkConfig_PeerSendQueue + } CheckUnused: - if err := config.ReportUnusedConfig(config_path, unused); err != nil { - return nil, err - } + if err := config.ReportUnusedConfig(config_path, unused); err != nil { + return nil, err + } - return ret, nil + return ret, nil } func (f *TransportZmqFactory) NewTransport(config *core.NetworkConfig) (core.Transport, error) { - return &TransportZmq{config: f, net_config: config}, nil + return &TransportZmq{config: f, net_config: config}, nil } func (t *TransportZmq) ReloadConfig(new_net_config *core.NetworkConfig) int { - // Check we can grab new ZMQ config to compare, if not force transport reinit - new_config, ok := new_net_config.TransportFactory.(*TransportZmqFactory) - if !ok { - return core.Reload_Transport - } + // Check we can grab new ZMQ config to compare, if not force transport reinit + new_config, ok := new_net_config.TransportFactory.(*TransportZmqFactory) + if !ok { + return core.Reload_Transport + } - if new_config.CurveServerkey != t.config.CurveServerkey || new_config.CurvePublickey != t.config.CurvePublickey || new_config.CurveSecretkey != t.config.CurveSecretkey { - return core.Reload_Transport - } + if new_config.CurveServerkey != t.config.CurveServerkey || new_config.CurvePublickey != t.config.CurvePublickey || new_config.CurveSecretkey != t.config.CurveSecretkey { + return core.Reload_Transport + } - // Publisher handles changes to net_config, but ensure we store the latest in case it asks for a reconnect - t.net_config = new_net_config + // Publisher handles changes to net_config, but ensure we store the latest in case it asks for a reconnect + t.net_config = new_net_config - return core.Reload_None + return core.Reload_None } func (t *TransportZmq) Init() (err error) { - // Initialise once for ZMQ - if t.ready { - // If already initialised, ask if we can send again - t.bridge_chan <- []byte(zmq_signal_output) - return nil - } - - t.context, err = zmq.NewContext() - if err != nil { - return fmt.Errorf("Failed to create ZMQ context: %s", err) - } - defer func() { - if err != nil { - t.context.Close() - } - }() - - // Control sockets to connect bridge to poller - bridge_in, err := t.context.NewSocket(zmq.PUSH) - if err != nil { - return fmt.Errorf("Failed to create internal ZMQ PUSH socket: %s", err) - } - defer func() { - if err != nil { - bridge_in.Close() - } - }() - - if err = bridge_in.Bind("inproc://notify"); err != nil { - return fmt.Errorf("Failed to bind internal ZMQ PUSH socket: %s", err) - } - - bridge_out, err := t.context.NewSocket(zmq.PULL) - if err != nil { - return fmt.Errorf("Failed to create internal ZMQ PULL socket: %s", err) - } - defer func() { - if err != nil { - bridge_out.Close() - } - }() - - if err = bridge_out.Connect("inproc://notify"); err != nil { - return fmt.Errorf("Failed to connect internal ZMQ PULL socket: %s", err) - } - - // Outbound dealer socket will fair-queue load balance amongst peers - if t.dealer, err = t.context.NewSocket(zmq.DEALER); err != nil { - return fmt.Errorf("Failed to create ZMQ DEALER socket: %s", err) - } - defer func() { - if err != nil { - t.dealer.Close() - } - }() - - if err = t.dealer.Monitor("inproc://monitor", zmq.EVENT_ALL); err != nil { - return fmt.Errorf("Failed to bind DEALER socket to monitor: %s", err) - } - - if err = t.configureSocket(); err != nil { - return fmt.Errorf("Failed to configure DEALER socket: %s", err) - } - - // Configure reconnect interval - if err = t.dealer.SetReconnectIvlMax(t.net_config.Reconnect); err != nil { - return fmt.Errorf("Failed to set ZMQ reconnect interval: %s", err) - } - - // We should not LINGER. If we do, socket Close and also context Close will - // block infinitely until the message queue is flushed. Set to 0 to discard - // all messages immediately when we call Close - if err = t.dealer.SetLinger(0); err != nil { - return fmt.Errorf("Failed to set ZMQ linger period: %s", err) - } - - // Set the outbound queue - if err = t.dealer.SetSndHWM(int(t.config.PeerSendQueue)); err != nil { - return fmt.Errorf("Failed to set ZMQ send highwater: %s", err) - } - - // Monitor socket - if t.monitor, err = t.context.NewSocket(zmq.PULL); err != nil { - return fmt.Errorf("Failed to create monitor ZMQ PULL socket: %s", err) - } - defer func() { - if err != nil { - t.monitor.Close() - } - }() - - if err = t.monitor.Connect("inproc://monitor"); err != nil { - return fmt.Errorf("Failed to connect monitor ZMQ PULL socket: %s", err) - } - - // Register endpoints - endpoints := 0 - for _, hostport := range t.net_config.Servers { - submatch := t.config.hostport_re.FindSubmatch([]byte(hostport)) - if submatch == nil { - log.Warning("Invalid host:port given: %s", hostport) - continue - } - - // Lookup the server in DNS (if this is IP it will implicitly return) - host := string(submatch[1]) - port := string(submatch[2]) - addresses, err := net.LookupHost(host) - if err != nil { - log.Warning("DNS lookup failure \"%s\": %s", host, err) - continue - } - - // Register each address - for _, address := range addresses { - addressport := net.JoinHostPort(address, port) - - if err = t.dealer.Connect("tcp://" + addressport); err != nil { - log.Warning("Failed to register %s (%s) with ZMQ, skipping", addressport, host) - continue - } - - log.Info("Registered %s (%s) with ZMQ", addressport, host) - endpoints++ - } - } - - if endpoints == 0 { - return errors.New("Failed to register any of the specified endpoints.") - } - - major, minor, patch := zmq.Version() - log.Info("libzmq version %d.%d.%d", major, minor, patch) - - // Signal channels - t.bridge_chan = make(chan []byte, 1) - t.send_chan = make(chan *ZMQMessage, 2) - t.recv_chan = make(chan interface{}, 1) - t.recv_bridge_chan = make(chan interface{}, 1) - t.can_send = make(chan int, 1) - - // Waiter we use to wait for shutdown - t.wait.Add(2) - - // Bridge between channels and ZMQ - go t.bridge(bridge_in) - - // The poller - go t.poller(bridge_out) - - t.ready = true - t.send_buff = nil - t.recv_buff = nil - t.recv_body = false - - return nil + // Initialise once for ZMQ + if t.ready { + // If already initialised, ask if we can send again + t.bridge_chan <- []byte(zmq_signal_output) + return nil + } + + t.context, err = zmq.NewContext() + if err != nil { + return fmt.Errorf("Failed to create ZMQ context: %s", err) + } + defer func() { + if err != nil { + t.context.Close() + } + }() + + // Control sockets to connect bridge to poller + bridge_in, err := t.context.NewSocket(zmq.PUSH) + if err != nil { + return fmt.Errorf("Failed to create internal ZMQ PUSH socket: %s", err) + } + defer func() { + if err != nil { + bridge_in.Close() + } + }() + + if err = bridge_in.Bind("inproc://notify"); err != nil { + return fmt.Errorf("Failed to bind internal ZMQ PUSH socket: %s", err) + } + + bridge_out, err := t.context.NewSocket(zmq.PULL) + if err != nil { + return fmt.Errorf("Failed to create internal ZMQ PULL socket: %s", err) + } + defer func() { + if err != nil { + bridge_out.Close() + } + }() + + if err = bridge_out.Connect("inproc://notify"); err != nil { + return fmt.Errorf("Failed to connect internal ZMQ PULL socket: %s", err) + } + + // Outbound dealer socket will fair-queue load balance amongst peers + if t.dealer, err = t.context.NewSocket(zmq.DEALER); err != nil { + return fmt.Errorf("Failed to create ZMQ DEALER socket: %s", err) + } + defer func() { + if err != nil { + t.dealer.Close() + } + }() + + if err = t.dealer.Monitor("inproc://monitor", zmq.EVENT_ALL); err != nil { + return fmt.Errorf("Failed to bind DEALER socket to monitor: %s", err) + } + + if err = t.configureSocket(); err != nil { + return fmt.Errorf("Failed to configure DEALER socket: %s", err) + } + + // Configure reconnect interval + if err = t.dealer.SetReconnectIvlMax(t.net_config.Reconnect); err != nil { + return fmt.Errorf("Failed to set ZMQ reconnect interval: %s", err) + } + + // We should not LINGER. If we do, socket Close and also context Close will + // block infinitely until the message queue is flushed. Set to 0 to discard + // all messages immediately when we call Close + if err = t.dealer.SetLinger(0); err != nil { + return fmt.Errorf("Failed to set ZMQ linger period: %s", err) + } + + // Set the outbound queue + if err = t.dealer.SetSndHWM(int(t.config.PeerSendQueue)); err != nil { + return fmt.Errorf("Failed to set ZMQ send highwater: %s", err) + } + + // Monitor socket + if t.monitor, err = t.context.NewSocket(zmq.PULL); err != nil { + return fmt.Errorf("Failed to create monitor ZMQ PULL socket: %s", err) + } + defer func() { + if err != nil { + t.monitor.Close() + } + }() + + if err = t.monitor.Connect("inproc://monitor"); err != nil { + return fmt.Errorf("Failed to connect monitor ZMQ PULL socket: %s", err) + } + + // Register endpoints + pool := NewAddressPool(t.net_config.Servers) + endpoints := 0 + + if t.net_config.Rfc2782Srv { + pool.SetRfc2782(true, t.net_config.Rfc2782Service) + } + + for { + addressport, err := pool.NextServer() + if err != nil { + return err + } + + if err = t.dealer.Connect("tcp://" + addressport); err != nil { + log.Warning("Failed to register %s with ZMQ, skipping", addressport) + goto NextAddress + } + + log.Info("Registered %s with ZMQ", addressport) + endpoints++ + + NextAddress: + if pool.IsLastServer() { + break + } + } + + if endpoints == 0 { + return errors.New("Failed to register any of the specified endpoints.") + } + + major, minor, patch := zmq.Version() + log.Info("libzmq version %d.%d.%d", major, minor, patch) + + // Signal channels + t.bridge_chan = make(chan []byte, 1) + t.send_chan = make(chan *ZMQMessage, 2) + t.recv_chan = make(chan interface{}, 1) + t.recv_bridge_chan = make(chan interface{}, 1) + t.can_send = make(chan int, 1) + + // Waiter we use to wait for shutdown + t.wait.Add(2) + + // Bridge between channels and ZMQ + go t.bridge(bridge_in) + + // The poller + go t.poller(bridge_out) + + t.ready = true + t.send_buff = nil + t.recv_buff = nil + t.recv_body = false + + return nil } func (t *TransportZmq) bridge(bridge_in *zmq.Socket) { - var message interface{} + var message interface{} - // Wait on channel, passing into socket - // This keeps the socket in a single thread, otherwise we have to lock the entire publisher - runtime.LockOSThread() + // Wait on channel, passing into socket + // This keeps the socket in a single thread, otherwise we have to lock the entire publisher + runtime.LockOSThread() BridgeLoop: - for { - select { - case notify := <-t.bridge_chan: - bridge_in.Send(notify, 0) - - // Shutdown? - if string(notify) == zmq_signal_shutdown { - break BridgeLoop - } - case message = <-t.recv_bridge_chan: - // The reason we flush recv through the bridge and not directly to recv_chan is so that if - // the poller was quick and had to cache a receive as the channel was full, it will stop - // polling - flushing through bridge allows us to signal poller to start polling again - // It is not the publisher's responsibility to do this, and TLS wouldn't need it - bridge_in.Send([]byte(zmq_signal_input), 0) - - // Keep trying to forward on the message - ForwardLoop: - for { - select { - case notify := <-t.bridge_chan: - bridge_in.Send(notify, 0) - - // Shutdown? - if string(notify) == zmq_signal_shutdown { - break BridgeLoop - } - case t.recv_chan <- message: - break ForwardLoop - } - } - } - } - - // We should linger by default to ensure shutdown is transmitted - bridge_in.Close() - runtime.UnlockOSThread() - t.wait.Done() + for { + select { + case notify := <-t.bridge_chan: + bridge_in.Send(notify, 0) + + // Shutdown? + if string(notify) == zmq_signal_shutdown { + break BridgeLoop + } + case message = <-t.recv_bridge_chan: + // The reason we flush recv through the bridge and not directly to recv_chan is so that if + // the poller was quick and had to cache a receive as the channel was full, it will stop + // polling - flushing through bridge allows us to signal poller to start polling again + // It is not the publisher's responsibility to do this, and TLS wouldn't need it + bridge_in.Send([]byte(zmq_signal_input), 0) + + // Keep trying to forward on the message + ForwardLoop: + for { + select { + case notify := <-t.bridge_chan: + bridge_in.Send(notify, 0) + + // Shutdown? + if string(notify) == zmq_signal_shutdown { + break BridgeLoop + } + case t.recv_chan <- message: + break ForwardLoop + } + } + } + } + + // We should linger by default to ensure shutdown is transmitted + bridge_in.Close() + runtime.UnlockOSThread() + t.wait.Done() } func (t *TransportZmq) poller(bridge_out *zmq.Socket) { - // ZMQ sockets are not thread-safe, so we have to send/receive on same thread - // Thus, we cannot use a sender/receiver thread pair like we can with TLS so we use a single threaded poller instead - // In order to asynchronously send and receive we just poll and do necessary actions - - // When data is ready to send we'll get a channel ping, that is bridged to ZMQ so we can then send data - // For receiving, we receive here and bridge it to the channels, then receive more once that's through - runtime.LockOSThread() - - t.poll_items = make([]zmq.PollItem, 3) - - // Listen always on bridge - t.poll_items[0].Socket = bridge_out - t.poll_items[0].Events = zmq.POLLIN | zmq.POLLOUT - - // Always check for input on dealer - but also initially check for OUT so we can flag send is ready - t.poll_items[1].Socket = t.dealer - t.poll_items[1].Events = zmq.POLLIN | zmq.POLLOUT - - // Always listen for input on monitor - t.poll_items[2].Socket = t.monitor - t.poll_items[2].Events = zmq.POLLIN - - for { - // Poll for events - if _, err := zmq.Poll(t.poll_items, -1); err != nil { - // Retry on EINTR - if err == syscall.EINTR { - continue - } - - // Failure - t.recv_chan <- fmt.Errorf("zmq.Poll failure %s", err) - break - } - - // Process control channel - if t.poll_items[0].REvents&zmq.POLLIN != 0 { - if !t.processControlIn(bridge_out) { - break - } - } - - // Process dealer receive - if t.poll_items[1].REvents&zmq.POLLIN != 0 { - if !t.processDealerIn() { - break - } - } - - // Process dealer send - if t.poll_items[1].REvents&zmq.POLLOUT != 0 { - if !t.processDealerOut() { - break - } - } - - // Process monitor receive - if t.poll_items[2].REvents&zmq.POLLIN != 0 { - if !t.processMonitorIn() { - break - } - } - } - - bridge_out.Close() - runtime.UnlockOSThread() - t.wait.Done() + // ZMQ sockets are not thread-safe, so we have to send/receive on same thread + // Thus, we cannot use a sender/receiver thread pair like we can with TLS so we use a single threaded poller instead + // In order to asynchronously send and receive we just poll and do necessary actions + + // When data is ready to send we'll get a channel ping, that is bridged to ZMQ so we can then send data + // For receiving, we receive here and bridge it to the channels, then receive more once that's through + runtime.LockOSThread() + + t.poll_items = make([]zmq.PollItem, 3) + + // Listen always on bridge + t.poll_items[0].Socket = bridge_out + t.poll_items[0].Events = zmq.POLLIN | zmq.POLLOUT + + // Always check for input on dealer - but also initially check for OUT so we can flag send is ready + t.poll_items[1].Socket = t.dealer + t.poll_items[1].Events = zmq.POLLIN | zmq.POLLOUT + + // Always listen for input on monitor + t.poll_items[2].Socket = t.monitor + t.poll_items[2].Events = zmq.POLLIN + + for { + // Poll for events + if _, err := zmq.Poll(t.poll_items, -1); err != nil { + // Retry on EINTR + if err == syscall.EINTR { + continue + } + + // Failure + t.recv_chan <- fmt.Errorf("zmq.Poll failure %s", err) + break + } + + // Process control channel + if t.poll_items[0].REvents&zmq.POLLIN != 0 { + if !t.processControlIn(bridge_out) { + break + } + } + + // Process dealer receive + if t.poll_items[1].REvents&zmq.POLLIN != 0 { + if !t.processDealerIn() { + break + } + } + + // Process dealer send + if t.poll_items[1].REvents&zmq.POLLOUT != 0 { + if !t.processDealerOut() { + break + } + } + + // Process monitor receive + if t.poll_items[2].REvents&zmq.POLLIN != 0 { + if !t.processMonitorIn() { + break + } + } + } + + bridge_out.Close() + runtime.UnlockOSThread() + t.wait.Done() } func (t *TransportZmq) processControlIn(bridge_out *zmq.Socket) (ok bool) { - for { - RetryControl: - msg, err := bridge_out.Recv(zmq.DONTWAIT) - if err != nil { - switch err { - case syscall.EINTR: - // Try again - goto RetryControl - case syscall.EAGAIN: - // No more messages - return true - } - - // Failure - t.recv_chan <- fmt.Errorf("Pull zmq.Socket.Recv failure %s", err) - return - } - - switch string(msg) { - case zmq_signal_output: - // Start polling for send - t.poll_items[1].Events = t.poll_items[1].Events | zmq.POLLOUT - case zmq_signal_input: - // If we staged a receive, process that - if t.recv_buff != nil { - select { - case t.recv_bridge_chan <- t.recv_buff: - t.recv_buff = nil - - // Start polling for receive - t.poll_items[1].Events = t.poll_items[1].Events | zmq.POLLIN - default: - // Do nothing, we were asked for receive but channel is already full - } - } else { - // Start polling for receive - t.poll_items[1].Events = t.poll_items[1].Events | zmq.POLLIN - } - case zmq_signal_shutdown: - // Shutdown - return - } - } + for { + RetryControl: + msg, err := bridge_out.Recv(zmq.DONTWAIT) + if err != nil { + switch err { + case syscall.EINTR: + // Try again + goto RetryControl + case syscall.EAGAIN: + // No more messages + return true + } + + // Failure + t.recv_chan <- fmt.Errorf("Pull zmq.Socket.Recv failure %s", err) + return + } + + switch string(msg) { + case zmq_signal_output: + // Start polling for send + t.poll_items[1].Events = t.poll_items[1].Events | zmq.POLLOUT + case zmq_signal_input: + // If we staged a receive, process that + if t.recv_buff != nil { + select { + case t.recv_bridge_chan <- t.recv_buff: + t.recv_buff = nil + + // Start polling for receive + t.poll_items[1].Events = t.poll_items[1].Events | zmq.POLLIN + default: + // Do nothing, we were asked for receive but channel is already full + } + } else { + // Start polling for receive + t.poll_items[1].Events = t.poll_items[1].Events | zmq.POLLIN + } + case zmq_signal_shutdown: + // Shutdown + return + } + } } func (t *TransportZmq) processDealerOut() (ok bool) { - var sent_one bool - - // Something in the staging buffer? - if t.send_buff != nil { - sent, s_ok := t.dealerSend(t.send_buff) - if !s_ok { - return - } - if !sent { - ok = true - return - } - - t.send_buff = nil - sent_one = true - } - - // Send messages from channel + var sent_one bool + + // Something in the staging buffer? + if t.send_buff != nil { + sent, s_ok := t.dealerSend(t.send_buff) + if !s_ok { + return + } + if !sent { + ok = true + return + } + + t.send_buff = nil + sent_one = true + } + + // Send messages from channel LoopSend: - for { - select { - case msg := <-t.send_chan: - sent, s_ok := t.dealerSend(msg) - if !s_ok { - return - } - if !sent { - t.send_buff = msg - break - } - - sent_one = true - default: - break LoopSend - } - } - - if sent_one { - // We just sent something, check POLLOUT still active before signalling we can send more - // TODO: Check why Events() is returning uint64 instead of PollEvents - // TODO: This is broken and actually returns an error - /*if events, _ := t.dealer.Events(); zmq.PollEvents(events)&zmq.POLLOUT != 0 { - t.poll_items[1].Events = t.poll_items[1].Events ^ zmq.POLLOUT - t.setChan(t.can_send) - }*/ - } else { - t.poll_items[1].Events = t.poll_items[1].Events ^ zmq.POLLOUT - t.setChan(t.can_send) - } - - ok = true - return + for { + select { + case msg := <-t.send_chan: + sent, s_ok := t.dealerSend(msg) + if !s_ok { + return + } + if !sent { + t.send_buff = msg + break + } + + sent_one = true + default: + break LoopSend + } + } + + if sent_one { + // We just sent something, check POLLOUT still active before signalling we can send more + // TODO: Check why Events() is returning uint64 instead of PollEvents + // TODO: This is broken and actually returns an error + /*if events, _ := t.dealer.Events(); zmq.PollEvents(events)&zmq.POLLOUT != 0 { + t.poll_items[1].Events = t.poll_items[1].Events ^ zmq.POLLOUT + t.setChan(t.can_send) + }*/ + } else { + t.poll_items[1].Events = t.poll_items[1].Events ^ zmq.POLLOUT + t.setChan(t.can_send) + } + + ok = true + return } func (t *TransportZmq) dealerSend(msg *ZMQMessage) (sent bool, ok bool) { - var err error + var err error RetrySend: - if msg.final { - err = t.dealer.Send(msg.part, zmq.DONTWAIT) - } else { - err = t.dealer.Send(msg.part, zmq.DONTWAIT|zmq.SNDMORE) - } - if err != nil { - switch err { - case syscall.EINTR: - // Try again - goto RetrySend - case syscall.EAGAIN: - // No more messages - ok = true - return - } - - // Failure - t.recv_chan <- fmt.Errorf("Dealer zmq.Socket.Send failure %s", err) - return - } - - sent = true - ok = true - return + if msg.final { + err = t.dealer.Send(msg.part, zmq.DONTWAIT) + } else { + err = t.dealer.Send(msg.part, zmq.DONTWAIT|zmq.SNDMORE) + } + if err != nil { + switch err { + case syscall.EINTR: + // Try again + goto RetrySend + case syscall.EAGAIN: + // No more messages + ok = true + return + } + + // Failure + t.recv_chan <- fmt.Errorf("Dealer zmq.Socket.Send failure %s", err) + return + } + + sent = true + ok = true + return } func (t *TransportZmq) processDealerIn() (ok bool) { - for { - // Bring in the messages - RetryRecv: - data, err := t.dealer.Recv(zmq.DONTWAIT) - if err != nil { - switch err { - case syscall.EINTR: - // Try again - goto RetryRecv - case syscall.EAGAIN: - // No more messages - ok = true - return - } - - // Failure - t.recv_chan <- fmt.Errorf("Dealer zmq.Socket.Recv failure %s", err) - return - } - - more, err := t.dealer.RcvMore() - if err != nil { - // Failure - t.recv_chan <- fmt.Errorf("Dealer zmq.Socket.RcvMore failure %s", err) - return - } - - // Sanity check, and don't save until empty message - if len(data) == 0 && more { - // Message separator, start returning - t.recv_body = true - continue - } else if more || !t.recv_body { - // Ignore all but last message - continue - } - - t.recv_body = false - - // Last message and receiving, validate it first - if len(data) < 8 { - log.Warning("Skipping invalid message: not enough data") - continue - } - - length := binary.BigEndian.Uint32(data[4:8]) - if length > 1048576 { - log.Warning("Skipping invalid message: data too large (%d)", length) - continue - } else if length != uint32(len(data))-8 { - log.Warning("Skipping invalid message: data has invalid length (%d != %d)", len(data)-8, length) - continue - } - - message := [][]byte{data[0:4], data[8:]} - - // Bridge to channels - select { - case t.recv_bridge_chan <- message: - default: - // We filled the channel, stop polling until we pull something off of it and stage the recv - t.recv_buff = message - t.poll_items[1].Events = t.poll_items[1].Events ^ zmq.POLLIN - ok = true - return - } - } + for { + // Bring in the messages + RetryRecv: + data, err := t.dealer.Recv(zmq.DONTWAIT) + if err != nil { + switch err { + case syscall.EINTR: + // Try again + goto RetryRecv + case syscall.EAGAIN: + // No more messages + ok = true + return + } + + // Failure + t.recv_chan <- fmt.Errorf("Dealer zmq.Socket.Recv failure %s", err) + return + } + + more, err := t.dealer.RcvMore() + if err != nil { + // Failure + t.recv_chan <- fmt.Errorf("Dealer zmq.Socket.RcvMore failure %s", err) + return + } + + // Sanity check, and don't save until empty message + if len(data) == 0 && more { + // Message separator, start returning + t.recv_body = true + continue + } else if more || !t.recv_body { + // Ignore all but last message + continue + } + + t.recv_body = false + + // Last message and receiving, validate it first + if len(data) < 8 { + log.Warning("Skipping invalid message: not enough data") + continue + } + + length := binary.BigEndian.Uint32(data[4:8]) + if length > 1048576 { + log.Warning("Skipping invalid message: data too large (%d)", length) + continue + } else if length != uint32(len(data))-8 { + log.Warning("Skipping invalid message: data has invalid length (%d != %d)", len(data)-8, length) + continue + } + + message := [][]byte{data[0:4], data[8:]} + + // Bridge to channels + select { + case t.recv_bridge_chan <- message: + default: + // We filled the channel, stop polling until we pull something off of it and stage the recv + t.recv_buff = message + t.poll_items[1].Events = t.poll_items[1].Events ^ zmq.POLLIN + ok = true + return + } + } } func (t *TransportZmq) setChan(set chan int) { - select { - case set <- 1: - default: - } + select { + case set <- 1: + default: + } } func (t *TransportZmq) CanSend() <-chan int { - return t.can_send + return t.can_send } func (t *TransportZmq) Write(signature string, message []byte) (err error) { - var write_buffer *bytes.Buffer - write_buffer = bytes.NewBuffer(make([]byte, 0, len(signature)+4+len(message))) - - if _, err = write_buffer.Write([]byte(signature)); err != nil { - return - } - if err = binary.Write(write_buffer, binary.BigEndian, uint32(len(message))); err != nil { - return - } - if len(message) != 0 { - if _, err = write_buffer.Write(message); err != nil { - return - } - } - - // TODO: Fix regression where we could pend all payloads on a single ZMQ peer - // We should switch to full freelance pattern - ROUTER-to-ROUTER - // For this to work with ZMQ 3.2 we need to force identities on the server - // as the connection IP:Port and use those identities as available send pool - // For ZMQ 4+ we can skip using those and enable ZMQ_ROUTER_PROBE which sends - // an empty message on connection - server should respond with empty message - // which will allow us to populate identity list that way. - // ZMQ 4 approach is rigid as it means we don't need to rely on fixed - // identities - - t.send_chan <- &ZMQMessage{part: []byte(""), final: false} - t.send_chan <- &ZMQMessage{part: write_buffer.Bytes(), final: true} - - // Ask for send to start - t.bridge_chan <- []byte(zmq_signal_output) - return nil + var write_buffer *bytes.Buffer + write_buffer = bytes.NewBuffer(make([]byte, 0, len(signature)+4+len(message))) + + if _, err = write_buffer.Write([]byte(signature)); err != nil { + return + } + if err = binary.Write(write_buffer, binary.BigEndian, uint32(len(message))); err != nil { + return + } + if len(message) != 0 { + if _, err = write_buffer.Write(message); err != nil { + return + } + } + + // TODO: Fix regression where we could pend all payloads on a single ZMQ peer + // We should switch to full freelance pattern - ROUTER-to-ROUTER + // For this to work with ZMQ 3.2 we need to force identities on the server + // as the connection IP:Port and use those identities as available send pool + // For ZMQ 4+ we can skip using those and enable ZMQ_ROUTER_PROBE which sends + // an empty message on connection - server should respond with empty message + // which will allow us to populate identity list that way. + // ZMQ 4 approach is rigid as it means we don't need to rely on fixed + // identities + + t.send_chan <- &ZMQMessage{part: []byte(""), final: false} + t.send_chan <- &ZMQMessage{part: write_buffer.Bytes(), final: true} + + // Ask for send to start + t.bridge_chan <- []byte(zmq_signal_output) + return nil } func (t *TransportZmq) Read() <-chan interface{} { - return t.recv_chan + return t.recv_chan } func (t *TransportZmq) Shutdown() { - if t.ready { - // Send shutdown request - t.bridge_chan <- []byte(zmq_signal_shutdown) - t.wait.Wait() - t.dealer.Close() - t.monitor.Close() - t.context.Close() - t.ready = false - } + if t.ready { + // Send shutdown request + t.bridge_chan <- []byte(zmq_signal_shutdown) + t.wait.Wait() + t.dealer.Close() + t.monitor.Close() + t.context.Close() + t.ready = false + } } // Register the transport func init() { - core.RegisterTransport("plainzmq", NewZmqTransportFactory) + core.RegisterTransport("plainzmq", NewZmqTransportFactory) } diff --git a/src/lc-lib/transports/zmq3.go b/src/lc-lib/transports/zmq3.go index f131b07e..543bae9f 100644 --- a/src/lc-lib/transports/zmq3.go +++ b/src/lc-lib/transports/zmq3.go @@ -31,75 +31,75 @@ struct zmq_event_t_wrap { import "C" import ( - "fmt" - zmq "github.com/alecthomas/gozmq" - "syscall" - "unsafe" + "fmt" + zmq "github.com/alecthomas/gozmq" + "syscall" + "unsafe" ) func (f *TransportZmqFactory) processConfig(config_path string) (err error) { - return nil + return nil } func (t *TransportZmq) configureSocket() error { - return nil + return nil } // Process ZMQ 3.2.x monitor messages // http://api.zeromq.org/3-2:zmq-socket-monitor func (t *TransportZmq) processMonitorIn() (ok bool) { - for { - // Bring in the messages - RetryRecv: - data, err := t.monitor.Recv(zmq.DONTWAIT) - if err != nil { - switch err { - case syscall.EINTR: - // Try again - goto RetryRecv - case syscall.EAGAIN: - // No more messages - ok = true - return - } + for { + // Bring in the messages + RetryRecv: + data, err := t.monitor.Recv(zmq.DONTWAIT) + if err != nil { + switch err { + case syscall.EINTR: + // Try again + goto RetryRecv + case syscall.EAGAIN: + // No more messages + ok = true + return + } - // Failure - t.recv_chan <- fmt.Errorf("Monitor zmq.Socket.Recv failure %s", err) - return - } + // Failure + t.recv_chan <- fmt.Errorf("Monitor zmq.Socket.Recv failure %s", err) + return + } - switch t.event.part { - case Monitor_Part_Header: - event := (*C.struct_zmq_event_t_wrap)(unsafe.Pointer(&data[0])) - t.event.event = zmq.Event(event.event) - if event.addr == nil { - t.event.data = "" - } else { - // TODO: Fix this - data has been feed by zmq_msg_close! - //t.event.data = C.GoString(event.addr) - t.event.data = "" - } - t.event.val = int32(event.fd) - t.event.Log() - default: - log.Debug("Extraneous data in monitor message. Silently discarding.") - continue - } + switch t.event.part { + case Monitor_Part_Header: + event := (*C.struct_zmq_event_t_wrap)(unsafe.Pointer(&data[0])) + t.event.event = zmq.Event(event.event) + if event.addr == nil { + t.event.data = "" + } else { + // TODO: Fix this - data has been feed by zmq_msg_close! + //t.event.data = C.GoString(event.addr) + t.event.data = "" + } + t.event.val = int32(event.fd) + t.event.Log() + default: + log.Debug("Extraneous data in monitor message. Silently discarding.") + continue + } - more, err := t.monitor.RcvMore() - if err != nil { - // Failure - t.recv_chan <- fmt.Errorf("Monitor zmq.Socket.RcvMore failure %s", err) - return - } + more, err := t.monitor.RcvMore() + if err != nil { + // Failure + t.recv_chan <- fmt.Errorf("Monitor zmq.Socket.RcvMore failure %s", err) + return + } - if !more { - t.event.part = Monitor_Part_Header - continue - } + if !more { + t.event.part = Monitor_Part_Header + continue + } - if t.event.part <= Monitor_Part_Data { - t.event.part++ - } - } + if t.event.part <= Monitor_Part_Data { + t.event.part++ + } + } } diff --git a/src/lc-lib/transports/zmq4.go b/src/lc-lib/transports/zmq4.go index 4b783292..10010c08 100644 --- a/src/lc-lib/transports/zmq4.go +++ b/src/lc-lib/transports/zmq4.go @@ -26,132 +26,132 @@ package transports import "C" import ( - "encoding/binary" - "fmt" - zmq "github.com/alecthomas/gozmq" - "lc-lib/core" - "syscall" - "unsafe" + "encoding/binary" + "fmt" + zmq "github.com/alecthomas/gozmq" + "github.com/driskell/log-courier/src/lc-lib/core" + "syscall" + "unsafe" ) func (f *TransportZmqFactory) processConfig(config_path string) (err error) { - if len(f.CurveServerkey) == 0 { - return fmt.Errorf("Option %scurve server key is required", config_path) - } else if len(f.CurveServerkey) != 40 || !z85Validate(f.CurveServerkey) { - return fmt.Errorf("Option %scurve server key must be a valid 40 character Z85 encoded string", config_path) - } - if len(f.CurvePublickey) == 0 { - return fmt.Errorf("Option %scurve public key is required", config_path) - } else if len(f.CurvePublickey) != 40 || !z85Validate(f.CurvePublickey) { - return fmt.Errorf("Option %scurve public key must be a valid 40 character Z85 encoded string", config_path) - } - if len(f.CurveSecretkey) == 0 { - return fmt.Errorf("Option %scurve secret key is required", config_path) - } else if len(f.CurveSecretkey) != 40 || !z85Validate(f.CurveSecretkey) { - return fmt.Errorf("Option %scurve secret key must be a valid 40 character Z85 encoded string", config_path) - } - - return nil + if len(f.CurveServerkey) == 0 { + return fmt.Errorf("Option %scurve server key is required", config_path) + } else if len(f.CurveServerkey) != 40 || !z85Validate(f.CurveServerkey) { + return fmt.Errorf("Option %scurve server key must be a valid 40 character Z85 encoded string", config_path) + } + if len(f.CurvePublickey) == 0 { + return fmt.Errorf("Option %scurve public key is required", config_path) + } else if len(f.CurvePublickey) != 40 || !z85Validate(f.CurvePublickey) { + return fmt.Errorf("Option %scurve public key must be a valid 40 character Z85 encoded string", config_path) + } + if len(f.CurveSecretkey) == 0 { + return fmt.Errorf("Option %scurve secret key is required", config_path) + } else if len(f.CurveSecretkey) != 40 || !z85Validate(f.CurveSecretkey) { + return fmt.Errorf("Option %scurve secret key must be a valid 40 character Z85 encoded string", config_path) + } + + return nil } func (t *TransportZmq) configureSocket() (err error) { - if t.config.transport == "zmq" { - // Configure CurveMQ security - if err = t.dealer.SetCurveServerkey(t.config.CurveServerkey); err != nil { - return fmt.Errorf("Failed to set ZMQ curve server key: %s", err) - } - if err = t.dealer.SetCurvePublickey(t.config.CurvePublickey); err != nil { - return fmt.Errorf("Failed to set ZMQ curve public key: %s", err) - } - if err = t.dealer.SetCurveSecretkey(t.config.CurveSecretkey); err != nil { - return fmt.Errorf("Failed to set ZMQ curve secret key: %s", err) - } - } - return + if t.config.transport == "zmq" { + // Configure CurveMQ security + if err = t.dealer.SetCurveServerkey(t.config.CurveServerkey); err != nil { + return fmt.Errorf("Failed to set ZMQ curve server key: %s", err) + } + if err = t.dealer.SetCurvePublickey(t.config.CurvePublickey); err != nil { + return fmt.Errorf("Failed to set ZMQ curve public key: %s", err) + } + if err = t.dealer.SetCurveSecretkey(t.config.CurveSecretkey); err != nil { + return fmt.Errorf("Failed to set ZMQ curve secret key: %s", err) + } + } + return } // Process ZMQ 4.0.x monitor messages // http://api.zeromq.org/4-0:zmq-socket-monitor func (t *TransportZmq) processMonitorIn() (ok bool) { - for { - // Bring in the messages - RetryRecv: - data, err := t.monitor.Recv(zmq.DONTWAIT) - if err != nil { - switch err { - case syscall.EINTR: - // Try again - goto RetryRecv - case syscall.EAGAIN: - // No more messages - ok = true - return - } - - // Failure - t.recv_chan <- fmt.Errorf("Monitor zmq.Socket.Recv failure %s", err) - return - } - - switch t.event.part { - case Monitor_Part_Header: - t.event.event = zmq.Event(binary.LittleEndian.Uint16(data[0:2])) - t.event.val = int32(binary.LittleEndian.Uint32(data[2:6])) - t.event.data = "" - case Monitor_Part_Data: - t.event.data = string(data) - t.event.Log() - default: - log.Debug("Extraneous data in monitor message. Silently discarding.") - continue - } - - more, err := t.monitor.RcvMore() - if err != nil { - // Failure - t.recv_chan <- fmt.Errorf("Monitor zmq.Socket.RcvMore failure %s", err) - return - } - - if !more { - if t.event.part < Monitor_Part_Data { - t.event.Log() - log.Debug("Unexpected end of monitor message. Skipping.") - } - - t.event.part = Monitor_Part_Header - continue - } - - if t.event.part <= Monitor_Part_Data { - t.event.part++ - } - } + for { + // Bring in the messages + RetryRecv: + data, err := t.monitor.Recv(zmq.DONTWAIT) + if err != nil { + switch err { + case syscall.EINTR: + // Try again + goto RetryRecv + case syscall.EAGAIN: + // No more messages + ok = true + return + } + + // Failure + t.recv_chan <- fmt.Errorf("Monitor zmq.Socket.Recv failure %s", err) + return + } + + switch t.event.part { + case Monitor_Part_Header: + t.event.event = zmq.Event(binary.LittleEndian.Uint16(data[0:2])) + t.event.val = int32(binary.LittleEndian.Uint32(data[2:6])) + t.event.data = "" + case Monitor_Part_Data: + t.event.data = string(data) + t.event.Log() + default: + log.Debug("Extraneous data in monitor message. Silently discarding.") + continue + } + + more, err := t.monitor.RcvMore() + if err != nil { + // Failure + t.recv_chan <- fmt.Errorf("Monitor zmq.Socket.RcvMore failure %s", err) + return + } + + if !more { + if t.event.part < Monitor_Part_Data { + t.event.Log() + log.Debug("Unexpected end of monitor message. Skipping.") + } + + t.event.part = Monitor_Part_Header + continue + } + + if t.event.part <= Monitor_Part_Data { + t.event.part++ + } + } } func z85Validate(z85 string) bool { - var decoded []C.uint8_t + var decoded []C.uint8_t - if len(z85)%5 != 0 { - return false - } else { - // Avoid literal floats - decoded = make([]C.uint8_t, 8*len(z85)/10) - } + if len(z85)%5 != 0 { + return false + } else { + // Avoid literal floats + decoded = make([]C.uint8_t, 8*len(z85)/10) + } - // Grab a CString of the z85 we need to decode - c_z85 := C.CString(z85) - defer C.free(unsafe.Pointer(c_z85)) + // Grab a CString of the z85 we need to decode + c_z85 := C.CString(z85) + defer C.free(unsafe.Pointer(c_z85)) - // Because gozmq does not yet expose this for us, we have to expose it ourselves - if ret := C.zmq_z85_decode(&decoded[0], c_z85); ret == nil { - return false - } + // Because gozmq does not yet expose this for us, we have to expose it ourselves + if ret := C.zmq_z85_decode(&decoded[0], c_z85); ret == nil { + return false + } - return true + return true } // Register the transport func init() { - core.RegisterTransport("zmq", NewZmqTransportFactory) + core.RegisterTransport("zmq", NewZmqTransportFactory) } diff --git a/src/lc-tlscert/lc-tlscert.go b/src/lc-tlscert/lc-tlscert.go index 1ab52375..64da1659 100644 --- a/src/lc-tlscert/lc-tlscert.go +++ b/src/lc-tlscert/lc-tlscert.go @@ -22,183 +22,183 @@ package main import ( - "bufio" - "crypto/rand" - "crypto/rsa" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" - "fmt" - "math/big" - "net" - "os" - "strconv" - "time" + "bufio" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net" + "os" + "strconv" + "time" ) var input *bufio.Reader func init() { - input = bufio.NewReader(os.Stdin) + input = bufio.NewReader(os.Stdin) } func readString(prompt string) string { - fmt.Printf("%s: ", prompt) - - var line []byte - for { - data, prefix, _ := input.ReadLine() - line = append(line, data...) - if !prefix { - break - } - } - - return string(line) + fmt.Printf("%s: ", prompt) + + var line []byte + for { + data, prefix, _ := input.ReadLine() + line = append(line, data...) + if !prefix { + break + } + } + + return string(line) } func readNumber(prompt string) (num int64) { - var err error - for { - if num, err = strconv.ParseInt(readString(prompt), 0, 64); err != nil { - fmt.Println("Please enter a valid numerical value") - continue - } - break - } - return + var err error + for { + if num, err = strconv.ParseInt(readString(prompt), 0, 64); err != nil { + fmt.Println("Please enter a valid numerical value") + continue + } + break + } + return } func anyKey() { - input.ReadRune() + input.ReadRune() } func main() { - var err error - - template := x509.Certificate{ - Subject: pkix.Name{ - Organization: []string{"Log Courier"}, - }, - NotBefore: time.Now(), - - KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - BasicConstraintsValid: true, - - IsCA: true, - } - - fmt.Println("Specify the Common Name for the certificate. The common name") - fmt.Println("can be anything, but is usually set to the server's primary") - fmt.Println("DNS name. Even if you plan to connect via IP address you") - fmt.Println("should specify the DNS name here.") - fmt.Println() - - template.Subject.CommonName = readString("Common name") - fmt.Println() - - fmt.Println("The next step is to add any additional DNS names and IP") - fmt.Println("addresses that clients may use to connect to the server. If") - fmt.Println("you plan to connect to the server via IP address and not DNS") - fmt.Println("then you must specify those IP addresses here.") - fmt.Println("When you are finished, just press enter.") - fmt.Println() - - var cnt = 0 - var val string - for { - cnt++ - - if val = readString(fmt.Sprintf("DNS or IP address %d", cnt)); val == "" { - break - } - - if ip := net.ParseIP(val); ip != nil { - template.IPAddresses = append(template.IPAddresses, ip) - } else { - template.DNSNames = append(template.DNSNames, val) - } - } - - fmt.Println() - - fmt.Println("How long should the certificate be valid for? A year (365") - fmt.Println("days) is usual but requires the certificate to be regenerated") - fmt.Println("within a year or the certificate will cease working.") - fmt.Println() - - template.NotAfter = template.NotBefore.Add(time.Duration(readNumber("Number of days")) * time.Hour * 24) - - fmt.Println("Common name:", template.Subject.CommonName) - fmt.Println("DNS SANs:") - if len(template.DNSNames) == 0 { - fmt.Println(" None") - } else { - for _, e := range template.DNSNames { - fmt.Println(" ", e) - } - } - fmt.Println("IP SANs:") - if len(template.IPAddresses) == 0 { - fmt.Println(" None") - } else { - for _, e := range template.IPAddresses { - fmt.Println(" ", e) - } - } - fmt.Println() - - fmt.Println("The certificate can now be generated") - fmt.Println("Press any key to begin generating the self-signed certificate.") - anyKey() - - priv, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - fmt.Println("Failed to generate private key:", err) - os.Exit(1) - } - - serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) - template.SerialNumber, err = rand.Int(rand.Reader, serialNumberLimit) - if err != nil { - fmt.Println("Failed to generate serial number:", err) - os.Exit(1) - } - - derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) - if err != nil { - fmt.Println("Failed to create certificate:", err) - os.Exit(1) - } - - certOut, err := os.Create("selfsigned.crt") - if err != nil { - fmt.Println("Failed to open selfsigned.pem for writing:", err) - os.Exit(1) - } - pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) - certOut.Close() - - keyOut, err := os.OpenFile("selfsigned.key", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) - if err != nil { - fmt.Println("failed to open selfsigned.key for writing:", err) - os.Exit(1) - } - pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}) - keyOut.Close() - - fmt.Println("Successfully generated certificate") - fmt.Println(" Certificate: selfsigned.crt") - fmt.Println(" Private Key: selfsigned.key") - fmt.Println() - fmt.Println("Copy and paste the following into your Log Courier") - fmt.Println("configuration, adjusting paths as necessary:") - fmt.Println(" \"transport\": \"tls\",") - fmt.Println(" \"ssl ca\": \"path/to/selfsigned.crt\",") - fmt.Println() - fmt.Println("Copy and paste the following into your LogStash configuration, ") - fmt.Println("adjusting paths as necessary:") - fmt.Println(" ssl_certificate => \"path/to/selfsigned.crt\",") - fmt.Println(" ssl_key => \"path/to/selfsigned.key\",") + var err error + + template := x509.Certificate{ + Subject: pkix.Name{ + Organization: []string{"Log Courier"}, + }, + NotBefore: time.Now(), + + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + + IsCA: true, + } + + fmt.Println("Specify the Common Name for the certificate. The common name") + fmt.Println("can be anything, but is usually set to the server's primary") + fmt.Println("DNS name. Even if you plan to connect via IP address you") + fmt.Println("should specify the DNS name here.") + fmt.Println() + + template.Subject.CommonName = readString("Common name") + fmt.Println() + + fmt.Println("The next step is to add any additional DNS names and IP") + fmt.Println("addresses that clients may use to connect to the server. If") + fmt.Println("you plan to connect to the server via IP address and not DNS") + fmt.Println("then you must specify those IP addresses here.") + fmt.Println("When you are finished, just press enter.") + fmt.Println() + + var cnt = 0 + var val string + for { + cnt++ + + if val = readString(fmt.Sprintf("DNS or IP address %d", cnt)); val == "" { + break + } + + if ip := net.ParseIP(val); ip != nil { + template.IPAddresses = append(template.IPAddresses, ip) + } else { + template.DNSNames = append(template.DNSNames, val) + } + } + + fmt.Println() + + fmt.Println("How long should the certificate be valid for? A year (365") + fmt.Println("days) is usual but requires the certificate to be regenerated") + fmt.Println("within a year or the certificate will cease working.") + fmt.Println() + + template.NotAfter = template.NotBefore.Add(time.Duration(readNumber("Number of days")) * time.Hour * 24) + + fmt.Println("Common name:", template.Subject.CommonName) + fmt.Println("DNS SANs:") + if len(template.DNSNames) == 0 { + fmt.Println(" None") + } else { + for _, e := range template.DNSNames { + fmt.Println(" ", e) + } + } + fmt.Println("IP SANs:") + if len(template.IPAddresses) == 0 { + fmt.Println(" None") + } else { + for _, e := range template.IPAddresses { + fmt.Println(" ", e) + } + } + fmt.Println() + + fmt.Println("The certificate can now be generated") + fmt.Println("Press any key to begin generating the self-signed certificate.") + anyKey() + + priv, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + fmt.Println("Failed to generate private key:", err) + os.Exit(1) + } + + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + template.SerialNumber, err = rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + fmt.Println("Failed to generate serial number:", err) + os.Exit(1) + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + if err != nil { + fmt.Println("Failed to create certificate:", err) + os.Exit(1) + } + + certOut, err := os.Create("selfsigned.crt") + if err != nil { + fmt.Println("Failed to open selfsigned.pem for writing:", err) + os.Exit(1) + } + pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + certOut.Close() + + keyOut, err := os.OpenFile("selfsigned.key", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + fmt.Println("failed to open selfsigned.key for writing:", err) + os.Exit(1) + } + pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}) + keyOut.Close() + + fmt.Println("Successfully generated certificate") + fmt.Println(" Certificate: selfsigned.crt") + fmt.Println(" Private Key: selfsigned.key") + fmt.Println() + fmt.Println("Copy and paste the following into your Log Courier") + fmt.Println("configuration, adjusting paths as necessary:") + fmt.Println(" \"transport\": \"tls\",") + fmt.Println(" \"ssl ca\": \"path/to/selfsigned.crt\",") + fmt.Println() + fmt.Println("Copy and paste the following into your LogStash configuration, ") + fmt.Println("adjusting paths as necessary:") + fmt.Println(" ssl_certificate => \"path/to/selfsigned.crt\",") + fmt.Println(" ssl_key => \"path/to/selfsigned.key\",") } diff --git a/src/log-courier/log-courier.go b/src/log-courier/log-courier.go index c600b497..1a6b3d85 100644 --- a/src/log-courier/log-courier.go +++ b/src/log-courier/log-courier.go @@ -20,267 +20,315 @@ package main import ( - "flag" - "fmt" - "github.com/op/go-logging" - "lc-lib/admin" - "lc-lib/core" - "lc-lib/prospector" - "lc-lib/spooler" - "lc-lib/publisher" - "lc-lib/registrar" - stdlog "log" - "os" - "runtime/pprof" - "time" + "flag" + "fmt" + "github.com/driskell/log-courier/src/lc-lib/admin" + "github.com/driskell/log-courier/src/lc-lib/core" + "github.com/driskell/log-courier/src/lc-lib/harvester" + "github.com/driskell/log-courier/src/lc-lib/prospector" + "github.com/driskell/log-courier/src/lc-lib/publisher" + "github.com/driskell/log-courier/src/lc-lib/registrar" + "github.com/driskell/log-courier/src/lc-lib/spooler" + "github.com/op/go-logging" + stdlog "log" + "os" + "runtime/pprof" + "time" ) -import _ "lc-lib/codecs" -import _ "lc-lib/transports" +import _ "github.com/driskell/log-courier/src/lc-lib/codecs" +import _ "github.com/driskell/log-courier/src/lc-lib/transports" func main() { - logcourier := NewLogCourier() - logcourier.Run() + logcourier := NewLogCourier() + logcourier.Run() } type LogCourier struct { - pipeline *core.Pipeline - config *core.Config - shutdown_chan chan os.Signal - reload_chan chan os.Signal - config_file string - from_beginning bool - log_file *os.File - last_snapshot time.Time - snapshot *core.Snapshot + pipeline *core.Pipeline + config *core.Config + shutdown_chan chan os.Signal + reload_chan chan os.Signal + config_file string + stdin bool + from_beginning bool + harvester *harvester.Harvester + log_file *DefaultLogBackend + last_snapshot time.Time + snapshot *core.Snapshot } func NewLogCourier() *LogCourier { - ret := &LogCourier{ - pipeline: core.NewPipeline(), - } - return ret + ret := &LogCourier{ + pipeline: core.NewPipeline(), + } + return ret } func (lc *LogCourier) Run() { - var admin_listener *admin.Listener - var on_command <-chan string + var admin_listener *admin.Listener + var on_command <-chan string + var harvester_wait <-chan *harvester.HarvesterFinish + var registrar_imp registrar.Registrator - lc.startUp() + lc.startUp() - log.Info("Log Courier version %s pipeline starting", core.Log_Courier_Version) + log.Info("Log Courier version %s pipeline starting", core.Log_Courier_Version) - if lc.config.General.AdminEnabled { - var err error + // If reading from stdin, skip admin, and set up a null registrar + if lc.stdin { + registrar_imp = newStdinRegistrar(lc.pipeline) + } else { + if lc.config.General.AdminEnabled { + var err error - admin_listener, err = admin.NewListener(lc.pipeline, &lc.config.General) - if err != nil { - log.Fatalf("Failed to initialise: %s", err) - } + admin_listener, err = admin.NewListener(lc.pipeline, &lc.config.General) + if err != nil { + log.Fatalf("Failed to initialise: %s", err) + } - on_command = admin_listener.OnCommand() - } + on_command = admin_listener.OnCommand() + } - registrar := registrar.NewRegistrar(lc.pipeline, lc.config.General.PersistDir) + registrar_imp = registrar.NewRegistrar(lc.pipeline, lc.config.General.PersistDir) + } - publisher, err := publisher.NewPublisher(lc.pipeline, &lc.config.Network, registrar) - if err != nil { - log.Fatalf("Failed to initialise: %s", err) - } + publisher_imp, err := publisher.NewPublisher(lc.pipeline, &lc.config.Network, registrar_imp) + if err != nil { + log.Fatalf("Failed to initialise: %s", err) + } - spooler := spooler.NewSpooler(lc.pipeline, &lc.config.General, publisher) + spooler_imp := spooler.NewSpooler(lc.pipeline, &lc.config.General, publisher_imp) - if _, err := prospector.NewProspector(lc.pipeline, lc.config, lc.from_beginning, registrar, spooler); err != nil { - log.Fatalf("Failed to initialise: %s", err) - } + // If reading from stdin, don't start prospector, directly start a harvester + if lc.stdin { + lc.harvester = harvester.NewHarvester(nil, lc.config, &lc.config.Stdin, 0) + lc.harvester.Start(spooler_imp.Connect()) + harvester_wait = lc.harvester.OnFinish() + } else { + if _, err := prospector.NewProspector(lc.pipeline, lc.config, lc.from_beginning, registrar_imp, spooler_imp); err != nil { + log.Fatalf("Failed to initialise: %s", err) + } + } - // Start the pipeline - lc.pipeline.Start() + // Start the pipeline + lc.pipeline.Start() - log.Notice("Pipeline ready") + log.Notice("Pipeline ready") - lc.shutdown_chan = make(chan os.Signal, 1) - lc.reload_chan = make(chan os.Signal, 1) - lc.registerSignals() + lc.shutdown_chan = make(chan os.Signal, 1) + lc.reload_chan = make(chan os.Signal, 1) + lc.registerSignals() SignalLoop: - for { - select { - case <-lc.shutdown_chan: - lc.cleanShutdown() - break SignalLoop - case <-lc.reload_chan: - lc.reloadConfig() - case command := <-on_command: - admin_listener.Respond(lc.processCommand(command)) - } - } - - log.Notice("Exiting") - - if lc.log_file != nil { - lc.log_file.Close() - } + for { + select { + case <-lc.shutdown_chan: + lc.cleanShutdown() + break SignalLoop + case <-lc.reload_chan: + lc.reloadConfig() + case command := <-on_command: + admin_listener.Respond(lc.processCommand(command)) + case finished := <-harvester_wait: + if finished.Error != nil { + log.Notice("An error occurred reading from stdin at offset %d: %s", finished.Last_Read_Offset, finished.Error) + } else { + log.Notice("Finished reading from stdin at offset %d", finished.Last_Read_Offset) + } + lc.harvester = nil + + // Flush the spooler + spooler_imp.Flush() + + // Wait for StdinRegistrar to receive ACK for the last event we sent + registrar_imp.(*StdinRegistrar).Wait(finished.Last_Event_Offset) + + lc.cleanShutdown() + break SignalLoop + } + } + + log.Notice("Exiting") + + if lc.log_file != nil { + lc.log_file.Close() + } } func (lc *LogCourier) startUp() { - var version bool - var config_test bool - var list_supported bool - var cpu_profile string - - flag.BoolVar(&version, "version", false, "show version information") - flag.BoolVar(&config_test, "config-test", false, "Test the configuration specified by -config and exit") - flag.BoolVar(&list_supported, "list-supported", false, "List supported transports and codecs") - flag.StringVar(&cpu_profile, "cpuprofile", "", "write cpu profile to file") - - flag.StringVar(&lc.config_file, "config", "", "The config file to load") - flag.BoolVar(&lc.from_beginning, "from-beginning", false, "On first run, read new files from the beginning instead of the end") - - flag.Parse() - - if version { - fmt.Printf("Log Courier version %s\n", core.Log_Courier_Version) - os.Exit(0) - } - - if list_supported { - fmt.Printf("Available transports:\n") - for _, transport := range core.AvailableTransports() { - fmt.Printf(" %s\n", transport) - } - - fmt.Printf("Available codecs:\n") - for _, codec := range core.AvailableCodecs() { - fmt.Printf(" %s\n", codec) - } - os.Exit(0) - } - - if lc.config_file == "" { - fmt.Fprintf(os.Stderr, "Please specify a configuration file with -config.\n\n") - flag.PrintDefaults() - os.Exit(1) - } - - err := lc.loadConfig() - - if config_test { - if err == nil { - fmt.Printf("Configuration OK\n") - os.Exit(0) - } - fmt.Printf("Configuration test failed: %s\n", err) - os.Exit(1) - } - - if err != nil { - fmt.Printf("Configuration error: %s\n", err) - os.Exit(1) - } - - if err = lc.configureLogging(); err != nil { - fmt.Printf("Failed to initialise logging: %s", err) - os.Exit(1) - } - - if cpu_profile != "" { - log.Notice("Starting CPU profiler") - f, err := os.Create(cpu_profile) - if err != nil { - log.Fatal(err) - } - pprof.StartCPUProfile(f) - go func() { - time.Sleep(60 * time.Second) - pprof.StopCPUProfile() - log.Panic("CPU profile completed") - }() - } + var version bool + var config_test bool + var list_supported bool + var cpu_profile string + + flag.BoolVar(&version, "version", false, "show version information") + flag.BoolVar(&config_test, "config-test", false, "Test the configuration specified by -config and exit") + flag.BoolVar(&list_supported, "list-supported", false, "List supported transports and codecs") + flag.StringVar(&cpu_profile, "cpuprofile", "", "write cpu profile to file") + + flag.StringVar(&lc.config_file, "config", "", "The config file to load") + flag.BoolVar(&lc.stdin, "stdin", false, "Read from stdin instead of files listed in the config file") + flag.BoolVar(&lc.from_beginning, "from-beginning", false, "On first run, read new files from the beginning instead of the end") + + flag.Parse() + + if version { + fmt.Printf("Log Courier version %s\n", core.Log_Courier_Version) + os.Exit(0) + } + + if list_supported { + fmt.Printf("Available transports:\n") + for _, transport := range core.AvailableTransports() { + fmt.Printf(" %s\n", transport) + } + + fmt.Printf("Available codecs:\n") + for _, codec := range core.AvailableCodecs() { + fmt.Printf(" %s\n", codec) + } + os.Exit(0) + } + + if lc.config_file == "" { + fmt.Fprintf(os.Stderr, "Please specify a configuration file with -config.\n\n") + flag.PrintDefaults() + os.Exit(1) + } + + err := lc.loadConfig() + + if config_test { + if err == nil { + fmt.Printf("Configuration OK\n") + os.Exit(0) + } + fmt.Printf("Configuration test failed: %s\n", err) + os.Exit(1) + } + + if err != nil { + fmt.Printf("Configuration error: %s\n", err) + os.Exit(1) + } + + if err = lc.configureLogging(); err != nil { + fmt.Printf("Failed to initialise logging: %s", err) + os.Exit(1) + } + + if cpu_profile != "" { + log.Notice("Starting CPU profiler") + f, err := os.Create(cpu_profile) + if err != nil { + log.Fatal(err) + } + pprof.StartCPUProfile(f) + go func() { + time.Sleep(60 * time.Second) + pprof.StopCPUProfile() + log.Panic("CPU profile completed") + }() + } } func (lc *LogCourier) configureLogging() (err error) { - backends := make([]logging.Backend, 0, 1) + backends := make([]logging.Backend, 0, 1) - // First, the stdout backend - if lc.config.General.LogStdout { - backends = append(backends, logging.NewLogBackend(os.Stdout, "", stdlog.LstdFlags|stdlog.Lmicroseconds)) - } + // First, the stdout backend + if lc.config.General.LogStdout { + backends = append(backends, logging.NewLogBackend(os.Stdout, "", stdlog.LstdFlags|stdlog.Lmicroseconds)) + } - // Log file? - if lc.config.General.LogFile != "" { - lc.log_file, err = os.OpenFile(lc.config.General.LogFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0640) - if err != nil { - return - } + // Log file? + if lc.config.General.LogFile != "" { + lc.log_file, err = NewDefaultLogBackend(lc.config.General.LogFile, "", stdlog.LstdFlags|stdlog.Lmicroseconds) + if err != nil { + return + } - backends = append(backends, logging.NewLogBackend(lc.log_file, "", stdlog.LstdFlags|stdlog.Lmicroseconds)) - } + backends = append(backends, lc.log_file) + } - if err = lc.configureLoggingPlatform(&backends); err != nil { - return - } + if err = lc.configureLoggingPlatform(&backends); err != nil { + return + } - // Set backends BEFORE log level (or we reset log level) - logging.SetBackend(backends...) + // Set backends BEFORE log level (or we reset log level) + logging.SetBackend(backends...) - // Set the logging level - logging.SetLevel(lc.config.General.LogLevel, "") + // Set the logging level + logging.SetLevel(lc.config.General.LogLevel, "") - return nil + return nil } func (lc *LogCourier) loadConfig() error { - lc.config = core.NewConfig() - if err := lc.config.Load(lc.config_file); err != nil { - return err - } - - if len(lc.config.Files) == 0 { - return fmt.Errorf("No file groups were found in the configuration.") - } - - return nil + lc.config = core.NewConfig() + if err := lc.config.Load(lc.config_file); err != nil { + return err + } + + if lc.stdin { + // TODO: Where to find stdin config for codec and fields? + } else if len(lc.config.Files) == 0 { + log.Warning("No file groups were found in the configuration.") + } + + return nil } func (lc *LogCourier) reloadConfig() error { - if err := lc.loadConfig(); err != nil { - return err - } + if err := lc.loadConfig(); err != nil { + return err + } + + log.Notice("Configuration reload successful") - log.Notice("Configuration reload successful") + // Update the log level + logging.SetLevel(lc.config.General.LogLevel, "") - // Update the log level - logging.SetLevel(lc.config.General.LogLevel, "") + // Reopen the log file if we specified one + if lc.log_file != nil { + lc.log_file.Reopen() + log.Notice("Log file reopened") + } - // Pass the new config to the pipeline workers - lc.pipeline.SendConfig(lc.config) + // Pass the new config to the pipeline workers + lc.pipeline.SendConfig(lc.config) - return nil + return nil } func (lc *LogCourier) processCommand(command string) *admin.Response { - switch command { - case "RELD": - if err := lc.reloadConfig(); err != nil { - return &admin.Response{&admin.ErrorResponse{Message: fmt.Sprintf("Configuration error, reload unsuccessful: %s", err.Error())}} - } - return &admin.Response{&admin.ReloadResponse{}} - case "SNAP": - if lc.snapshot == nil || time.Since(lc.last_snapshot) >= time.Second { - lc.snapshot = lc.pipeline.Snapshot() - lc.snapshot.Sort() - lc.last_snapshot = time.Now() - } - return &admin.Response{lc.snapshot} - } - - - return &admin.Response{&admin.ErrorResponse{Message: "Unknown command"}} + switch command { + case "RELD": + if err := lc.reloadConfig(); err != nil { + return &admin.Response{&admin.ErrorResponse{Message: fmt.Sprintf("Configuration error, reload unsuccessful: %s", err.Error())}} + } + return &admin.Response{&admin.ReloadResponse{}} + case "SNAP": + if lc.snapshot == nil || time.Since(lc.last_snapshot) >= time.Second { + lc.snapshot = lc.pipeline.Snapshot() + lc.snapshot.Sort() + lc.last_snapshot = time.Now() + } + return &admin.Response{lc.snapshot} + } + + return &admin.Response{&admin.ErrorResponse{Message: "Unknown command"}} } func (lc *LogCourier) cleanShutdown() { - log.Notice("Initiating shutdown") - lc.pipeline.Shutdown() - lc.pipeline.Wait() + log.Notice("Initiating shutdown") + + if lc.harvester != nil { + lc.harvester.Stop() + finished := <-lc.harvester.OnFinish() + log.Notice("Aborted reading from stdin at offset %d", finished.Last_Read_Offset) + } + + lc.pipeline.Shutdown() + lc.pipeline.Wait() } diff --git a/src/log-courier/log-courier_nonwindows.go b/src/log-courier/log-courier_nonwindows.go index 7bc5ab01..99da751f 100644 --- a/src/log-courier/log-courier_nonwindows.go +++ b/src/log-courier/log-courier_nonwindows.go @@ -19,46 +19,46 @@ package main import ( - "fmt" - "github.com/op/go-logging" - "os" - "os/signal" - "syscall" - "unsafe" + "fmt" + "github.com/op/go-logging" + "os" + "os/signal" + "syscall" + "unsafe" ) func (lc *LogCourier) registerSignals() { - // *nix systems support SIGTERM so handle shutdown on that too - signal.Notify(lc.shutdown_chan, os.Interrupt, syscall.SIGTERM) + // *nix systems support SIGTERM so handle shutdown on that too + signal.Notify(lc.shutdown_chan, os.Interrupt, syscall.SIGTERM) - // *nix has SIGHUP for reload - signal.Notify(lc.reload_chan, syscall.SIGHUP) + // *nix has SIGHUP for reload + signal.Notify(lc.reload_chan, syscall.SIGHUP) } func (lc *LogCourier) configureLoggingPlatform(backends *[]logging.Backend) error { - // Make it color if it's a TTY - // TODO: This could be prone to problems when updating logging in future - if lc.isatty(os.Stdout) && lc.config.General.LogStdout { - (*backends)[0].(*logging.LogBackend).Color = true - } + // Make it color if it's a TTY + // TODO: This could be prone to problems when updating logging in future + if lc.isatty(os.Stdout) && lc.config.General.LogStdout { + (*backends)[0].(*logging.LogBackend).Color = true + } - if lc.config.General.LogSyslog { - syslog_backend, err := logging.NewSyslogBackend("log-courier") - if err != nil { - return fmt.Errorf("Failed to open syslog: %s", err) - } - new_backends := append(*backends, syslog_backend) - *backends = new_backends - } + if lc.config.General.LogSyslog { + syslog_backend, err := logging.NewSyslogBackend("log-courier") + if err != nil { + return fmt.Errorf("Failed to open syslog: %s", err) + } + new_backends := append(*backends, syslog_backend) + *backends = new_backends + } - return nil + return nil } func (lc *LogCourier) isatty(f *os.File) bool { - var pgrp int64 - // Most real isatty implementations use TIOCGETA - // However, TIOCGPRGP is easier than TIOCGETA as it only requires an int and not a termios struct - // There is a possibility it may not have the exact same effect - but seems fine to me - _, _, err := syscall.Syscall(syscall.SYS_IOCTL, f.Fd(), syscall.TIOCGPGRP, uintptr(unsafe.Pointer(&pgrp))) - return err == 0 + var pgrp int64 + // Most real isatty implementations use TIOCGETA + // However, TIOCGPRGP is easier than TIOCGETA as it only requires an int and not a termios struct + // There is a possibility it may not have the exact same effect - but seems fine to me + _, _, err := syscall.Syscall(syscall.SYS_IOCTL, f.Fd(), syscall.TIOCGPGRP, uintptr(unsafe.Pointer(&pgrp))) + return err == 0 } diff --git a/src/log-courier/log-courier_windows.go b/src/log-courier/log-courier_windows.go index 570e2c31..58f40622 100644 --- a/src/log-courier/log-courier_windows.go +++ b/src/log-courier/log-courier_windows.go @@ -19,18 +19,18 @@ package main import ( - "github.com/op/go-logging" - "os" - "os/signal" + "github.com/op/go-logging" + "os" + "os/signal" ) func (lc *LogCourier) registerSignals() { - // Windows onyl supports os.Interrupt - signal.Notify(lc.shutdown_chan, os.Interrupt) + // Windows onyl supports os.Interrupt + signal.Notify(lc.shutdown_chan, os.Interrupt) - // No reload signal for Windows - implementation will have to wait + // No reload signal for Windows - implementation will have to wait } func (lc *LogCourier) configureLoggingPlatform(backends *[]logging.Backend) error { - return nil + return nil } diff --git a/src/log-courier/logging.go b/src/log-courier/logging.go index 2142e3ba..9c4eec91 100644 --- a/src/log-courier/logging.go +++ b/src/log-courier/logging.go @@ -12,14 +12,76 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. -*/ + */ package main -import "github.com/op/go-logging" +import ( + "github.com/op/go-logging" + "io/ioutil" + golog "log" + "os" +) var log *logging.Logger func init() { log = logging.MustGetLogger("log-courier") } + +type DefaultLogBackend struct { + file *os.File + path string +} + +func NewDefaultLogBackend(path string, prefix string, flag int) (*DefaultLogBackend, error) { + ret := &DefaultLogBackend{ + path: path, + } + + golog.SetPrefix(prefix) + golog.SetFlags(flag) + + err := ret.Reopen() + if err != nil { + return nil, err + } + + return ret, nil +} + +func (f *DefaultLogBackend) Log(level logging.Level, calldepth int, rec *logging.Record) error { + golog.Print(rec.Formatted(calldepth + 1)) + return nil +} + +func (f *DefaultLogBackend) Reopen() (err error) { + var new_file *os.File + + new_file, err = os.OpenFile(f.path, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0640) + if err != nil { + return + } + + // Switch to new output before closing + golog.SetOutput(new_file) + + if f.file != nil { + f.file.Close() + } + + f.file = new_file + + return nil +} + +func (f *DefaultLogBackend) Close() { + // Discard logs before closing + golog.SetOutput(ioutil.Discard) + + if f.file != nil { + f.file.Close() + } + + f.file = nil +} diff --git a/src/log-courier/stdin_registrar.go b/src/log-courier/stdin_registrar.go new file mode 100644 index 00000000..cddfc71a --- /dev/null +++ b/src/log-courier/stdin_registrar.go @@ -0,0 +1,155 @@ +/* + * Copyright 2014 Jason Woods. + * + * This file is a modification of code from Logstash Forwarder. + * Copyright 2012-2013 Jordan Sissel and contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "github.com/driskell/log-courier/src/lc-lib/core" + "github.com/driskell/log-courier/src/lc-lib/registrar" + "sync" +) + +type StdinRegistrar struct { + core.PipelineSegment + + sync.Mutex + + group sync.WaitGroup + registrar_chan chan []registrar.EventProcessor + signal_chan chan int64 + references int + wait_offset *int64 + last_offset int64 +} + +func newStdinRegistrar(pipeline *core.Pipeline) *StdinRegistrar { + ret := &StdinRegistrar{ + registrar_chan: make(chan []registrar.EventProcessor, 16), + signal_chan: make(chan int64, 1), + } + + ret.group.Add(1) + + pipeline.Register(ret) + + return ret +} + +func (r *StdinRegistrar) Run() { + defer func() { + r.Done() + r.group.Done() + }() + + state := make(map[core.Stream]*registrar.FileState) + state[nil] = ®istrar.FileState{} + +RegistrarLoop: + for { + select { + case signal := <-r.signal_chan: + r.wait_offset = new(int64) + *r.wait_offset = signal + + if r.last_offset == signal { + break RegistrarLoop + } + + log.Debug("Stdin registrar received stdin EOF offset of %d", *r.wait_offset) + case events := <-r.registrar_chan: + for _, event := range events { + event.Process(state) + } + + r.last_offset = state[nil].Offset + + if r.wait_offset != nil && state[nil].Offset >= *r.wait_offset { + log.Debug("Stdin registrar has reached end of stdin") + break RegistrarLoop + } + case <-r.OnShutdown(): + break RegistrarLoop + } + } + + log.Info("Stdin registrar exiting") +} + +func (r *StdinRegistrar) Connect() registrar.EventSpooler { + r.Lock() + defer r.Unlock() + r.references++ + return newStdinEventSpool(r) +} + +func (r *StdinRegistrar) Wait(offset int64) { + r.signal_chan <- offset + r.group.Wait() +} + +func (r *StdinRegistrar) LoadPrevious(registrar.LoadPreviousFunc) (bool, error) { + return false, nil +} + +func (r *StdinRegistrar) dereferenceSpooler() { + r.Lock() + defer r.Unlock() + r.references-- + if r.references == 0 { + close(r.registrar_chan) + } +} + +type StdinEventSpool struct { + registrar *StdinRegistrar + events []registrar.EventProcessor +} + +func newStdinEventSpool(r *StdinRegistrar) *StdinEventSpool { + ret := &StdinEventSpool{ + registrar: r, + } + ret.reset() + return ret +} + +func (r *StdinEventSpool) Close() { + r.registrar.dereferenceSpooler() + r.registrar = nil +} + +func (r *StdinEventSpool) Add(event registrar.EventProcessor) { + // StdinEventSpool is only interested in AckEvents + if _, ok := event.(*registrar.AckEvent); !ok { + return + } + + r.events = append(r.events, event) +} + +func (r *StdinEventSpool) Send() { + if len(r.events) != 0 { + r.registrar.registrar_chan <- r.events + r.reset() + } +} + +func (r *StdinEventSpool) reset() { + r.events = make([]registrar.EventProcessor, 0, 0) +} diff --git a/src/log-courier/stdin_registrar_test.go b/src/log-courier/stdin_registrar_test.go new file mode 100644 index 00000000..a3aa0e45 --- /dev/null +++ b/src/log-courier/stdin_registrar_test.go @@ -0,0 +1,78 @@ +/* + * Copyright 2014 Jason Woods. + * + * This file is a modification of code from Logstash Forwarder. + * Copyright 2012-2013 Jordan Sissel and contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "github.com/driskell/log-courier/src/lc-lib/core" + "github.com/driskell/log-courier/src/lc-lib/registrar" + "testing" + "time" +) + +func newTestStdinRegistrar() (*core.Pipeline, *StdinRegistrar) { + pipeline := core.NewPipeline() + return pipeline, newStdinRegistrar(pipeline) +} + +func newEventSpool(offset int64) []*core.EventDescriptor { + // Prepare an event spool with single event of specified offset + return []*core.EventDescriptor{ + &core.EventDescriptor{ + Stream: nil, + Offset: offset, + Event: []byte{}, + }, + } +} + +func TestStdinRegistrarWait(t *testing.T) { + p, r := newTestStdinRegistrar() + + // Start the stdin registrar + go func() { + r.Run() + }() + + c := r.Connect() + c.Add(registrar.NewAckEvent(newEventSpool(13))) + c.Send() + + r.Wait(13) + + wait := make(chan int) + go func() { + p.Wait() + wait <- 1 + }() + + select { + case <-wait: + break + case <-time.After(5 * time.Second): + t.Error("Timeout waiting for stdin registrar shutdown") + return + } + + if r.last_offset != 13 { + t.Error("Last offset was incorrect: ", r.last_offset) + } else if r.wait_offset == nil || *r.wait_offset != 13 { + t.Error("Wait offset was incorrect: ", r.wait_offset) + } +} diff --git a/version_short.txt b/version_short.txt new file mode 100644 index 00000000..c239c60c --- /dev/null +++ b/version_short.txt @@ -0,0 +1 @@ +1.5