Merge branch 'develop' into stable

Gemfile

@@ -1,15 +1,16 @@
 source 'https://rubygems.org'
 
 # Log Courier gem dependencies
-gem 'ffi-rzmq'
-gem 'multi_json'
+gem 'cabin',      '~> 0.6'
+gem 'ffi-rzmq',   '~> 2.0'
+gem 'multi_json', '~> 1.10'
 
 # Log Courier gem JSON parsers
-gem 'oj',        :platforms => :mri
-gem 'jrjackson', :platforms => :jruby
+gem 'oj',        '~> 2.11', :platforms => :mri
+gem 'jrjackson', '~> 0.2',  :platforms => :jruby
 
 # Profiler for MRI
-gem 'ruby-prof', :platforms => :mri
+gem 'ruby-prof', '~> 0.15', :platforms => :mri
 
 # Tests
-gem 'rspec'
+gem 'rspec',     '~> 3.1'

Makefile

@@ -1,4 +1,4 @@
-.PHONY: prepare fix_version all log-courier gem gem_plugins push_gems test doc profile benchmark jrprofile jrbenchmark clean
+.PHONY: prepare fix_version all log-courier gem gem_plugins push_gems test test_go test_rspec doc profile benchmark jrprofile jrbenchmark clean
 
 MAKEFILE := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST))
 GOPATH := $(patsubst %/,%,$(dir $(abspath $(MAKEFILE))))
@@ -43,14 +43,18 @@ gem_plugins: | fix_version
 	gem build logstash-input-log-courier.gemspec
 	gem build logstash-output-log-courier.gemspec
 
-push_gems: | gem gem_plugins fix_version vendor/bundle/.GemfileModT
+push_gems: | gem gem_plugins
 	build/push_gems
 
-test: | all vendor/bundle/.GemfileModT
+test_go: | all
 	go get -d -tags "$(TAGS)" $(GOTESTS)
 	go test -tags "$(TAGS)" $(GOTESTS)
+
+test_rspec: | all vendor/bundle/.GemfileModT
 	bundle exec rspec $(TESTS)
 
+test: | test_go test_rspec
+
 selfsigned: | bin/lc-tlscert
 	bin/lc-tlscert
 

build/fix_version

@@ -11,6 +11,7 @@ else
 	# Describe version from Git, and ensure the only "-xxx" is the git revision
 	# This ensures that gem builds only add one ".pre" tag automatically
 	VERSION="$(git describe | sed 's/-\([0-9][0-9]*\)-\([0-9a-z][0-9a-z]*\)$/.\1.\2/g')"
+	VERSION="${VERSION#v}"
 fi
 
 # Patch version.go
@@ -19,13 +20,13 @@ sed "s/\\(const  *Log_Courier_Version  *string  *=  *\"\\)[^\"]*\\(\"\\)/\\1${VE
 
 # Patch the gemspecs
 for GEM in log-courier logstash-input-log-courier logstash-output-log-courier; do
-	sed "s/\\(gem.version *= *'\\)[^']*\\('\\)/\\1${VERSION#v}\\2/g" ${GEM}.gemspec > ${GEM}.gemspec.tmp
+	sed "s/\\(gem.version *= *'\\)[^']*\\('\\)/\\1${VERSION}\\2/g" ${GEM}.gemspec > ${GEM}.gemspec.tmp
 	\mv -f ${GEM}.gemspec.tmp ${GEM}.gemspec
 	[ ${GEM#logstash-} != $GEM ] && {
-		sed "s/\\(gem.add_runtime_dependency  *'log-courier' *,  *'= *\\)[^']*\\('\\)/\\1${VERSION#v}\\2/g" ${GEM}.gemspec > ${GEM}.gemspec.tmp
+		sed "s/\\(gem.add_runtime_dependency  *'log-courier' *,  *'= *\\)[^']*\\('\\)/\\1${VERSION}\\2/g" ${GEM}.gemspec > ${GEM}.gemspec.tmp
 		\mv -f ${GEM}.gemspec.tmp ${GEM}.gemspec
 	}
 done
 
-echo "${VERSION#v}" > version.txt
+echo "${VERSION}" > version.txt
 echo "Set Log Courier Version ${VERSION}"

contrib/initscripts/log-courier.sysconfig

@@ -0,0 +1,3 @@
+# log-courier config file location
+CONFIG_FILE="/etc/log-courier/log-courier.conf"
+

contrib/log-courier.init → contrib/initscripts/redhat-sysv.init

@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# log-courier Log Courier
+# log-courier Log Courier, a lightweight log shipper with Logstash integration.
 #
 # chkconfig: 2345 90 10
 # description: Controls the Log Courier daemon
@@ -11,17 +11,26 @@
 # Required-Stop:     $local_fs $remote_fs $syslog
 # Default-Start:     2 3 4 5
 # Default-Stop:      0 1 6
-# Short-Description: Log Courier
+# Short-Description: Log Courier, a lightweight log shipper with Logstash integration.
 ### END INIT INFO
 
 # source function library
 . /etc/rc.d/init.d/functions
 
 DAEMON='/usr/sbin/log-courier'
 DATA_DIR='/var/lib/log-courier'
-CONFIG_FILE='/etc/log-courier/log-courier.conf'
 PID_FILE='/var/run/log-courier.pid'
 
+# check if file exists
+if [ -f /etc/sysconfig/log-courier ]; then
+	. /etc/sysconfig/log-courier
+fi
+
+# set default config file, if $CONFIG_FILE is not set
+if [ -z ${CONFIG_FILE} ]; then
+	CONFIG_FILE='/etc/log-courier/log-courier.conf'
+fi
+
 do_start() {
 	echo -n "Starting Log Courier: "
 	status -p $PID_FILE $DAEMON &>/dev/null
@@ -74,6 +83,14 @@ case "$1" in
 			RC=$?
 		fi
 	;;
+	condrestart|try-restart)
+		status -p $PID_FILE $DAEMON
+		RC=$?
+		if [ $RC -eq 0 ]; then
+			$0 restart
+			RC=$?
+		fi
+	;;
 	configtest)
 		echo -n "Configuration test: "
 		TESTRESULT=$( ${DAEMON} -config="${CONFIG_FILE}" -config-test )
@@ -88,7 +105,7 @@ case "$1" in
 		fi
 	;;
 	*)
-		echo "Usage: $0 start|stop|status|reload|restart|configtest"
+		echo "Usage: $0 start|stop|status|reload|restart|condrestart|try-restart|configtest"
 		exit 1
 	;;
 esac

contrib/initscripts/systemd.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Log Courier, a lightweight log shipper with Logstash integration.
+After=syslog.target network.target remote-fs.target nss-lookup.target
+
+[Service]
+Type=simple
+EnvironmentFile=/etc/sysconfig/log-courier
+ExecStartPre=/usr/sbin/log-courier -config-test=true -config=${CONFIG_FILE}
+ExecStart=/usr/sbin/log-courier -config=${CONFIG_FILE}
+ExecReload=/bin/kill -s HUP $MAINPID
+ExecStop=/bin/kill -s TERM $MAINPID
+PrivateTmp=true
+
+[Install]
+WantedBy=multi-user.target

contrib/rpm/log-courier.spec

@@ -0,0 +1,162 @@
+# Debug packaging does not work due to the go build building in extra debug sections RPM does not understand
+# Maybe we patch something later to fix this, but for now just don't build a debug package
+%define debug_package %{nil}
+
+Summary: Log Courier
+Name: log-courier
+Version: 1.2
+Release: 4%{dist}
+License: GPL
+Group: System Environment/Libraries
+Packager: Jason Woods <packages@jasonwoods.me.uk>
+URL: https://github.com/driskell/log-courier
+Source: https://github.com/driskell/log-courier/archive/v%{version}.zip
+BuildRoot: %{_tmppath}/%{name}-%{version}-root
+
+BuildRequires: golang >= 1.2
+BuildRequires: git
+BuildRequires: zeromq3-devel
+
+# Maybe tests in future - mock won't build ffi gem
+#BuildRequires: ruby >= 1.9.3, ruby-devel >= 1.9.3
+#BuildRequires: rubygem-bundler
+
+%if 0%{?rhel} >= 7
+Requires(post): systemd
+Requires(preun): systemd
+Requires(postun): systemd
+BuildRequires: systemd
+%endif
+
+Requires: zeromq3
+Requires: logrotate
+
+%description
+Log Courier is a tool created to transmit log files speedily and securely to
+remote Logstash instances for processing whilst using small amounts of local
+resources. The project is an enhanced fork of Logstash Forwarder 0.3.1 with many
+enhancements and behavioural improvements.
+
+%prep
+%setup -q -n %{name}-%{version}
+
+%build
+make with=zmq3
+# See notes above for BuildRequires ruby
+#make with=zmq3 test
+
+%install
+# Install binaries
+mkdir -p %{buildroot}%{_sbindir}
+install -m 0755 bin/log-courier %{buildroot}%{_sbindir}/log-courier
+mkdir -p %{buildroot}%{_bindir}
+install -m 0755 bin/lc-admin %{buildroot}%{_bindir}/lc-admin
+install -m 0755 bin/lc-tlscert %{buildroot}%{_bindir}/lc-tlscert
+
+# Install example configuration
+mkdir -p %{buildroot}%{_sysconfdir}/log-courier %{buildroot}%{_sysconfdir}/log-courier/examples/
+install -m 0644 docs/examples/* %{buildroot}%{_sysconfdir}/log-courier/examples/
+
+# Make the run dir
+mkdir -p %{buildroot}%{_var}/run %{buildroot}%{_var}/run/log-courier
+touch %{buildroot}%{_var}/run/log-courier/admin.socket
+
+# Install init script and related paraphernalia
+%if 0%{?rhel} >= 7
+mkdir -p %{buildroot}%{_unitdir}
+# No systemd script in log-courier release yet
+install -m 0644 contrib/initscripts/systemd.service %{buildroot}%{_unitdir}/log-courier.service
+%else
+mkdir -p %{buildroot}%{_sysconfdir}/init.d
+install -m 0755 contrib/initscripts/redhat-sysv.init %{buildroot}%{_sysconfdir}/init.d/log-courier
+touch %{buildroot}%{_var}/run/log-courier.pid
+%endif
+
+# Make the state dir
+mkdir -p %{buildroot}%{_var}/lib/log-courier
+touch %{buildroot}%{_var}/lib/log-courier/.log-courier
+
+%clean
+rm -rf $RPM_BUILD_ROOT
+
+%post
+%if 0%{?rhel} >= 7
+%systemd_post log-courier.service
+%else
+/sbin/chkconfig --add log-courier
+%endif
+
+%preun
+%if 0%{?rhel} >= 7
+%systemd_preun log-courier.service
+%else
+if [ $1 -eq 0 ]; then
+	/sbin/service log-courier stop >/dev/null 2>&1
+	/sbin/chkconfig --del log-courier
+fi
+%endif
+
+%postun
+%if 0%{?rhel} >= 7
+%systemd_postun_with_restart log-courier.service
+%else
+if [ $1 -ge 1 ]; then
+	/sbin/service log-courier restart >/dev/null 2>&1
+fi
+%endif
+
+%files
+%defattr(0755,root,root,0755)
+%{_sbindir}/log-courier
+%{_bindir}/lc-admin
+%{_bindir}/lc-tlscert
+%if 0%{?rhel} >= 7
+%{_unitdir}/log-courier.service
+%else
+%{_sysconfdir}/init.d/log-courier
+%endif
+
+%defattr(0644,root,root,0755)
+%{_sysconfdir}/log-courier
+%if 0%{?rhel} < 7
+%ghost %{_var}/run/log-courier.pid
+%endif
+%dir %attr(0700,root,root) %{_var}/run/log-courier
+%ghost %{_var}/run/log-courier/admin.socket
+%dir %{_var}/lib/log-courier
+%ghost %{_var}/lib/log-courier/.log-courier
+
+%changelog
+* Sat Nov 8 2014 Jason Woods <devel@jasonwoods.me.uk> - 1.2-4
+- Upgrade to v1.2
+- Fix stop message on future upgrade
+
+* Wed Nov 5 2014 Jason Woods <devel@jasonwoods.me.uk> - 1.1-4
+- Build with ZMQ 3 support
+
+* Mon Nov 3 2014 Jason Woods <devel@jasonwoods.me.uk> - 1.1-3
+- Fix init/systemd registration
+
+* Sun Nov 2 2014 Jason Woods <devel@jasonwoods.me.uk> - 1.1-2
+- Package for EL7
+- Restart service on upgrade
+
+* Fri Oct 31 2014 Jason Woods <devel@jasonwoods.me.uk> 1.1-1
+- Released 1.1
+- Cleanup for EL7 build
+
+* Mon Oct 13 2014 Jason Woods <packages@jasonwoods.me.uk> 0.15.1-1
+- Rebuild from v0.15 develop to fix more issues
+- Label as v0.15.1
+
+* Thu Sep 4 2014 Jason Woods <packages@jasonwoods.me.uk> 0.14.rc2-1
+- Rebuild from develop to fix more issues and enable unix socket
+	for administration
+- Label as v0.14.rc2
+
+* Wed Sep 3 2014 Jason Woods <packages@jasonwoods.me.uk> 0.14.rc1-1
+- Rebuild from develop to fix various reconnect hang issues
+- Label as v0.14.rc1
+
+* Mon Sep 1 2014 Jason Woods <packages@jasonwoods.me.uk> 0.13-1
+- Initial build of v0.13

docs/AdministrationUtility.md

@@ -24,6 +24,11 @@ listen address, set the `admin listen address` entry. See
 [Configuration](Configuration.md) for more information on these options and the
 default listen address.
 
+The `lc-admin` utility aims to be always backwards compatible whenever possible.
+This means a newer version of `lc-admin` should be able to connect to any older
+version of `log-courier`. The same is not true in reverse, and an older
+`lc-admin` may be unable to connect or communicate with a newer `log-courier`.
+
 ## Available Commands
 
 ### `help`

docs/ChangeLog.md

@@ -4,6 +4,7 @@
 <!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
 **Table of Contents**  *generated with [DocToc](http://doctoc.herokuapp.com/)*
 
+- [1.2](#12)
 - [1.1](#11)
 - [1.0](#10)
 - [0.15](#015)
@@ -17,6 +18,27 @@
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 
+## 1.2
+
+*1st December 2014*
+
+* Fix repeated partial Acks triggering an incorrect flush of events to registrar
+* Fix a loop that could occur when using ZMQ transport (#68)
+* TLS and TCP transport will now round robin the available server addresses
+instead of randomising
+* Implemented "Dead time in" on `lc-admin` harvester statuses
+* `lc-admin` status output is now sorted and no longer in random order
+* Add a workaround for logstash shutdown looping with "Context is terminated"
+messages (#73)
+* Implement asynchronous ZMQ receive pipeline in the Logstash gem to resolve
+timeout issues with multiple clients and a busy pipeline
+* Implement multithreaded SSL accept in the Logstash gem to prevent a single
+hung handshake attempt from blocking new connections
+* Switch to ruby-cabin logging in the gems to match Logstash logging
+* Updated the RedHat/CentOS 5/6 SysV init script in contrib to follow Fedora
+packaging guidelines
+* Provided a RedHat/CentOS 7 systemd service configuration in contrib
+
 ## 1.1
 
 *30th October 2014*