Merge pull request #20 from istresearch/unit-testing

Unit testing

.gitignore

@@ -1,12 +1,25 @@
-# Ignore local settings
-localsettings.py
+# Python binaries
+*.pyc
+
+# Sphinx
+docs/_build
+docs/_build_html
+
+# OSX garbage
+.DS_STORE
 
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
+# Scrapy Cluster
+kafka-monitor/logs/*
+redis-monitor/logs/*
+crawler/logs/*
+crawler/main.log
+localsettings.py
 
-# C extensions
-*.so
+# Vagrant test VM
+.vagrant
+local/
+bin/
+pip-selfcheck.json
 
 # Distribution / packaging
 .Python
@@ -24,38 +37,4 @@ sdist/
 var/
 *.egg-info/
 .installed.cfg
-*.egg
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*,cover
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-
-# Sphinx documentation
-docs/_build/
-docs/_build_html/
-
-# PyBuilder
-target/
+*.egg

Vagrantfile

@@ -0,0 +1,29 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+Vagrant.require_version ">= 1.7.4"
+
+Vagrant.configure(2) do |config|
+
+  # Configure general VM options
+  config.vm.provider "virtualbox" do |vb|
+    vb.memory = 2048
+    vb.cpus = 4
+  end
+
+  config.vm.define 'scdev' do |node|
+    node.vm.box = 'ubuntu/trusty64'
+    node.vm.hostname = 'scdev'
+    node.vm.network "private_network", ip: "192.168.33.99"
+    node.vm.provision "ansible" do |ansible|
+      ansible.verbose = true
+      ansible.groups = {
+        "kafka" => ["scdev"],
+        "zookeeper" => ["scdev"],
+        "redis" => ["scdev"],
+        "all_groups:children" => ["kafka", "zookeeper", "redis"]
+      }
+      ansible.playbook = "ansible/scrapy-cluster.yml"
+    end
+  end
+end

ansible/kafka.yml

@@ -0,0 +1,12 @@
+---
+
+- name: Kafka Brokers
+  hosts: kafka
+
+  sudo: yes
+
+  vars:
+   - kafka_host_list: "{{ groups['kafka'] }}"
+   - zookeeper_host_list: "{{ groups['zookeeper'] }}"
+  roles:
+    - kafka

ansible/redis.yml

@@ -0,0 +1,9 @@
+---
+
+- name: Redis Master
+  hosts: redis
+
+  sudo: yes
+
+  roles:
+    - redis

ansible/roles/java/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+# file: roles/common/defaults/main.yml
+
+# The specific version of Oracle Java that can be found in YUM
+java_version: 1.7.0_71
+

ansible/roles/java/files/java.sh

@@ -0,0 +1,3 @@
+# Initialization script for Java
+JAVA_HOME="/usr/java/default"
+export JAVA_HOME

ansible/roles/java/tasks/main.yml

@@ -0,0 +1,54 @@
+---
+# file: roles/common/tasks/main.yml
+
+- name: apt install java 
+  apt: 
+    name=default-jdk
+    state=present
+    # update-cache=yes
+  tags: java
+  when: ansible_os_family == "Debian"
+
+- name: yum install java 
+  yum: 
+    name=jdk-{{ java_version }}
+    state=present
+  tags: java
+  when: ansible_os_family == "RedHat"
+
+- name: java system environment configuration
+  copy:
+    src=java.sh
+    dest=/etc/profile.d/java.sh
+    owner=0
+    group=0
+    mode=0755
+  tags: java
+
+- name: Set JAVA_HOME ansible fact
+  set_fact:
+    java_home=/usr/java/default
+  tags: java
+
+- name: Create Ansible facts.d directory
+  file:
+    state=directory
+    dest=/etc/ansible/facts.d
+    owner=0
+    group=0
+    mode=0755
+  tags: java
+
+- name: Install java facts
+  template:
+    src=facts.j2
+    dest=/etc/ansible/facts.d/java.fact
+    owner=0
+    group=0
+    mode=0644
+  tags: java
+
+- name: Re-read facts
+  setup:
+    filter=ansible_local
+  tags: java

ansible/roles/java/templates/facts.j2

@@ -0,0 +1,2 @@
+[general]
+java_home={{ java_home }}

ansible/roles/kafka/defaults/main.yml

@@ -0,0 +1,20 @@
+---
+
+kafka_version: 0.8.2.2
+
+kafka_install_dir: /opt/kafka
+kafka_config_dir: /opt/kafka/default/config
+kafka_log_dir: /opt/kafka/default/logs
+kafka_data_log_dir:
+  - /opt/kafka/topic-logs
+
+kafka_port: 9092
+kafka_message_max: 10000000
+kafka_replica_fetch_max_bytes: 15000000
+kafka_consumer_message_max: 16777216
+kafka_num_partitions: "{{ groups['kafka'] | length }}"
+kafka_replication_factor: "{{ groups['kafka'] | length }}"
+kafka_log_retention_hours: 168
+kafka_num_io_threads: 8
+
+kafka_source: "http://www.carfab.com/apachesoftware/kafka"

ansible/roles/kafka/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: restart kafka
+  supervisorctl:
+    name=kafka
+    state=restarted

ansible/roles/kafka/meta/main.yml

@@ -0,0 +1,4 @@
+---
+dependencies:
+  - { role: supervisord }
+  - { role: java }

ansible/roles/kafka/tasks/main.yml

@@ -0,0 +1,90 @@
+---
+
+- name: create kafka directories
+  file:
+    path={{ item }}
+    state=directory
+    mode=0744
+  with_items:
+    - "{{ kafka_install_dir }}"
+    - "{{ kafka_data_log_dir }}"
+  tags: kafka
+
+- name: check for existing install
+  stat: path={{ kafka_install_dir }}/kafka_2.9.2-{{ kafka_version }}
+  register: kafka
+  tags: kafka
+
+- name: download kafka
+  get_url:
+    url="{{ kafka_source }}/{{ kafka_version }}/kafka_2.9.2-{{ kafka_version }}.tgz"
+    dest=/tmp/kafka_2.9.2-{{ kafka_version }}.tgz
+    mode=0644
+    validate_certs=no
+  when: kafka.stat.isdir is not defined
+  tags: kafka
+
+- name: extract kafka
+  unarchive:
+    src=/tmp/kafka_2.9.2-{{ kafka_version }}.tgz
+    dest={{ kafka_install_dir }}
+    copy=no
+  when: kafka.stat.isdir is not defined
+  tags: kafka
+
+- name: delete temporary kafka file
+  file:
+    path=/tmp/kafka_2.9.2-{{ kafka_version }}.tgz
+    state=absent
+  ignore_errors: yes
+  tags: kafka
+
+- name: create kafka symlink
+  file:
+    path={{ kafka_install_dir }}/default
+    state=link
+    src={{ kafka_install_dir }}/kafka_2.9.2-{{ kafka_version }}
+  tags: kafka
+
+- name: configure kafka brokers
+  template:
+    src=server.properties.j2
+    dest={{ kafka_config_dir }}/server.properties
+    mode=0644
+  notify:
+    - restart kafka
+  tags: kafka
+
+- name: configure log4j
+  template:
+    src=log4j.properties.j2
+    dest={{ kafka_config_dir }}/log4j.properties
+    mode=0644
+  notify:
+    - restart kafka
+  tags: kafka
+
+- name: configure kafka consumer
+  template:
+    src=consumer.properties.j2
+    dest={{ kafka_config_dir }}/consumer.properties
+    mode=0644
+  notify:
+    - restart kafka
+  tags: kafka
+
+- name: copy supervisord config
+  template:
+    src=kafka-supervisord.conf.j2
+    dest={{ supervisord_programs_dir }}/kafka-supervisord.conf
+    mode=0644
+  notify:
+    - reread supervisord
+  tags: kafka
+
+- name: set up aliases
+  lineinfile: dest="/root/.bashrc" line="export KAFKA={{ kafka_install_dir }}/default"
+  lineinfile: dest="/root/.bashrc" line="export PATH={{ kafka_install_dir }}/default/bin:$PATH"
+  tags: env
+
+- cron: name="clear old kafka app logs" job="find /opt/kafka/default/logs -mtime +7 -exec rm {} \; > /dev/null" minute="0"

ansible/roles/kafka/templates/consumer.properties.j2

@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# see kafka.consumer.ConsumerConfig for more details
+
+# Zookeeper connection string
+# comma separated host:port pairs, each corresponding to a zk
+# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002"
+zookeeper.connect=127.0.0.1:2181
+
+# timeout in ms for connecting to zookeeper
+zookeeper.connection.timeout.ms=6000
+
+#consumer group id
+group.id=test-consumer-group
+
+#consumer timeout
+#consumer.timeout.ms=5000
+
+# Need to increase this to play nice with message.max.bytes = 10000000
+fetch.message.max.bytes={{ kafka_consumer_message_max }}

ansible/roles/kafka/templates/kafka-supervisord.conf.j2

@@ -0,0 +1,6 @@
+[program:kafka]
+command={{ kafka_install_dir }}/default/bin/kafka-server-start.sh {{ kafka_config_dir }}/server.properties
+autostart=true
+autorestart=true
+startsecs=5
+stopsignal=KILL