From 83c2a3762c75c92b667e9c3690075a48962703e0 Mon Sep 17 00:00:00 2001
From: Bill Rascher <wrascher@gmail.com>
Date: Mon, 3 Apr 2017 17:09:28 -0400
Subject: [PATCH 1/2] Updated the integration tests so they work correctly.

---
 .gitignore                         |  5 ++++-
 tests/traptor_integration_tests.py | 15 ++++++++++-----
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/.gitignore b/.gitignore
index 2477c06..7f29ea8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -57,4 +57,7 @@ build_full_environment.sh
 docker-compose.build.yml
 
 # Local environment things
-.condaauto
\ No newline at end of file
+.condaauto
+
+# VS Code
+/.vscode
diff --git a/tests/traptor_integration_tests.py b/tests/traptor_integration_tests.py
index da8d583..b0109a7 100644
--- a/tests/traptor_integration_tests.py
+++ b/tests/traptor_integration_tests.py
@@ -32,6 +32,7 @@
 HOST_FOR_TESTING = os.getenv('REDIS_HOST', 'localhost')
 TRAPTOR_TYPE = os.getenv('TRAPTOR_TYPE', 'track')
 TRAPTOR_ID = int(os.getenv('TRAPTOR_ID', 0))
+RULE_ID = 12348
 
 # Create a connection to Redis
 
@@ -51,14 +52,18 @@
     # Collect on tweets with the keyword `python`
     # Change the rule value if you want to collect on a different keyword or hashtag. Go nuts!
     test_track_rule = {
-      "tag": "Traptor.Test",
-      "type": "track",
-      "value": "python",
-      "description": "Test track rule for python"
+        "tag": "Traptor.Test", 
+        "value": "python", 
+        "status": "active", 
+        "description": "Test track rule for python", 
+        "appid": "test-appid", 
+        "date_added": "2017-04-02 16:58:34", 
+        "rule_type": "track", 
+        "rule_id": RULE_ID
     }
 
     try:
-        redis_key = "traptor-{}:{}:{}".format(TRAPTOR_TYPE, TRAPTOR_ID, 123)
+        redis_key = "traptor-{}:{}:{}".format(TRAPTOR_TYPE, TRAPTOR_ID, RULE_ID)
         redis_connection.hmset(redis_key, test_track_rule)
 
         print("Rule added")

From 0d8e17ce26ae5915c619cdde2540eded39e7790b Mon Sep 17 00:00:00 2001
From: Robert Dempsey <robertonrails@gmail.com>
Date: Tue, 4 Apr 2017 09:59:45 -0400
Subject: [PATCH 2/2] Delete rule counters when processing starts

In order to ensure that Traptor is logging the most recent stats, delete the rule counters before stream processing begins. This has been added in such a way that each time Traptor restarts its internal rule processing the rule counters will match with the current set of rules.
---
 tests/test_traptor_offline.py | 32 ++++++++++++++++++++++++++++++++
 traptor/traptor.py            | 16 ++++++++++++++++
 2 files changed, 48 insertions(+)

diff --git a/tests/test_traptor_offline.py b/tests/test_traptor_offline.py
index 73c4a72..942e5b9 100644
--- a/tests/test_traptor_offline.py
+++ b/tests/test_traptor_offline.py
@@ -378,6 +378,38 @@ def test_ensure_internal_rule_counters_are_correctly_made_for_rules(self, redis_
         if traptor.traptor_type == 'locations':
             assert len(traptor.rule_counters) == 0
 
+    def test_ensure_internal_rule_counters_are_properly_deleted(self, redis_rules, traptor):
+        """Ensure _delete_rule_counters are properly deleted."""
+        traptor._setup()
+        traptor.redis_conn = redis_rules
+
+        traptor.logger.info = MagicMock()
+        traptor.logger.error = MagicMock()
+
+        traptor.redis_rules = [rule for rule in traptor._get_redis_rules()]
+        traptor.twitter_rules = traptor._make_twitter_rules(traptor.redis_rules)
+
+        if traptor.traptor_type != 'locations':
+            traptor._make_rule_counters()
+            assert len(traptor.rule_counters) == 1
+
+        if traptor.traptor_type == 'track':
+            assert traptor.rule_counters['12347'] is not None
+
+            traptor._delete_rule_counters()
+            assert traptor.logger.error.call_count == 0
+            assert traptor.logger.info.call_count == 3
+
+        if traptor.traptor_type == 'follow':
+            assert traptor.rule_counters['12345'] is not None
+
+            traptor._delete_rule_counters()
+            assert traptor.logger.error.call_count == 0
+            assert traptor.logger.info.call_count == 3
+
+        if traptor.traptor_type == 'locations':
+            assert len(traptor.rule_counters) == 0
+
     def test_ensure_limit_message_counter_is_correctly_created(self, redis_rules, traptor):
         """Ensure _make_limit_message_counter makes the limit counter"""
         traptor._setup()
diff --git a/traptor/traptor.py b/traptor/traptor.py
index 867c3b5..24fa460 100644
--- a/traptor/traptor.py
+++ b/traptor/traptor.py
@@ -377,6 +377,21 @@ def _increment_rule_counter(self, tweet):
             dd_monitoring.increment('redis_error',
                                     tags=['error_type:connection_error'])
 
+    def _delete_rule_counters(self):
+        """
+        Delete the existing rule counters.
+        """
+        if len(self.rule_counters) > 0:
+            for counter in self.rule_counters:
+                try:
+                    self.rule_counters[counter].delete_key()
+                except:
+                    self.logger.error("Caught exception while deleting a rule counter", extra={
+                        'error_type': 'RedisConnectionError',
+                        'ex': traceback.format_exc()
+                    })
+            self.logger.info("Rule counters deleted successfully.")
+
     def _make_limit_message_counter(self):
         """
         Make a limit message counter to track the values of incoming limit messages.
@@ -1032,6 +1047,7 @@ def run(self):
 
         # Do all the things
         while True:
+            self._delete_rule_counters()
             self._wait_for_rules()
 
             # Concatenate all of the rule['value'] fields