Merge pull request #47 from istresearch/update-rule-matching-round-2

Update rule matching round 2

tests/data/no_match_tweet.json

@@ -0,0 +1,91 @@
+{
+  "favorited": false,
+  "contributors": null,
+  "truncated": false,
+  "text": "This tweet will not match anything.",
+  "is_quote_status": false,
+  "in_reply_to_status_id": 701579336004972545,
+  "user": {
+    "follow_request_sent": false,
+    "has_extended_profile": false,
+    "profile_use_background_image": true,
+    "time_zone": "Central Time (US & Canada)",
+    "id": 35969755,
+    "description": "Show No Love, Love Will Get You Hurt!!! #TeamNunns Instagram- itaint_nunn1 snapchat: Nunnwhitney FEBRUARY 20th",
+    "verified": false,
+    "entities": {
+      "description": {
+        "urls": []
+      }
+    },
+    "profile_image_url_https": "https://pbs.twimg.com/profile_images/673722421677981696/-MNnci6a_normal.jpg",
+    "profile_sidebar_fill_color": "EFEFEF",
+    "is_translator": false,
+    "geo_enabled": false,
+    "profile_text_color": "333333",
+    "followers_count": 757,
+    "protected": false,
+    "id_str": "35969755",
+    "default_profile_image": false,
+    "listed_count": 3,
+    "lang": "en",
+    "utc_offset": -21600,
+    "statuses_count": 94410,
+    "profile_background_color": "131516",
+    "friends_count": 714,
+    "profile_link_color": "009999",
+    "profile_image_url": "http://pbs.twimg.com/profile_images/673722421677981696/-MNnci6a_normal.jpg",
+    "notifications": false,
+    "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif",
+    "profile_banner_url": "https://pbs.twimg.com/profile_banners/35969755/1441172070",
+    "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif",
+    "name": "February20",
+    "is_translation_enabled": false,
+    "profile_background_tile": true,
+    "favourites_count": 3669,
+    "screen_name": "ItAint_Nunn",
+    "url": null,
+    "created_at": "Tue Apr 28 03:11:26 +0000 2009",
+    "contributors_enabled": false,
+    "location": "Where the money at.!!! ",
+    "profile_sidebar_border_color": "EEEEEE",
+    "default_profile": false,
+    "following": false
+  },
+  "geo": null,
+  "id": 701580888455389187,
+  "favorite_count": 0,
+  "lang": "en",
+  "entities": {
+    "symbols": [],
+    "user_mentions": [
+      {
+        "indices": [
+          0,
+          16
+        ],
+        "screen_name": "DannyTaughtYou_",
+        "id": 101886075,
+        "name": "DANNYDAY2/21",
+        "id_str": "101886075"
+      }
+    ],
+    "hashtags": [],
+    "urls": []
+  },
+  "created_at": "Mon Feb 22 01:34:53 +0000 2016",
+  "retweeted": false,
+  "metadata": {
+    "iso_language_code": "en",
+    "result_type": "recent"
+  },
+  "coordinates": null,
+  "in_reply_to_user_id_str": "101886075",
+  "source": "<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>",
+  "in_reply_to_status_id_str": "701579336004972545",
+  "in_reply_to_screen_name": "DannyTaughtYou_",
+  "in_reply_to_user_id": 101886075,
+  "place": null,
+  "retweet_count": 0,
+  "id_str": "701580888455389187"
+}

tests/test_traptor_offline.py

@@ -92,6 +92,14 @@ def tweets(request, traptor):
 
     return loaded_tweet
 
+@pytest.fixture
+def no_match_tweets(request, traptor):
+    """Create a list of tweets."""
+    with open('tests/data/no_match_tweet.json') as f:
+        loaded_tweet = json.load(f)
+
+    return loaded_tweet
+
 
 @pytest.fixture
 def non_tweet_stream_messages(request, traptor):
@@ -225,6 +233,24 @@ def test_main_loop(self, traptor, tweets):
         if traptor.traptor_type == 'locations':
             assert enriched_data['traptor']['created_at_iso'] == '2016-02-23T02:02:54+00:00'
 
+    def test_ensure_traptor_is_in_tweet_on_no_match(self, traptor, no_match_tweets):
+        """Ensure that the traptor section is added to a tweet when no rule matches."""
+        traptor._setup()
+        traptor.redis_rules = [rule for rule in traptor._get_redis_rules()]
+        traptor.twitter_rules = traptor._make_twitter_rules(traptor.redis_rules)
+        traptor.birdy_stream = MagicMock(return_value=no_match_tweets)
+        traptor.birdy_stream.stream = traptor.birdy_stream
+
+        tweet = traptor.birdy_stream.stream()
+        enriched_data = traptor._enrich_tweet(tweet)
+
+        if traptor.traptor_type in ['track', 'follow', 'locations']:
+            assert enriched_data['traptor']['created_at_iso'] == '2016-02-22T01:34:53+00:00'
+
+        if traptor.traptor_type in ['track', 'follow']:    
+            assert enriched_data['traptor']['rule_tag'] == 'Not found'
+            assert enriched_data['traptor']['rule_value'] == 'Not found'
+
     def test_ensure_heartbeat_message_is_produced(self, traptor):
         """Ensure Traptor can produce heartbeat messages."""
         traptor._setup()

traptor/traptor.py

@@ -217,6 +217,9 @@ def _add_rule_tag_and_value_to_tweet(self, tweet_dict, search_str, matched_rule)
                 # Pass all key/value pairs from matched rule through to Traptor
                 for key, value in matched_rule.iteritems():
                     tweet_dict['traptor'][key] = value
+
+                # Log that a rule was matched
+                self.logger.info("Rule matched for tweet id: {}".format(tweet_dict['id_str']))
 
         return tweet_dict
 
@@ -240,8 +243,6 @@ def _find_rule_matches(self, tweet_dict):
                 for key, value in rule.iteritems():
                     new_dict['traptor'][key] = value
 
-                return new_dict
-
         if self.traptor_type == 'track':
 
             for rule in self.redis_rules:
@@ -263,14 +264,6 @@ def _find_rule_matches(self, tweet_dict):
                                                                  search_str,
                                                                  rule)
 
-                if 'rule_tag' not in new_dict['traptor']:
-                    self.logger.warning('Could not find rule_tag: {}, rule_value: {}, in tweet {}'.format(
-                                        rule['tag'], rule['value'].encode('utf-8'), new_dict.get('id_str')))
-                    new_dict['traptor']['rule_tag'] = 'Not found'
-                    new_dict['traptor']['rule_value'] = 'Not found'
-
-            return new_dict
-
         # If this is a follow Traptor, only check the user/id field of the tweet
         if self.traptor_type == 'follow':
             for rule in self.redis_rules:
@@ -286,6 +279,14 @@ def _find_rule_matches(self, tweet_dict):
                         new_dict['traptor'][key] = value
 
                     return new_dict
+
+        if 'rule_tag' not in new_dict['traptor']:
+            new_dict['traptor']['rule_tag'] = 'Not found'
+            new_dict['traptor']['rule_value'] = 'Not found'
+            # Log that a rule was matched
+            self.logger.warning("No rule matched for tweet id: {}".format(tweet_dict['id_str']))
+
+        return new_dict
 
 
     def _get_redis_rules(self):
@@ -586,4 +587,4 @@ def main(sentry, stdout, info, debug, delay, id, type, key):
                           REDIS_PUBSUB_CHANNEL, SENTRY_SECRET)
     from raven import Client
 
-    sys.exit(main())
+    sys.exit(main())