Merge pull request #43 from istresearch/make-traptor-process-faster

Make traptor process faster
istresearch · May 3, 2016 · 288c9cd · 288c9cd
2 parents ca673c5 + 466cb6b
commit 288c9cd
Show file tree

Hide file tree

Showing 7 changed files with 418 additions and 62 deletions.
diff --git a/tests/data/follow_tweet.json b/tests/data/follow_tweet.json
@@ -1 +1,89 @@
-{"favorited": false, "contributors": null, "truncated": false, "text": "I'm very happy to say the next single from 1989 will be 'New Romantics'.", "is_quote_status": false, "in_reply_to_status_id": null, "user": {"follow_request_sent": false, "has_extended_profile": false, "profile_use_background_image": false, "time_zone": "Dublin", "id": 17919972, "description": "Born in 1989.", "verified": true, "entities": {"url": {"urls": [{"url": "https://t.co/blhi4NNEJr", "indices": [0, 23], "expanded_url": "http://smarturl.it/1989TourLIVE", "display_url": "smarturl.it/1989TourLIVE"}]}, "description": {"urls": []}}, "profile_image_url_https": "https://pbs.twimg.com/profile_images/505200807503867904/osJXmYRl_normal.jpeg", "profile_sidebar_fill_color": "DDEEF6", "is_translator": false, "geo_enabled": false, "profile_text_color": "333333", "followers_count": 71643131, "protected": false, "id_str": "17919972", "default_profile_image": false, "listed_count": 122360, "lang": "en", "utc_offset": 0, "statuses_count": 4106, "profile_background_color": "C0DEED", "friends_count": 245, "profile_link_color": "0084B4", "profile_image_url": "http://pbs.twimg.com/profile_images/505200807503867904/osJXmYRl_normal.jpeg", "notifications": false, "profile_background_image_url_https": "https://pbs.twimg.com/profile_background_images/687293757/6d2ec27f32fa8cc2fcb7e6a9eada9945.jpeg", "profile_banner_url": "https://pbs.twimg.com/profile_banners/17919972/1409286315", "profile_background_image_url": "http://pbs.twimg.com/profile_background_images/687293757/6d2ec27f32fa8cc2fcb7e6a9eada9945.jpeg", "name": "Taylor Swift", "is_translation_enabled": false, "profile_background_tile": false, "favourites_count": 2109, "screen_name": "taylorswift13", "url": "https://t.co/blhi4NNEJr", "created_at": "Sat Dec 06 10:10:54 +0000 2008", "contributors_enabled": false, "location": "", "profile_sidebar_border_color": "FFFFFF", "default_profile": false, "following": false}, "geo": null, "id": 700890866920067072, "favorite_count": 146590, "lang": "en", "entities": {"symbols": [], "user_mentions": [], "hashtags": [], "urls": []}, "created_at": "Sat Feb 20 03:52:59 +0000 2016", "retweeted": false, "coordinates": null, "in_reply_to_user_id_str": null, "source": "<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>", "in_reply_to_status_id_str": null, "in_reply_to_screen_name": null, "id_str": "700890866920067072", "place": null, "retweet_count": 69616, "in_reply_to_user_id": null}
+{
+  "favorited": false,
+  "contributors": null,
+  "truncated": false,
+  "text": "I'm very happy to say the next single from 1989 will be 'New Romantics'.",
+  "is_quote_status": false,
+  "in_reply_to_status_id": null,
+  "user": {
+    "follow_request_sent": false,
+    "has_extended_profile": false,
+    "profile_use_background_image": false,
+    "time_zone": "Dublin",
+    "id": 17919972,
+    "description": "Born in 1989.",
+    "verified": true,
+    "entities": {
+      "url": {
+        "urls": [
+          {
+            "url": "https://t.co/blhi4NNEJr",
+            "indices": [
+              0,
+              23
+            ],
+            "expanded_url": "http://smarturl.it/1989TourLIVE",
+            "display_url": "smarturl.it/1989TourLIVE"
+          }
+        ]
+      },
+      "description": {
+        "urls": []
+      }
+    },
+    "profile_image_url_https": "https://pbs.twimg.com/profile_images/505200807503867904/osJXmYRl_normal.jpeg",
+    "profile_sidebar_fill_color": "DDEEF6",
+    "is_translator": false,
+    "geo_enabled": false,
+    "profile_text_color": "333333",
+    "followers_count": 71643131,
+    "protected": false,
+    "id_str": "17919972",
+    "default_profile_image": false,
+    "listed_count": 122360,
+    "lang": "en",
+    "utc_offset": 0,
+    "statuses_count": 4106,
+    "profile_background_color": "C0DEED",
+    "friends_count": 245,
+    "profile_link_color": "0084B4",
+    "profile_image_url": "http://pbs.twimg.com/profile_images/505200807503867904/osJXmYRl_normal.jpeg",
+    "notifications": false,
+    "profile_background_image_url_https": "https://pbs.twimg.com/profile_background_images/687293757/6d2ec27f32fa8cc2fcb7e6a9eada9945.jpeg",
+    "profile_banner_url": "https://pbs.twimg.com/profile_banners/17919972/1409286315",
+    "profile_background_image_url": "http://pbs.twimg.com/profile_background_images/687293757/6d2ec27f32fa8cc2fcb7e6a9eada9945.jpeg",
+    "name": "Taylor Swift",
+    "is_translation_enabled": false,
+    "profile_background_tile": false,
+    "favourites_count": 2109,
+    "screen_name": "taylorswift13",
+    "url": "https://t.co/blhi4NNEJr",
+    "created_at": "Sat Dec 06 10:10:54 +0000 2008",
+    "contributors_enabled": false,
+    "location": "",
+    "profile_sidebar_border_color": "FFFFFF",
+    "default_profile": false,
+    "following": false
+  },
+  "geo": null,
+  "id": 700890866920067072,
+  "favorite_count": 146590,
+  "lang": "en",
+  "entities": {
+    "symbols": [],
+    "user_mentions": [],
+    "hashtags": [],
+    "urls": []
+  },
+  "created_at": "Sat Feb 20 03:52:59 +0000 2016",
+  "retweeted": false,
+  "coordinates": null,
+  "in_reply_to_user_id_str": null,
+  "source": "<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>",
+  "in_reply_to_status_id_str": null,
+  "in_reply_to_screen_name": null,
+  "id_str": "700890866920067072",
+  "place": null,
+  "retweet_count": 69616,
+  "in_reply_to_user_id": null
+}
diff --git a/tests/data/locations_tweet.json b/tests/data/locations_tweet.json
@@ -1 +1,117 @@
-{"contributors": null, "truncated": false, "text": "@Alexx__27 idk", "is_quote_status": false, "in_reply_to_status_id": 701949606007996416, "id": 701950328078348288, "favorite_count": 0, "source": "<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>", "retweeted": false, "coordinates": null, "timestamp_ms": "1456192974477", "entities": {"user_mentions": [{"indices": [0, 10], "screen_name": "Alexx__27", "id": 2431399232, "name": "Alex \u2668\ufe0f", "id_str": "2431399232"}], "symbols": [], "hashtags": [], "urls": []}, "in_reply_to_screen_name": "Alexx__27", "in_reply_to_user_id": 2431399232, "traptor": {"created_at_iso": "2016-02-23T02:02:54+00:00"}, "id_str": "701950328078348288", "favorited": false, "user": {"follow_request_sent": null, "profile_use_background_image": true, "id": 562524394, "verified": false, "profile_image_url_https": "https://pbs.twimg.com/profile_images/701947518737715200/7vPEepur_normal.jpg", "profile_sidebar_fill_color": "DDEEF6", "is_translator": false, "geo_enabled": true, "profile_text_color": "333333", "followers_count": 754, "protected": false, "location": null, "default_profile_image": false, "id_str": "562524394", "utc_offset": -21600, "statuses_count": 45525, "description": null, "friends_count": 665, "profile_link_color": "B40B43", "profile_image_url": "http://pbs.twimg.com/profile_images/701947518737715200/7vPEepur_normal.jpg", "notifications": null, "profile_background_image_url_https": "https://pbs.twimg.com/profile_background_images/437502442461069312/yGDRa5xj.jpeg", "profile_background_color": "FF6699", "profile_banner_url": "https://pbs.twimg.com/profile_banners/562524394/1455651000", "profile_background_image_url": "http://pbs.twimg.com/profile_background_images/437502442461069312/yGDRa5xj.jpeg", "screen_name": "jeennnyy_", "lang": "en", "profile_background_tile": true, "favourites_count": 31358, "name": "Jenny Hernandez", "url": "http://Instagram.com/jeennnnyy_", "created_at": "Wed Apr 25 02:25:20 +0000 2012", "contributors_enabled": false, "time_zone": "Central Time (US & Canada)", "profile_sidebar_border_color": "000000", "default_profile": false, "following": null, "listed_count": 3}, "geo": null, "in_reply_to_user_id_str": "2431399232", "lang": "und", "retweet_count": 0, "created_at": "Tue Feb 23 02:02:54 +0000 2016", "filter_level": "low", "in_reply_to_status_id_str": "701949606007996416", "place": {"country_code": "US", "url": "https://api.twitter.com/1.1/geo/id/45cadd6ef118ec9f.json", "country": "United States", "place_type": "city", "bounding_box": {"type": "Polygon", "coordinates": [[[-122.065206, 37.330068], [-122.065206, 37.426726], [-121.982475, 37.426726], [-121.982475, 37.330068]]]}, "full_name": "Sunnyvale, CA", "attributes": {}, "id": "45cadd6ef118ec9f", "name": "Sunnyvale"}}
+{
+  "contributors": null,
+  "truncated": false,
+  "text": "@Alexx__27 idk",
+  "is_quote_status": false,
+  "in_reply_to_status_id": 701949606007996416,
+  "id": 701950328078348288,
+  "favorite_count": 0,
+  "source": "<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>",
+  "retweeted": false,
+  "coordinates": null,
+  "timestamp_ms": "1456192974477",
+  "entities": {
+    "user_mentions": [
+      {
+        "indices": [
+          0,
+          10
+        ],
+        "screen_name": "Alexx__27",
+        "id": 2431399232,
+        "name": "Alex ♨️",
+        "id_str": "2431399232"
+      }
+    ],
+    "symbols": [],
+    "hashtags": [],
+    "urls": []
+  },
+  "in_reply_to_screen_name": "Alexx__27",
+  "in_reply_to_user_id": 2431399232,
+  "traptor": {
+    "created_at_iso": "2016-02-23T02:02:54+00:00"
+  },
+  "id_str": "701950328078348288",
+  "favorited": false,
+  "user": {
+    "follow_request_sent": null,
+    "profile_use_background_image": true,
+    "id": 562524394,
+    "verified": false,
+    "profile_image_url_https": "https://pbs.twimg.com/profile_images/701947518737715200/7vPEepur_normal.jpg",
+    "profile_sidebar_fill_color": "DDEEF6",
+    "is_translator": false,
+    "geo_enabled": true,
+    "profile_text_color": "333333",
+    "followers_count": 754,
+    "protected": false,
+    "location": null,
+    "default_profile_image": false,
+    "id_str": "562524394",
+    "utc_offset": -21600,
+    "statuses_count": 45525,
+    "description": null,
+    "friends_count": 665,
+    "profile_link_color": "B40B43",
+    "profile_image_url": "http://pbs.twimg.com/profile_images/701947518737715200/7vPEepur_normal.jpg",
+    "notifications": null,
+    "profile_background_image_url_https": "https://pbs.twimg.com/profile_background_images/437502442461069312/yGDRa5xj.jpeg",
+    "profile_background_color": "FF6699",
+    "profile_banner_url": "https://pbs.twimg.com/profile_banners/562524394/1455651000",
+    "profile_background_image_url": "http://pbs.twimg.com/profile_background_images/437502442461069312/yGDRa5xj.jpeg",
+    "screen_name": "jeennnyy_",
+    "lang": "en",
+    "profile_background_tile": true,
+    "favourites_count": 31358,
+    "name": "Jenny Hernandez",
+    "url": "http://Instagram.com/jeennnnyy_",
+    "created_at": "Wed Apr 25 02:25:20 +0000 2012",
+    "contributors_enabled": false,
+    "time_zone": "Central Time (US & Canada)",
+    "profile_sidebar_border_color": "000000",
+    "default_profile": false,
+    "following": null,
+    "listed_count": 3
+  },
+  "geo": null,
+  "in_reply_to_user_id_str": "2431399232",
+  "lang": "und",
+  "retweet_count": 0,
+  "created_at": "Tue Feb 23 02:02:54 +0000 2016",
+  "filter_level": "low",
+  "in_reply_to_status_id_str": "701949606007996416",
+  "place": {
+    "country_code": "US",
+    "url": "https://api.twitter.com/1.1/geo/id/45cadd6ef118ec9f.json",
+    "country": "United States",
+    "place_type": "city",
+    "bounding_box": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [
+            -122.065206,
+            37.330068
+          ],
+          [
+            -122.065206,
+            37.426726
+          ],
+          [
+            -121.982475,
+            37.426726
+          ],
+          [
+            -121.982475,
+            37.330068
+          ]
+        ]
+      ]
+    },
+    "full_name": "Sunnyvale, CA",
+    "attributes": {},
+    "id": "45cadd6ef118ec9f",
+    "name": "Sunnyvale"
+  }
+}
diff --git a/tests/data/other_tweet_messages.json b/tests/data/other_tweet_messages.json
@@ -0,0 +1,67 @@
+{
+  "delete":{
+    "status":{
+      "id":1234,
+      "id_str":"1234",
+      "user_id":3,
+      "user_id_str":"3"
+    }
+  }
+},
+{
+  "warning":{
+    "code":"FALLING_BEHIND",
+    "message":"Your connection is falling behind and messages are being queued for delivery to you. Your queue is now over 60% full. You will be disconnected when the queue is full.",
+    "percent_full": 60
+  }
+},
+{
+  "scrub_geo":{
+    "user_id":14090452,
+    "user_id_str":"14090452",
+    "up_to_status_id":23260136625,
+    "up_to_status_id_str":"23260136625"
+  }
+},
+{
+  "limit":{
+    "track":1234
+  }
+},
+{
+  "status_withheld":{
+      "id":1234567890,
+      "user_id":123456,
+      "withheld_in_countries":["DE", "AR"]
+  }
+},
+{
+  "user_withheld":{
+    "id":123456,
+    "withheld_in_countries":["DE","AR"]
+  }
+},
+{
+  "disconnect":{
+    "code": 4,
+    "stream_name":"",
+    "reason":""
+  }
+},
+{
+  "warning":{
+    "code":"FALLING_BEHIND",
+    "message":"Your connection is falling behind and messages are being queued for delivery to you. Your queue is now over 60% full. You will be disconnected when the queue is full.",
+    "percent_full": 60
+  }
+},
+{
+    "created_at": "Tue Aug 06 02:23:21 +0000 2013",
+    "source": {
+         ...
+    },
+    "target": {
+      ...
+    },
+    "event": "user_update"
+}
diff --git a/tests/data/track_tweet.json b/tests/data/track_tweet.json
@@ -1 +1,91 @@
-{"favorited": false, "contributors": null, "truncated": false, "text": "@DannyTaughtYou_ happy birthday", "is_quote_status": false, "in_reply_to_status_id": 701579336004972545, "user": {"follow_request_sent": false, "has_extended_profile": false, "profile_use_background_image": true, "time_zone": "Central Time (US & Canada)", "id": 35969755, "description": "Show No Love, Love Will Get You Hurt!!! #TeamNunns Instagram- itaint_nunn1 snapchat: Nunnwhitney FEBRUARY 20th", "verified": false, "entities": {"description": {"urls": []}}, "profile_image_url_https": "https://pbs.twimg.com/profile_images/673722421677981696/-MNnci6a_normal.jpg", "profile_sidebar_fill_color": "EFEFEF", "is_translator": false, "geo_enabled": false, "profile_text_color": "333333", "followers_count": 757, "protected": false, "id_str": "35969755", "default_profile_image": false, "listed_count": 3, "lang": "en", "utc_offset": -21600, "statuses_count": 94410, "profile_background_color": "131516", "friends_count": 714, "profile_link_color": "009999", "profile_image_url": "http://pbs.twimg.com/profile_images/673722421677981696/-MNnci6a_normal.jpg", "notifications": false, "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", "profile_banner_url": "https://pbs.twimg.com/profile_banners/35969755/1441172070", "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", "name": "February20", "is_translation_enabled": false, "profile_background_tile": true, "favourites_count": 3669, "screen_name": "ItAint_Nunn", "url": null, "created_at": "Tue Apr 28 03:11:26 +0000 2009", "contributors_enabled": false, "location": "Where the money at.!!! ", "profile_sidebar_border_color": "EEEEEE", "default_profile": false, "following": false}, "geo": null, "id": 701580888455389187, "favorite_count": 0, "lang": "en", "entities": {"symbols": [], "user_mentions": [{"indices": [0, 16], "screen_name": "DannyTaughtYou_", "id": 101886075, "name": "DANNYDAY2/21", "id_str": "101886075"}], "hashtags": [], "urls": []}, "created_at": "Mon Feb 22 01:34:53 +0000 2016", "retweeted": false, "metadata": {"iso_language_code": "en", "result_type": "recent"}, "coordinates": null, "in_reply_to_user_id_str": "101886075", "source": "<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>", "in_reply_to_status_id_str": "701579336004972545", "in_reply_to_screen_name": "DannyTaughtYou_", "in_reply_to_user_id": 101886075, "place": null, "retweet_count": 0, "id_str": "701580888455389187"}
+{
+  "favorited": false,
+  "contributors": null,
+  "truncated": false,
+  "text": "@DannyTaughtYou_ happy birthday",
+  "is_quote_status": false,
+  "in_reply_to_status_id": 701579336004972545,
+  "user": {
+    "follow_request_sent": false,
+    "has_extended_profile": false,
+    "profile_use_background_image": true,
+    "time_zone": "Central Time (US & Canada)",
+    "id": 35969755,
+    "description": "Show No Love, Love Will Get You Hurt!!! #TeamNunns Instagram- itaint_nunn1 snapchat: Nunnwhitney FEBRUARY 20th",
+    "verified": false,
+    "entities": {
+      "description": {
+        "urls": []
+      }
+    },
+    "profile_image_url_https": "https://pbs.twimg.com/profile_images/673722421677981696/-MNnci6a_normal.jpg",
+    "profile_sidebar_fill_color": "EFEFEF",
+    "is_translator": false,
+    "geo_enabled": false,
+    "profile_text_color": "333333",
+    "followers_count": 757,
+    "protected": false,
+    "id_str": "35969755",
+    "default_profile_image": false,
+    "listed_count": 3,
+    "lang": "en",
+    "utc_offset": -21600,
+    "statuses_count": 94410,
+    "profile_background_color": "131516",
+    "friends_count": 714,
+    "profile_link_color": "009999",
+    "profile_image_url": "http://pbs.twimg.com/profile_images/673722421677981696/-MNnci6a_normal.jpg",
+    "notifications": false,
+    "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif",
+    "profile_banner_url": "https://pbs.twimg.com/profile_banners/35969755/1441172070",
+    "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif",
+    "name": "February20",
+    "is_translation_enabled": false,
+    "profile_background_tile": true,
+    "favourites_count": 3669,
+    "screen_name": "ItAint_Nunn",
+    "url": null,
+    "created_at": "Tue Apr 28 03:11:26 +0000 2009",
+    "contributors_enabled": false,
+    "location": "Where the money at.!!! ",
+    "profile_sidebar_border_color": "EEEEEE",
+    "default_profile": false,
+    "following": false
+  },
+  "geo": null,
+  "id": 701580888455389187,
+  "favorite_count": 0,
+  "lang": "en",
+  "entities": {
+    "symbols": [],
+    "user_mentions": [
+      {
+        "indices": [
+          0,
+          16
+        ],
+        "screen_name": "DannyTaughtYou_",
+        "id": 101886075,
+        "name": "DANNYDAY2/21",
+        "id_str": "101886075"
+      }
+    ],
+    "hashtags": [],
+    "urls": []
+  },
+  "created_at": "Mon Feb 22 01:34:53 +0000 2016",
+  "retweeted": false,
+  "metadata": {
+    "iso_language_code": "en",
+    "result_type": "recent"
+  },
+  "coordinates": null,
+  "in_reply_to_user_id_str": "101886075",
+  "source": "<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>",
+  "in_reply_to_status_id_str": "701579336004972545",
+  "in_reply_to_screen_name": "DannyTaughtYou_",
+  "in_reply_to_user_id": 101886075,
+  "place": null,
+  "retweet_count": 0,
+  "id_str": "701580888455389187"
+}