Add query_cache.py to speed repeated queries (fossasia#399)

Add query_cache.py to speed repeated queries

Author: cclauss

.travis.yml

@@ -2,6 +2,8 @@ language: python
 python:
   - "2.7"
   - "3.6"
+services:
+  - mongodb
 install:
   - pip install codecov flake8 pytest
   - pip install -r requirements.txt

app/query_cache.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+
+"""
+query_cache.py -- Implements a caching system for query server based on MongoDB
+
+Before sending a query to a remote search engine, use lookup() see if results
+from that same search engine and query are already in the cache.  If so, then
+print a cache hit message return the cached results.  If not, then use store()
+to write the search engine, query, query results, and a datetime created into
+the cache.  MongoDB will use the datetime to automatically delete out dated
+query results.
+
+Ideas for improvement:
+* Add a lookup_count to see how often cache actually saves us time.
+"""
+
+import datetime as dt
+import os
+
+from pymongo import DESCENDING, MongoClient
+
+client = MongoClient(os.environ.get('MONGO_URI', 'mongodb://localhost:27017/'))
+db = client['query-server-v2']
+db = db['queries']  # Automatically delete records that are older than one day
+db.create_index([('createdAt', DESCENDING)], expireAfterSeconds=60 * 60 * 24)
+
+
+def lookup(url):
+    """return search result if the URL is in the db or None on a cache miss."""
+    data = db.find_one({'url': url}) or {}
+    return data.get('links', None)
+
+
+def store(url, links):
+    """write the URL, the links, and a UTC timestamp into the database."""
+    db.delete_many({'url': url})  # remove all records for this URL
+    db.insert({'url': url, 'links': links, 'createdAt': dt.datetime.utcnow()})
+
+
+if __name__ == '__main__':
+    url = 'test_url'
+    print(lookup(url))
+    store(url, 'a b c d e'.split())
+    print(lookup(url))

app/server.py

@@ -5,15 +5,13 @@
 from dicttoxml import dicttoxml
 from flask import (Flask, Response, abort, jsonify, make_response,
                    render_template, request)
-from pymongo import MongoClient
 
+from query_cache import lookup, store
 from scrapers import feed_gen, scrapers
 
 app = Flask(__name__)
 err = ""
 
-client = MongoClient(os.environ.get('MONGO_URI', 'mongodb://localhost:27017/'))
-db = client['query-server-v2']
 errorObj = {
     'type': 'Internal Server Error',
     'status_code': 500,
@@ -59,14 +57,19 @@ def search(search_engine):
             error = [400, 'Not Found - missing query', qformat]
             return bad_request(error)
 
-        result = feed_gen(query, engine, count)
-        if not result:
-            error = [404, 'No response', qformat]
-            return bad_request(error)
-
-        if db['queries'].find({query: query}).limit(1) is False:
-            db['queries'].insert(
-                {"query": query, "engine": engine, "qformat": qformat})
+        # first see if we can get the results for the cache
+        engine_and_query = engine + ':' + query
+        result = lookup(engine_and_query)
+        if result:
+            print("cache hit: {}".format(engine_and_query))
+        else:
+            result = feed_gen(query, engine, count)
+            if result:
+                # store the result in the cache to speed up future searches
+                store(engine_and_query, result)
+            else:
+                error = [404, 'No response', engine_and_query]
+                return bad_request(error)
 
         try:
             unicode  # unicode is undefined in Python 3 so NameError is raised

app/test_server.py

@@ -3,6 +3,7 @@
 import pytest
 import requests
 
+from .scrapers import small_test
 
 REASON = 'Do you have query-server running on http://127.0.0.1:7001 ?'
 TRAVIS_CI = os.getenv('TRAVIS', False)  # Running in Travis CI?
@@ -12,6 +13,10 @@ def test_true():
     assert True, "We have a problem!"
 
 
+def test_small_test():
+    small_test()
+
+
 @pytest.mark.xfail(not TRAVIS_CI, reason=REASON)
 def test_invalid_url_api_call():
     response = requests.get('http://localhost:7001/api/v1/search/invalid_url')

requirements.txt

@@ -8,9 +8,9 @@ futures>=3.0.5
 html5lib>=0.9999999
 Jinja2>=2.9.5
 lxml>=3.7.2
-pymongo>=3.4.0
+pymongo>=3.6.0
 pytest>=3.0.6
 pytest-cov>=2.4.0
 requests>=2.13.0
 webencodings>=0.5
-defusedxml>=0.5.0
+defusedxml>=0.5.0