Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add sqlite database adapter #175

Open
wants to merge 13 commits into
base: master
Choose a base branch
from
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@ mp3
*.mp3
.DS_Store
*.cnf
.idea/
12 changes: 12 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -295,3 +295,15 @@ wav | 1885
fingerprints | 377

There's a pretty direct trade-off between the necessary record time and the amount of storage needed. Adjusting the amplitude threshold for peaks and the fan value for fingerprinting will add more fingerprints and bolster the accuracy at the expense of more space.

## Update - SQLite is now supported

For the moment, in order to use SQLite as the database solution, you must add

import dejavu.database_sqlite

to the end of 'dejavu/database.py'.

SQLite support requires python's 'sqlite3' library, but does not need mysql. The 'dejavu.database_sql' import at the end of 'dejavu/database.py' is optional for SQLite only, but still necessary for mysql support.

An example of a configuration file for SQLite is in the project root titled 'dejavu.cnf.SQLITE_SAMPLE'.
1 change: 1 addition & 0 deletions dejavu.cnf.SQLITE_SAMPLE
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"database": {"db_path": "audio_print_data.sqlite3"}, "database_type": "sqlite3"}
8 changes: 4 additions & 4 deletions dejavu/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from dejavu.database import get_database, Database
import dejavu.decoder as decoder
import fingerprint
from dejavu import fingerprint
import multiprocessing
import os
import traceback
Expand Down Expand Up @@ -58,7 +58,7 @@ def fingerprint_directory(self, path, extensions, nprocesses=None):

# don't refingerprint already fingerprinted files
if decoder.unique_hash(filename) in self.songhashes_set:
print "%s already fingerprinted, continuing..." % filename
print("%s already fingerprinted, continuing..." % filename)
continue

filenames_to_fingerprint.append(filename)
Expand Down Expand Up @@ -99,7 +99,7 @@ def fingerprint_file(self, filepath, song_name=None):
song_name = song_name or songname
# don't refingerprint already fingerprinted files
if song_hash in self.songhashes_set:
print "%s already fingerprinted, continuing..." % song_name
print("%s already fingerprinted, continuing..." % song_name)
else:
song_name, hashes, file_hash = _fingerprint_worker(
filepath,
Expand Down Expand Up @@ -199,4 +199,4 @@ def chunkify(lst, n):
Splits a list into roughly n equal parts.
http://stackoverflow.com/questions/2130016/splitting-a-list-of-arbitrary-size-into-only-roughly-n-equal-parts
"""
return [lst[i::n] for i in xrange(n)]
return [lst[i::n] for i in range(n)]
21 changes: 14 additions & 7 deletions dejavu/database_sql.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,18 @@
from __future__ import absolute_import
from itertools import izip_longest
import Queue
from itertools import zip_longest
import queue

try:
import pymysql
pymysql.install_as_MySQLdb()
except ImportError:
pass

import MySQLdb as mysql
from MySQLdb.cursors import DictCursor

from dejavu.database import Database


class SQLDatabase(Database):
"""
Queries:
Expand Down Expand Up @@ -291,6 +296,8 @@ def return_matches(self, hashes):

with self.cursor() as cur:
for split_values in grouper(values, 1000):
split_values = list(split_values)

# Create our IN part of the query
query = self.SELECT_MULTIPLE
query = query % ', '.join(['UNHEX(%s)'] * len(split_values))
Expand All @@ -312,7 +319,7 @@ def __setstate__(self, state):
def grouper(iterable, n, fillvalue=None):
args = [iter(iterable)] * n
return (filter(None, values) for values
in izip_longest(fillvalue=fillvalue, *args))
in zip_longest(fillvalue=fillvalue, *args))


def cursor_factory(**factory_options):
Expand All @@ -333,14 +340,14 @@ class Cursor(object):
cur.execute(query)
```
"""
_cache = Queue.Queue(maxsize=5)
_cache = queue.Queue(maxsize=5)

def __init__(self, cursor_type=mysql.cursors.Cursor, **options):
super(Cursor, self).__init__()

try:
conn = self._cache.get_nowait()
except Queue.Empty:
except queue.Empty:
conn = mysql.connect(**options)
else:
# Ping the connection before using it from the cache.
Expand Down Expand Up @@ -369,5 +376,5 @@ def __exit__(self, extype, exvalue, traceback):
# Put it back on the queue
try:
self._cache.put_nowait(self.conn)
except Queue.Full:
except queue.Full:
self.conn.close()
Loading